From feb057d0f046e50be8817d813d00961c8d254bf7 Mon Sep 17 00:00:00 2001 From: Steve Gordon Date: Thu, 4 May 2023 09:28:44 +0100 Subject: [PATCH] Misc cleanup (#34) * Add analyzer and editor config to ensure Async suffix is used * Rename Export to ExportAsync * Make field readonly * Simplify object creation * Remove whitespace * Add VS 2022 spell check settings * Fix for https://github.com/Microsoft/vs-threading/blob/main/doc/analyzers/VSTHRD105.md * Typo and formatting * Rename async methods * Ignore IDE0022 in code * Remove test word from exclusions * Fix whitespace * Convert to file-scoped namespaces --- .editorconfig | 6 + .../Elastic.Ingest.Apm.Example/Program.cs | 179 +++++----- exclusion.dic | 2 + src/Directory.Build.props | 4 + src/Elastic.Channels/BufferOptions.cs | 99 +++--- src/Elastic.Channels/BufferedChannelBase.cs | 45 +-- .../Buffers/OutboundBuffer.cs | 4 +- src/Elastic.Channels/ChannelOptionsBase.cs | 95 +++-- .../Diagnostics/ChannelDiagnosticsListener.cs | 2 +- .../Diagnostics/DiagnosticsBufferedChannel.cs | 10 +- .../Diagnostics/IChannelCallbacks.cs | 4 +- .../Diagnostics/NoopBufferedChannel.cs | 14 +- src/Elastic.Ingest.Apm/ApmChannel.cs | 163 +++++---- src/Elastic.Ingest.Apm/ApmChannelOptions.cs | 17 +- src/Elastic.Ingest.Apm/Helpers/Time.cs | 59 ++-- .../Model/IngestResponse.cs | 39 +-- src/Elastic.Ingest.Apm/Model/Transaction.cs | 263 +++++++------- .../DataStreams/DataStreamChannel.cs | 85 +++-- .../DataStreams/DataStreamChannelOptions.cs | 19 +- .../DataStreams/DataStreamName.cs | 97 +++-- .../ElasticsearchChannelBase.Bootstrap.cs | 331 +++++++++--------- .../ElasticsearchChannelBase.cs | 239 ++++++------- .../ElasticsearchChannelOptionsBase.cs | 49 ++- .../ElasticsearchChannelStatics.cs | 37 +- .../Indices/IndexChannel.cs | 81 +++-- .../Indices/IndexChannelOptions.cs | 81 +++-- .../IsExternalInit.cs | 7 +- .../Serialization/BulkOperationHeader.cs | 151 ++++---- .../Serialization/BulkResponse.cs | 84 +++-- .../Serialization/BulkResponseItem.cs | 121 ++++--- .../CustomActivityExporter.cs | 35 +- .../CustomOtlpTraceExporter.cs | 175 ++++----- .../TransportChannelBase.cs | 50 ++- .../TransportChannelOptionsBase.cs | 25 +- tests/Elastic.Channels.Tests/BehaviorTests.cs | 263 +++++++------- .../TroubleshootTests.cs | 121 ++++--- .../DataStreamIngestionTests.cs | 91 +++-- .../IndexIngestionTests.cs | 105 +++--- .../IngestionCluster.cs | 53 ++- .../BulkResponseBuilder.cs | 47 ++- .../ElasticsearchChannelTests.cs | 225 ++++++------ .../SerializationTests.cs | 21 +- .../Setup.cs | 157 +++++---- 43 files changed, 1869 insertions(+), 1886 deletions(-) create mode 100644 exclusion.dic diff --git a/.editorconfig b/.editorconfig index 935e327..e85c192 100644 --- a/.editorconfig +++ b/.editorconfig @@ -134,6 +134,12 @@ resharper_redundant_argument_default_value_highlighting=do_not_show # Do not penalize code that explicitly lists generic arguments resharper_redundant_type_arguments_of_method_highlighting=do_not_show +# Spell checker VS2022 +spelling_exclusion_path = .\exclusion.dic + +# Microsoft.VisualStudio.Threading.Analyzers +dotnet_diagnostic.VSTHRD200.severity = error + [Jenkinsfile] indent_style = space indent_size = 2 diff --git a/examples/Elastic.Ingest.Apm.Example/Program.cs b/examples/Elastic.Ingest.Apm.Example/Program.cs index f2a0873..259fa57 100644 --- a/examples/Elastic.Ingest.Apm.Example/Program.cs +++ b/examples/Elastic.Ingest.Apm.Example/Program.cs @@ -9,111 +9,110 @@ using Elastic.Ingest.Apm.Model; using Elastic.Transport; -namespace Elastic.Ingest.Apm.Example +namespace Elastic.Ingest.Apm.Example; + +internal class Program { - internal class Program + private static int _rejections; + private static int _requests; + private static int _responses; + private static int _retries; + private static int _maxRetriesExceeded; + private static Exception _exception; + + private static int Main(string[] args) { - private static int _rejections; - private static int _requests; - private static int _responses; - private static int _retries; - private static int _maxRetriesExceeded; - private static Exception _exception; - - private static int Main(string[] args) + if (args.Length != 2) { - if (args.Length != 2) - { - Console.Error.WriteLine("Please specify "); - return 1; - } - - var config = new TransportConfiguration(new Uri(args[0])) - .EnableDebugMode() - .Authentication(new ApiKey(args[1])); - //TODO needs - var transport = new DefaultHttpTransport(config); - - var numberOfEvents = 800; - var maxBufferSize = 200; - var handle = new CountdownEvent(numberOfEvents / maxBufferSize); - - var options = - new BufferOptions - { - ExportMaxConcurrency = 1, - OutboundBufferMaxSize = 200, - OutboundBufferMaxLifetime = TimeSpan.FromSeconds(10), - WaitHandle = handle, - ExportMaxRetries = 3, - ExportBackoffPeriod = times => TimeSpan.FromMilliseconds(1), - }; - var channelOptions = new ApmChannelOptions(transport) + Console.Error.WriteLine("Please specify "); + return 1; + } + + var config = new TransportConfiguration(new Uri(args[0])) + .EnableDebugMode() + .Authentication(new ApiKey(args[1])); + //TODO needs + var transport = new DefaultHttpTransport(config); + + var numberOfEvents = 800; + var maxBufferSize = 200; + var handle = new CountdownEvent(numberOfEvents / maxBufferSize); + + var options = + new BufferOptions { - BufferOptions = options, - ServerRejectionCallback = (list) => Interlocked.Increment(ref _rejections), - ExportItemsAttemptCallback = (c, a) => Interlocked.Increment(ref _requests), - ExportResponseCallback = (r, b) => - { - Interlocked.Increment(ref _responses); - Console.WriteLine(r.ApiCallDetails.DebugInformation); - }, - ExportBufferCallback = () => Console.WriteLine("Flushed"), - ExportMaxRetriesCallback = (list) => Interlocked.Increment(ref _maxRetriesExceeded), - ExportRetryCallback = (list) => Interlocked.Increment(ref _retries), - ExportExceptionCallback = (e) => _exception = e + ExportMaxConcurrency = 1, + OutboundBufferMaxSize = 200, + OutboundBufferMaxLifetime = TimeSpan.FromSeconds(10), + WaitHandle = handle, + ExportMaxRetries = 3, + ExportBackoffPeriod = times => TimeSpan.FromMilliseconds(1), }; - var channel = new ApmChannel(channelOptions); - - string Id() => RandomGenerator.GenerateRandomBytesAsString(8); - var random = new Random(); - for (var i = 0; i < numberOfEvents; i++) + var channelOptions = new ApmChannelOptions(transport) + { + BufferOptions = options, + ServerRejectionCallback = (list) => Interlocked.Increment(ref _rejections), + ExportItemsAttemptCallback = (c, a) => Interlocked.Increment(ref _requests), + ExportResponseCallback = (r, b) => { - channel.TryWrite(new Transaction("http", Id(), Id(), new SpanCount(), random.NextDouble() * random.Next(100, 1000) , Epoch.UtcNow) { Name = "x" }); - } - handle.Wait(TimeSpan.FromSeconds(20)); + Interlocked.Increment(ref _responses); + Console.WriteLine(r.ApiCallDetails.DebugInformation); + }, + ExportBufferCallback = () => Console.WriteLine("Flushed"), + ExportMaxRetriesCallback = (list) => Interlocked.Increment(ref _maxRetriesExceeded), + ExportRetryCallback = (list) => Interlocked.Increment(ref _retries), + ExportExceptionCallback = (e) => _exception = e + }; + var channel = new ApmChannel(channelOptions); - return 0; + string Id() => RandomGenerator.GenerateRandomBytesAsString(8); + var random = new Random(); + for (var i = 0; i < numberOfEvents; i++) + { + channel.TryWrite(new Transaction("http", Id(), Id(), new SpanCount(), random.NextDouble() * random.Next(100, 1000) , Epoch.UtcNow) { Name = "x" }); } - } + handle.Wait(TimeSpan.FromSeconds(20)); - internal static class RandomGenerator - { - [ThreadStatic] - private static Random _local; + return 0; + } +} - private static readonly Random Global = new Random(); +internal static class RandomGenerator +{ + [ThreadStatic] + private static Random _local; - internal static Random GetInstance() - { - var inst = _local; - if (inst == null) - { - int seed; - lock (Global) seed = Global.Next(); - _local = inst = new Random(seed); - } - return inst; - } + private static readonly Random Global = new Random(); - internal static void GenerateRandomBytes(byte[] bytes) => GetInstance().NextBytes(bytes); - - /// - /// Creates a random generated byte array hex encoded into a string. - /// - /// - /// The byte array that will be filled with a random number - this defines the length of the generated - /// random bits - /// - /// The random number hex encoded as string - internal static string GenerateRandomBytesAsString(byte[] bytes) + internal static Random GetInstance() + { + var inst = _local; + if (inst == null) { - GenerateRandomBytes(bytes); - return BitConverter.ToString(bytes).Replace("-", "").ToLowerInvariant(); + int seed; + lock (Global) seed = Global.Next(); + _local = inst = new Random(seed); } + return inst; + } - internal static string GenerateRandomBytesAsString(int numberOfBytes) => GenerateRandomBytesAsString(new byte[numberOfBytes]); + internal static void GenerateRandomBytes(byte[] bytes) => GetInstance().NextBytes(bytes); - internal static double GenerateRandomDoubleBetween0And1() => GetInstance().NextDouble(); + /// + /// Creates a random generated byte array hex encoded into a string. + /// + /// + /// The byte array that will be filled with a random number - this defines the length of the generated + /// random bits + /// + /// The random number hex encoded as string + internal static string GenerateRandomBytesAsString(byte[] bytes) + { + GenerateRandomBytes(bytes); + return BitConverter.ToString(bytes).Replace("-", "").ToLowerInvariant(); } + + internal static string GenerateRandomBytesAsString(int numberOfBytes) => GenerateRandomBytesAsString(new byte[numberOfBytes]); + + internal static double GenerateRandomDoubleBetween0And1() => GetInstance().NextDouble(); } diff --git a/exclusion.dic b/exclusion.dic new file mode 100644 index 0000000..ff95438 --- /dev/null +++ b/exclusion.dic @@ -0,0 +1,2 @@ +async +retryable \ No newline at end of file diff --git a/src/Directory.Build.props b/src/Directory.Build.props index b5d7b69..7018add 100644 --- a/src/Directory.Build.props +++ b/src/Directory.Build.props @@ -41,5 +41,9 @@ + + all + runtime; build; native; contentfiles; analyzers + diff --git a/src/Elastic.Channels/BufferOptions.cs b/src/Elastic.Channels/BufferOptions.cs index 6288828..4e70f24 100644 --- a/src/Elastic.Channels/BufferOptions.cs +++ b/src/Elastic.Channels/BufferOptions.cs @@ -5,57 +5,56 @@ using System; using System.Threading; -namespace Elastic.Channels +namespace Elastic.Channels; + +/// +/// Controls how data should be buffered in implementations +/// +public class BufferOptions { /// - /// Controls how data should be buffered in implementations + /// The maximum number of in flight instances that can be queued in memory. If this threshold is reached, events will be dropped + /// Defaults to 100_000 + /// + public int InboundBufferMaxSize { get; set; } = 100_000; + + /// + /// The maximum size to export to at once. + /// Defaults to 1_000 + /// + public int OutboundBufferMaxSize { get; set; } = 1_000; + + /// + /// The maximum lifetime of a buffer to export to . + /// If a buffer is older then the configured it will be flushed to + /// regardless of it's current size + /// Defaults to 5 seconds + /// + public TimeSpan OutboundBufferMaxLifetime { get; set; } = TimeSpan.FromSeconds(5); + + /// + /// The maximum number of consumers allowed to poll for new events on the channel. + /// Defaults to 1, increase to introduce concurrency. + /// + public int ExportMaxConcurrency { get; set; } = 1; + + /// + /// The times to retry an export if yields items to retry. + /// Whether or not items are selected for retrying depends on the actual channel implementation + /// to implement a backoff period of your choosing. + /// Defaults to 3, when yields any items + /// + public int ExportMaxRetries { get; set; } = 3; + + + /// + /// A function to calculate the backoff period, gets passed the number of retries attempted starting at 0. + /// By default backs off in increments of 2 seconds. + /// + public Func ExportBackoffPeriod { get; set; } = (i) => TimeSpan.FromSeconds(2 * (i + 1)); + + /// + /// Allows you to inject a to wait for N number of buffers to flush. /// - public class BufferOptions - { - /// - /// The maximum number of in flight instances that can be queued in memory. If this threshold is reached, events will be dropped - /// Defaults to 100_000 - /// - public int InboundBufferMaxSize { get; set; } = 100_000; - - /// - /// The maximum size to export to at once. - /// Defaults to 1_000 - /// - public int OutboundBufferMaxSize { get; set; } = 1_000; - - /// - /// The maximum lifetime of a buffer to export to . - /// If a buffer is older then the configured it will be flushed to - /// regardless of it's current size - /// Defaults to 5 seconds - /// - public TimeSpan OutboundBufferMaxLifetime { get; set; } = TimeSpan.FromSeconds(5); - - /// - /// The maximum number of consumers allowed to poll for new events on the channel. - /// Defaults to 1, increase to introduce concurrency. - /// - public int ExportMaxConcurrency { get; set; } = 1; - - /// - /// The times to retry an export if yields items to retry. - /// Whether or not items are selected for retrying depends on the actual channel implementation - /// to implement a backoff period of your choosing. - /// Defaults to 3, when yields any items - /// - public int ExportMaxRetries { get; set; } = 3; - - - /// - /// A function to calculate the backoff period, gets passed the number of retries attempted starting at 0. - /// By default backs off in increments of 2 seconds. - /// - public Func ExportBackoffPeriod { get; set; } = (i) => TimeSpan.FromSeconds(2 * (i + 1)); - - /// - /// Allows you to inject a to wait for N number of buffers to flush. - /// - public CountdownEvent? WaitHandle { get; set; } - } + public CountdownEvent? WaitHandle { get; set; } } diff --git a/src/Elastic.Channels/BufferedChannelBase.cs b/src/Elastic.Channels/BufferedChannelBase.cs index 5afe01d..51b89b3 100644 --- a/src/Elastic.Channels/BufferedChannelBase.cs +++ b/src/Elastic.Channels/BufferedChannelBase.cs @@ -33,11 +33,11 @@ public interface IBufferedChannel : IDisposable /// /// Waits for availability on the inbound channel before attempting to write each item in . /// - /// A bool indicating if all writes werwase successful + /// A bool indicating if all writes were successful Task WaitToWriteManyAsync(IEnumerable events, CancellationToken ctx = default); /// - /// Tries to write many to the channel returning true if ALL messages were written succesfully + /// Tries to write many to the channel returning true if ALL messages were written successfully /// bool TryWriteMany(IEnumerable events); @@ -51,8 +51,8 @@ public interface IBufferedChannel : IDisposable /// data from one to the other /// /// Concrete channel options implementation -/// The type of data we are looking to -/// The type of responses we are expecting to get back from +/// The type of data we are looking to +/// The type of responses we are expecting to get back from public abstract class BufferedChannelBase : ChannelWriter, IBufferedChannel where TChannelOptions : ChannelOptionsBase @@ -80,11 +80,11 @@ protected BufferedChannelBase(TChannelOptions options, ICollection (l is IChannelDiagnosticsListener c) ? c : null) - .FirstOrDefault(e=> e != null); + .FirstOrDefault(e => e != null); if (DiagnosticsListener == null && !options.DisableDiagnostics) { // if no debug listener was already provided but was requested explicitly create one. - var l = new ChannelDiagnosticsListener(GetType().Name); + var l = new ChannelDiagnosticsListener(GetType().Name); DiagnosticsListener = l; listeners = listeners.Concat(new[] { l }).ToArray(); } @@ -122,21 +122,24 @@ protected BufferedChannelBase(TChannelOptions options, ICollection(maxOut, BufferOptions.OutboundBufferMaxLifetime); - _outThread = Task.Factory.StartNew(async () => await ConsumeOutboundEvents().ConfigureAwait(false), - TaskCreationOptions.LongRunning | TaskCreationOptions.PreferFairness); - _inThread = Task.Factory.StartNew(async () => - await ConsumeInboundEvents(maxOut, BufferOptions.OutboundBufferMaxLifetime) - .ConfigureAwait(false) - , TaskCreationOptions.LongRunning | TaskCreationOptions.PreferFairness - ); + _outThread = Task.Factory.StartNew(async () => + await ConsumeOutboundEventsAsync().ConfigureAwait(false), + CancellationToken.None, + TaskCreationOptions.LongRunning | TaskCreationOptions.PreferFairness, + TaskScheduler.Default); + _inThread = Task.Factory.StartNew(async () => + await ConsumeInboundEventsAsync(maxOut, BufferOptions.OutboundBufferMaxLifetime).ConfigureAwait(false), + CancellationToken.None, + TaskCreationOptions.LongRunning | TaskCreationOptions.PreferFairness, + TaskScheduler.Default); } /// /// All subclasses of need to at a minimum /// implement this method to export buffered collection of /// - protected abstract Task Export(ArraySegment buffer, CancellationToken ctx = default); + protected abstract Task ExportAsync(ArraySegment buffer, CancellationToken ctx = default); /// The channel options currently in use public TChannelOptions Options { get; } @@ -197,7 +200,7 @@ public virtual async Task WaitToWriteAsync(TEvent item, CancellationToken { ctx = ctx == default ? TokenSource.Token : ctx; if (await InChannel.Writer.WaitToWriteAsync(ctx).ConfigureAwait(false) && - InChannel.Writer.TryWrite(item)) + InChannel.Writer.TryWrite(item)) { _callbacks.PublishToInboundChannelCallback?.Invoke(); return true; @@ -215,7 +218,7 @@ protected virtual ArraySegment RetryBuffer(TResponse response, IWriteTrackingBuffer statistics ) => EmptyArraySegments.Empty; - private async Task ConsumeOutboundEvents() + private async Task ConsumeOutboundEventsAsync() { _callbacks.OutboundChannelStartedCallback?.Invoke(); @@ -223,7 +226,7 @@ private async Task ConsumeOutboundEvents() var taskList = new List(maxConsumers); while (await OutChannel.Reader.WaitToReadAsync().ConfigureAwait(false)) - // ReSharper disable once RemoveRedundantBraces + // ReSharper disable once RemoveRedundantBraces { if (TokenSource.Token.IsCancellationRequested) break; if (_signal is { IsSet: true }) break; @@ -234,7 +237,7 @@ private async Task ConsumeOutboundEvents() { var items = buffer.GetArraySegment(); await _throttleTasks.WaitAsync().ConfigureAwait(false); - var t = ExportBuffer(items, buffer); + var t = ExportBufferAsync(items, buffer); taskList.Add(t); if (taskList.Count >= maxConsumers) @@ -250,7 +253,7 @@ private async Task ConsumeOutboundEvents() _callbacks.OutboundChannelExitedCallback?.Invoke(); } - private async Task ExportBuffer(ArraySegment items, IOutboundBuffer buffer) + private async Task ExportBufferAsync(ArraySegment items, IOutboundBuffer buffer) { var maxRetries = Options.BufferOptions.ExportMaxRetries; for (var i = 0; i <= maxRetries && items.Count > 0; i++) @@ -262,7 +265,7 @@ private async Task ExportBuffer(ArraySegment items, IOutboundBuffer items, IOutboundBuffer -/// The buffer to be exported over +/// The buffer to be exported over /// /// Due to change as we move this over to use ArrayPool public interface IOutboundBuffer : IWriteTrackingBuffer, IDisposable @@ -36,7 +36,7 @@ public OutboundBuffer(InboundBuffer buffer) ArrayItems = buffer.Reset(); } - public ArraySegment GetArraySegment() => new ArraySegment(ArrayItems, 0, Count); + public ArraySegment GetArraySegment() => new(ArrayItems, 0, Count); public void Dispose() => ArrayPool.Shared.Return(ArrayItems); } diff --git a/src/Elastic.Channels/ChannelOptionsBase.cs b/src/Elastic.Channels/ChannelOptionsBase.cs index a476d7f..41c8061 100644 --- a/src/Elastic.Channels/ChannelOptionsBase.cs +++ b/src/Elastic.Channels/ChannelOptionsBase.cs @@ -10,70 +10,69 @@ using Elastic.Channels.Buffers; using Elastic.Channels.Diagnostics; -namespace Elastic.Channels +namespace Elastic.Channels; + +/// +/// +/// +/// +/// +public abstract class ChannelOptionsBase : IChannelCallbacks { - /// - /// - /// - /// - /// - public abstract class ChannelOptionsBase : IChannelCallbacks - { - /// - public BufferOptions BufferOptions { get; set; } = new(); + /// + public BufferOptions BufferOptions { get; set; } = new(); - /// - /// Ensures a gets registered so this - /// implementation returns diagnostics in its implementation - /// - public bool DisableDiagnostics { get; set; } + /// + /// Ensures a gets registered so this + /// implementation returns diagnostics in its implementation + /// + public bool DisableDiagnostics { get; set; } - /// - /// Optionally provides a custom write implementation to a channel. Concrete channel implementations are not required to adhere to this config - /// - public Func? WriteEvent { get; set; } = null; + /// + /// Optionally provides a custom write implementation to a channel. Concrete channel implementations are not required to adhere to this config + /// + public Func? WriteEvent { get; set; } = null; - /// - public Action? ExportExceptionCallback { get; set; } + /// + public Action? ExportExceptionCallback { get; set; } - /// - public Action? ExportItemsAttemptCallback { get; set; } + /// + public Action? ExportItemsAttemptCallback { get; set; } - /// - public Action>? ExportMaxRetriesCallback { get; set; } + /// + public Action>? ExportMaxRetriesCallback { get; set; } - /// - public Action>? ExportRetryCallback { get; set; } + /// + public Action>? ExportRetryCallback { get; set; } - /// - public Action? ExportResponseCallback { get; set; } + /// + public Action? ExportResponseCallback { get; set; } - /// - public Action? ExportBufferCallback { get; set; } + /// + public Action? ExportBufferCallback { get; set; } - /// - public Action? ExportRetryableCountCallback { get; set; } + /// + public Action? ExportRetryableCountCallback { get; set; } - /// - public Action? PublishToInboundChannelCallback { get; set; } + /// + public Action? PublishToInboundChannelCallback { get; set; } - /// - public Action? PublishToInboundChannelFailureCallback { get; set; } + /// + public Action? PublishToInboundChannelFailureCallback { get; set; } - /// - public Action? PublishToOutboundChannelCallback { get; set; } + /// + public Action? PublishToOutboundChannelCallback { get; set; } - /// - public Action? OutboundChannelStartedCallback { get; set; } + /// + public Action? OutboundChannelStartedCallback { get; set; } - /// - public Action? OutboundChannelExitedCallback { get; set; } + /// + public Action? OutboundChannelExitedCallback { get; set; } - /// - public Action? InboundChannelStartedCallback { get; set; } + /// + public Action? InboundChannelStartedCallback { get; set; } - /// - public Action? PublishToOutboundChannelFailureCallback { get; set; } - } + /// + public Action? PublishToOutboundChannelFailureCallback { get; set; } } diff --git a/src/Elastic.Channels/Diagnostics/ChannelDiagnosticsListener.cs b/src/Elastic.Channels/Diagnostics/ChannelDiagnosticsListener.cs index d15d23f..9137448 100644 --- a/src/Elastic.Channels/Diagnostics/ChannelDiagnosticsListener.cs +++ b/src/Elastic.Channels/Diagnostics/ChannelDiagnosticsListener.cs @@ -15,7 +15,7 @@ namespace Elastic.Channels.Diagnostics; public interface IChannelDiagnosticsListener { /// - /// Keeps track of the first observed exception to calls to + /// Keeps track of the first observed exception to calls to /// public Exception? ObservedException { get; } diff --git a/src/Elastic.Channels/Diagnostics/DiagnosticsBufferedChannel.cs b/src/Elastic.Channels/Diagnostics/DiagnosticsBufferedChannel.cs index 0ff4516..12c3df2 100644 --- a/src/Elastic.Channels/Diagnostics/DiagnosticsBufferedChannel.cs +++ b/src/Elastic.Channels/Diagnostics/DiagnosticsBufferedChannel.cs @@ -11,8 +11,8 @@ namespace Elastic.Channels.Diagnostics; /// /// A NOOP implementation of that: -/// - tracks the number of times is invoked under -/// - observes the maximum concurrent calls to under +/// - tracks the number of times is invoked under +/// - observes the maximum concurrent calls to under /// - tracks how often the buffer does not match the export size or the export buffers segment does not start at the expected offset /// public class DiagnosticsBufferedChannel : NoopBufferedChannel @@ -33,8 +33,8 @@ public DiagnosticsBufferedChannel(NoopChannelOptions options, string? name = nul /// Keeps track of the number of times the buffer size or the buffer offset was off public long BufferMismatches => _bufferMismatches; - /// - protected override Task Export(ArraySegment buffer, CancellationToken ctx = default) + /// + protected override Task ExportAsync(ArraySegment buffer, CancellationToken ctx = default) { #if NETSTANDARD2_1 var b = buffer; @@ -51,7 +51,7 @@ protected override Task Export(ArraySegment buffer, Can Interlocked.Increment(ref _bufferMismatches); } - return base.Export(buffer, ctx); + return base.ExportAsync(buffer, ctx); } /// diff --git a/src/Elastic.Channels/Diagnostics/IChannelCallbacks.cs b/src/Elastic.Channels/Diagnostics/IChannelCallbacks.cs index f52eed7..d572d3c 100644 --- a/src/Elastic.Channels/Diagnostics/IChannelCallbacks.cs +++ b/src/Elastic.Channels/Diagnostics/IChannelCallbacks.cs @@ -15,7 +15,7 @@ namespace Elastic.Channels.Diagnostics; /// public interface IChannelCallbacks { - /// Called if the call to throws. + /// Called if the call to throws. Action? ExportExceptionCallback { get; } /// Called with (number of retries) (number of items to be exported) @@ -53,7 +53,7 @@ public interface IChannelCallbacks /// /// Called once after a buffer has been flushed, if the buffer is retried this callback is only called once - /// all retries have been exhausted. Its called regardless of whether the call to + /// all retries have been exhausted. Its called regardless of whether the call to /// succeeded. /// Action? ExportBufferCallback { get; } diff --git a/src/Elastic.Channels/Diagnostics/NoopBufferedChannel.cs b/src/Elastic.Channels/Diagnostics/NoopBufferedChannel.cs index 1d31602..fe2dbc4 100644 --- a/src/Elastic.Channels/Diagnostics/NoopBufferedChannel.cs +++ b/src/Elastic.Channels/Diagnostics/NoopBufferedChannel.cs @@ -11,8 +11,8 @@ namespace Elastic.Channels.Diagnostics; /// /// A NOOP implementation of that: -/// -tracks the number of times is invoked under -/// -observes the maximum concurrent calls to under +/// -tracks the number of times is invoked under +/// -observes the maximum concurrent calls to under /// public class NoopBufferedChannel : BufferedChannelBase @@ -30,7 +30,7 @@ public class NoopResponse { } /// Provides options how the should behave public class NoopChannelOptions : ChannelOptionsBase { - /// If set (defaults:false) will track the max observed concurrency to + /// If set (defaults:false) will track the max observed concurrency to public bool TrackConcurrency { get; set; } } @@ -50,18 +50,18 @@ public NoopBufferedChannel( } - /// Returns the number of times was called + /// Returns the number of times was called public long ExportedBuffers => _exportedBuffers; private long _exportedBuffers; - /// The maximum observed concurrency to calls to , requires to be set + /// The maximum observed concurrency to calls to , requires to be set public int ObservedConcurrency { get; private set; } private int _currentMax; - /// - protected override async Task Export(ArraySegment buffer, CancellationToken ctx = default) + /// + protected override async Task ExportAsync(ArraySegment buffer, CancellationToken ctx = default) { Interlocked.Increment(ref _exportedBuffers); if (!Options.TrackConcurrency) return new NoopResponse(); diff --git a/src/Elastic.Ingest.Apm/ApmChannel.cs b/src/Elastic.Ingest.Apm/ApmChannel.cs index 835f740..05b991f 100644 --- a/src/Elastic.Ingest.Apm/ApmChannel.cs +++ b/src/Elastic.Ingest.Apm/ApmChannel.cs @@ -16,110 +16,109 @@ using Elastic.Ingest.Transport; using Elastic.Transport; -namespace Elastic.Ingest.Apm -{ - internal static class ApmChannelStatics - { - public static readonly byte[] LineFeed = { (byte)'\n' }; +namespace Elastic.Ingest.Apm; - public static readonly DefaultRequestParameters RequestParams = new() - { - RequestConfiguration = new RequestConfiguration { ContentType = "application/x-ndjson" } - }; +internal static class ApmChannelStatics +{ + public static readonly byte[] LineFeed = { (byte)'\n' }; - public static readonly JsonSerializerOptions SerializerOptions = new() - { - DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, MaxDepth = 64, Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping, - }; - } + public static readonly DefaultRequestParameters RequestParams = new() + { + RequestConfiguration = new RequestConfiguration { ContentType = "application/x-ndjson" } + }; - /// - /// An implementation that sends V2 intake API data - /// to APM server. - /// - public class ApmChannel : TransportChannelBase + public static readonly JsonSerializerOptions SerializerOptions = new() { - /// - public ApmChannel(ApmChannelOptions options) : base(options) { } + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, MaxDepth = 64, Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping, + }; +} - //retry if APM server returns 429 - /// - protected override bool Retry(EventIntakeResponse response) => response.ApiCallDetails.HttpStatusCode == 429; +/// +/// An implementation that sends V2 intake API data +/// to APM server. +/// +public class ApmChannel : TransportChannelBase +{ + /// + public ApmChannel(ApmChannelOptions options) : base(options) { } - /// - protected override bool RetryAllItems(EventIntakeResponse response) => response.ApiCallDetails.HttpStatusCode == 429; + //retry if APM server returns 429 + /// + protected override bool Retry(EventIntakeResponse response) => response.ApiCallDetails.HttpStatusCode == 429; - //APM does not return the status for all events sent. Therefor we always return an empty set for individual items to retry - /// - protected override List<(IIntakeObject, IntakeErrorItem)> Zip(EventIntakeResponse response, IReadOnlyCollection page) => - _emptyZip; + /// + protected override bool RetryAllItems(EventIntakeResponse response) => response.ApiCallDetails.HttpStatusCode == 429; - private List<(IIntakeObject, IntakeErrorItem)> _emptyZip = new(); + //APM does not return the status for all events sent. Therefor we always return an empty set for individual items to retry + /// + protected override List<(IIntakeObject, IntakeErrorItem)> Zip(EventIntakeResponse response, IReadOnlyCollection page) => + _emptyZip; - /// - protected override bool RetryEvent((IIntakeObject, IntakeErrorItem) @event) => false; + private readonly List<(IIntakeObject, IntakeErrorItem)> _emptyZip = new(); - /// - protected override bool RejectEvent((IIntakeObject, IntakeErrorItem) @event) => false; + /// + protected override bool RetryEvent((IIntakeObject, IntakeErrorItem) @event) => false; - /// - protected override Task Export(HttpTransport transport, ArraySegment page, CancellationToken ctx = default) => - transport.RequestAsync(HttpMethod.POST, "/intake/v2/events", - PostData.StreamHandler(page, - (_, _) => - { - /* NOT USED */ - }, - async (b, stream, ctx) => { await WriteBufferToStreamAsync(b, stream, ctx).ConfigureAwait(false); }) - , ApmChannelStatics.RequestParams, ctx); + /// + protected override bool RejectEvent((IIntakeObject, IntakeErrorItem) @event) => false; - private async Task WriteStanzaToStreamAsync(Stream stream, CancellationToken ctx) + /// + protected override Task ExportAsync(HttpTransport transport, ArraySegment page, CancellationToken ctx = default) => + transport.RequestAsync(HttpMethod.POST, "/intake/v2/events", + PostData.StreamHandler(page, + (_, _) => + { + /* NOT USED */ + }, + async (b, stream, ctx) => { await WriteBufferToStreamAsync(b, stream, ctx).ConfigureAwait(false); }) + , ApmChannelStatics.RequestParams, ctx); + + private async Task WriteStanzaToStreamAsync(Stream stream, CancellationToken ctx) + { + // {"metadata":{"process":{"pid":1234,"title":"/usr/lib/jvm/java-10-openjdk-amd64/bin/java","ppid":1,"argv":["-v"]}, + // "system":{"architecture":"amd64","detected_hostname":"8ec7ceb99074","configured_hostname":"host1","platform":"Linux","container":{"id":"8ec7ceb990749e79b37f6dc6cd3628633618d6ce412553a552a0fa6b69419ad4"}, + // "kubernetes":{"namespace":"default","pod":{"uid":"b17f231da0ad128dc6c6c0b2e82f6f303d3893e3","name":"instrumented-java-service"},"node":{"name":"node-name"}}}, + // "service":{"name":"1234_service-12a3","version":"4.3.0","node":{"configured_name":"8ec7ceb990749e79b37f6dc6cd3628633618d6ce412553a552a0fa6b69419ad4"},"environment":"production","language":{"name":"Java","version":"10.0.2"}, + // "agent":{"version":"1.10.0","name":"java","ephemeral_id":"e71be9ac-93b0-44b9-a997-5638f6ccfc36"},"framework":{"name":"spring","version":"5.0.0"},"runtime":{"name":"Java","version":"10.0.2"}},"labels":{"group":"experimental","ab_testing":true,"segment":5}}} + // TODO cache + var p = Process.GetCurrentProcess(); + var metadata = new { - // {"metadata":{"process":{"pid":1234,"title":"/usr/lib/jvm/java-10-openjdk-amd64/bin/java","ppid":1,"argv":["-v"]}, - // "system":{"architecture":"amd64","detected_hostname":"8ec7ceb99074","configured_hostname":"host1","platform":"Linux","container":{"id":"8ec7ceb990749e79b37f6dc6cd3628633618d6ce412553a552a0fa6b69419ad4"}, - // "kubernetes":{"namespace":"default","pod":{"uid":"b17f231da0ad128dc6c6c0b2e82f6f303d3893e3","name":"instrumented-java-service"},"node":{"name":"node-name"}}}, - // "service":{"name":"1234_service-12a3","version":"4.3.0","node":{"configured_name":"8ec7ceb990749e79b37f6dc6cd3628633618d6ce412553a552a0fa6b69419ad4"},"environment":"production","language":{"name":"Java","version":"10.0.2"}, - // "agent":{"version":"1.10.0","name":"java","ephemeral_id":"e71be9ac-93b0-44b9-a997-5638f6ccfc36"},"framework":{"name":"spring","version":"5.0.0"},"runtime":{"name":"Java","version":"10.0.2"}},"labels":{"group":"experimental","ab_testing":true,"segment":5}}} - // TODO cache - var p = Process.GetCurrentProcess(); - var metadata = new + metadata = new { - metadata = new + process = new { pid = p.Id, title = p.ProcessName }, + service = new { - process = new { pid = p.Id, title = p.ProcessName }, - service = new - { - name = System.Text.RegularExpressions.Regex.Replace(p.ProcessName, "[^a-zA-Z0-9 _-]", "_"), - version = "1.0.0", - agent = new { name = "dotnet", version = "0.0.1" } - } + name = System.Text.RegularExpressions.Regex.Replace(p.ProcessName, "[^a-zA-Z0-9 _-]", "_"), + version = "1.0.0", + agent = new { name = "dotnet", version = "0.0.1" } } - }; - await JsonSerializer.SerializeAsync(stream, metadata, metadata.GetType(), ApmChannelStatics.SerializerOptions, ctx) - .ConfigureAwait(false); - await stream.WriteAsync(ApmChannelStatics.LineFeed, 0, 1, ctx).ConfigureAwait(false); - } + } + }; + await JsonSerializer.SerializeAsync(stream, metadata, metadata.GetType(), ApmChannelStatics.SerializerOptions, ctx) + .ConfigureAwait(false); + await stream.WriteAsync(ApmChannelStatics.LineFeed, 0, 1, ctx).ConfigureAwait(false); + } - private async Task WriteBufferToStreamAsync(IReadOnlyCollection b, Stream stream, CancellationToken ctx) + private async Task WriteBufferToStreamAsync(IReadOnlyCollection b, Stream stream, CancellationToken ctx) + { + await WriteStanzaToStreamAsync(stream, ctx).ConfigureAwait(false); + foreach (var @event in b) { - await WriteStanzaToStreamAsync(stream, ctx).ConfigureAwait(false); - foreach (var @event in b) - { - if (@event == null) continue; + if (@event == null) continue; - var type = @event switch - { - Transaction _ => "transaction", - _ => "unknown" - }; - var dictionary = new Dictionary() { { type, @event } }; + var type = @event switch + { + Transaction _ => "transaction", + _ => "unknown" + }; + var dictionary = new Dictionary() { { type, @event } }; - await JsonSerializer.SerializeAsync(stream, dictionary, dictionary.GetType(), ApmChannelStatics.SerializerOptions, ctx) - .ConfigureAwait(false); + await JsonSerializer.SerializeAsync(stream, dictionary, dictionary.GetType(), ApmChannelStatics.SerializerOptions, ctx) + .ConfigureAwait(false); - await stream.WriteAsync(ApmChannelStatics.LineFeed, 0, 1, ctx).ConfigureAwait(false); - } + await stream.WriteAsync(ApmChannelStatics.LineFeed, 0, 1, ctx).ConfigureAwait(false); } } } diff --git a/src/Elastic.Ingest.Apm/ApmChannelOptions.cs b/src/Elastic.Ingest.Apm/ApmChannelOptions.cs index 6085fc5..fb5f988 100644 --- a/src/Elastic.Ingest.Apm/ApmChannelOptions.cs +++ b/src/Elastic.Ingest.Apm/ApmChannelOptions.cs @@ -6,14 +6,13 @@ using Elastic.Ingest.Transport; using Elastic.Transport; -namespace Elastic.Ingest.Apm +namespace Elastic.Ingest.Apm; + +/// +/// Channel options for +/// +public class ApmChannelOptions : TransportChannelOptionsBase { - /// - /// Channel options for - /// - public class ApmChannelOptions : TransportChannelOptionsBase - { - /// - public ApmChannelOptions(HttpTransport transport) : base(transport) { } - } + /// + public ApmChannelOptions(HttpTransport transport) : base(transport) { } } diff --git a/src/Elastic.Ingest.Apm/Helpers/Time.cs b/src/Elastic.Ingest.Apm/Helpers/Time.cs index abe7b4c..0e65729 100644 --- a/src/Elastic.Ingest.Apm/Helpers/Time.cs +++ b/src/Elastic.Ingest.Apm/Helpers/Time.cs @@ -3,39 +3,38 @@ // See the LICENSE file in the project root for more information using System; -namespace Elastic.Ingest.Apm.Helpers -{ - /// - public static class Epoch - { - /// - /// DateTime.UnixEpoch Field does not exist in .NET Standard 2.0 - /// https://docs.microsoft.com/en-us/dotnet/api/system.datetime.unixepoch - /// - internal static readonly DateTime UnixEpochDateTime = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc); +namespace Elastic.Ingest.Apm.Helpers; - /// - public static long ToEpoch(this DateTime d) => ToTimestamp(d); - /// - public static long UtcNow => DateTime.UtcNow.ToEpoch(); +/// +public static class Epoch +{ + /// + /// DateTime.UnixEpoch Field does not exist in .NET Standard 2.0 + /// https://docs.microsoft.com/en-us/dotnet/api/system.datetime.unixepoch + /// + internal static readonly DateTime UnixEpochDateTime = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc); - /// - /// UTC based and formatted as microseconds since Unix epoch. - /// - /// - /// DateTime instance to convert to timestamp - its should be - /// - /// - /// UTC based and formatted as microseconds since Unix epoch - internal static long ToTimestamp(DateTime dateTimeToConvert) - { - if (dateTimeToConvert.Kind != DateTimeKind.Utc) - throw new ArgumentException($"{nameof(dateTimeToConvert)}'s Kind should be UTC but instead its Kind is {dateTimeToConvert.Kind}" + - $". {nameof(dateTimeToConvert)}'s value: {dateTimeToConvert}", nameof(dateTimeToConvert)); + /// + public static long ToEpoch(this DateTime d) => ToTimestamp(d); + /// + public static long UtcNow => DateTime.UtcNow.ToEpoch(); - return RoundTimeValue((dateTimeToConvert - UnixEpochDateTime).TotalMilliseconds * 1000); - } + /// + /// UTC based and formatted as microseconds since Unix epoch. + /// + /// + /// DateTime instance to convert to timestamp - its should be + /// + /// + /// UTC based and formatted as microseconds since Unix epoch + internal static long ToTimestamp(DateTime dateTimeToConvert) + { + if (dateTimeToConvert.Kind != DateTimeKind.Utc) + throw new ArgumentException($"{nameof(dateTimeToConvert)}'s Kind should be UTC but instead its Kind is {dateTimeToConvert.Kind}" + + $". {nameof(dateTimeToConvert)}'s value: {dateTimeToConvert}", nameof(dateTimeToConvert)); - internal static long RoundTimeValue(double value) => (long)Math.Round(value, MidpointRounding.AwayFromZero); + return RoundTimeValue((dateTimeToConvert - UnixEpochDateTime).TotalMilliseconds * 1000); } + + internal static long RoundTimeValue(double value) => (long)Math.Round(value, MidpointRounding.AwayFromZero); } diff --git a/src/Elastic.Ingest.Apm/Model/IngestResponse.cs b/src/Elastic.Ingest.Apm/Model/IngestResponse.cs index 473ecf7..d8393e5 100644 --- a/src/Elastic.Ingest.Apm/Model/IngestResponse.cs +++ b/src/Elastic.Ingest.Apm/Model/IngestResponse.cs @@ -5,30 +5,29 @@ using System.Text.Json.Serialization; using Elastic.Transport; -namespace Elastic.Ingest.Apm.Model +namespace Elastic.Ingest.Apm.Model; + +/// +public class EventIntakeResponse : TransportResponse { /// - public class EventIntakeResponse : TransportResponse - { - /// - [JsonPropertyName("accepted")] - public long Accepted { get; set; } + [JsonPropertyName("accepted")] + public long Accepted { get; set; } - /// - [JsonPropertyName("errors")] - //[JsonConverter(typeof(ResponseItemsConverter))] - public IReadOnlyCollection Errors { get; set; } = null!; - } + /// + [JsonPropertyName("errors")] + //[JsonConverter(typeof(ResponseItemsConverter))] + public IReadOnlyCollection Errors { get; set; } = null!; +} +/// +public class IntakeErrorItem +{ /// - public class IntakeErrorItem - { - /// - [JsonPropertyName("message")] - public string Message { get; set; } = null!; + [JsonPropertyName("message")] + public string Message { get; set; } = null!; - /// - [JsonPropertyName("document")] - public string Document { get; set; } = null!; - } + /// + [JsonPropertyName("document")] + public string Document { get; set; } = null!; } diff --git a/src/Elastic.Ingest.Apm/Model/Transaction.cs b/src/Elastic.Ingest.Apm/Model/Transaction.cs index 9da60dc..efab3b1 100644 --- a/src/Elastic.Ingest.Apm/Model/Transaction.cs +++ b/src/Elastic.Ingest.Apm/Model/Transaction.cs @@ -4,135 +4,135 @@ using System.Collections.Generic; using System.Text.Json.Serialization; -namespace Elastic.Ingest.Apm.Model +namespace Elastic.Ingest.Apm.Model; + +/// Marker interface for V2 intake objects +public interface IIntakeObject { } + +/// +/// An event corresponding to an incoming request or similar task occurring in a monitored +/// service +/// +public class Transaction : IIntakeObject { - /// Marker interface for V2 intake objects - public interface IIntakeObject { } + /// + public Transaction(string type, string id, string traceId, SpanCount spanCount, double duration, long timestamp) + { + Type = type; + Id = id; + TraceId = traceId; + SpanCount = spanCount; + Duration = duration; + Timestamp = timestamp; + } /// - /// An event corresponding to an incoming request or similar task occurring in a monitored - /// service + /// Recorded time of the event, UTC based and formatted as microseconds since Unix epoch /// - public class Transaction : IIntakeObject - { - /// - public Transaction(string type, string id, string traceId, SpanCount spanCount, double duration, long timestamp) - { - Type = type; - Id = id; - TraceId = traceId; - SpanCount = spanCount; - Duration = duration; - Timestamp = timestamp; - } - - /// - /// Recorded time of the event, UTC based and formatted as microseconds since Unix epoch - /// - [JsonPropertyName("timestamp")] - public long Timestamp { get; set; } - - /// - /// How long the transaction took to complete, in ms with 3 decimal points - /// - [JsonPropertyName("duration")] - public double? Duration { get; set; } - - /// - /// Hex encoded 64 random bits ID of the transaction. - /// - [JsonPropertyName("id")] - public string Id { get; set; } - - /// - /// Generic designation of a transaction in the scope of a single service (eg: 'GET - /// /users/:id') - /// - [JsonPropertyName("name")] - public string? Name { get; set; } - - /// - /// Hex encoded 64 random bits ID of the parent transaction or span. Only root transactions - /// of a trace do not have a parent_id, otherwise it needs to be set. - /// - [JsonPropertyName("parent_id")] - public string? ParentId { get; set; } - - /// - /// The result of the transaction. For HTTP-related transactions, this should be the status - /// code formatted like 'HTTP 2xx'. - /// - [JsonPropertyName("result")] - public string? Result { get; set; } - - /// - /// Sampling rate - /// - [JsonPropertyName("sample_rate")] - public double? SampleRate { get; set; } - - /// - /// Transactions that are 'sampled' will include all available information. Transactions that - /// are not sampled will not have 'spans' or 'context'. Defaults to true. - /// - [JsonPropertyName("sampled")] - public bool? Sampled { get; set; } - - /// - [JsonPropertyName("span_count")] - public SpanCount SpanCount { get; set; } - - /// - /// Hex encoded 128 random bits ID of the correlated trace. - /// - [JsonPropertyName("trace_id")] - public string TraceId { get; set; } - - /// - /// Keyword of specific relevance in the service's domain (eg: 'request', 'backgroundjob', - /// etc) - /// - [JsonPropertyName("type")] - public string Type { get; set; } - } + [JsonPropertyName("timestamp")] + public long Timestamp { get; set; } + + /// + /// How long the transaction took to complete, in ms with 3 decimal points + /// + [JsonPropertyName("duration")] + public double? Duration { get; set; } + + /// + /// Hex encoded 64 random bits ID of the transaction. + /// + [JsonPropertyName("id")] + public string Id { get; set; } + + /// + /// Generic designation of a transaction in the scope of a single service (eg: 'GET + /// /users/:id') + /// + [JsonPropertyName("name")] + public string? Name { get; set; } + + /// + /// Hex encoded 64 random bits ID of the parent transaction or span. Only root transactions + /// of a trace do not have a parent_id, otherwise it needs to be set. + /// + [JsonPropertyName("parent_id")] + public string? ParentId { get; set; } + + /// + /// The result of the transaction. For HTTP-related transactions, this should be the status + /// code formatted like 'HTTP 2xx'. + /// + [JsonPropertyName("result")] + public string? Result { get; set; } + + /// + /// Sampling rate + /// + [JsonPropertyName("sample_rate")] + public double? SampleRate { get; set; } + + /// + /// Transactions that are 'sampled' will include all available information. Transactions that + /// are not sampled will not have 'spans' or 'context'. Defaults to true. + /// + [JsonPropertyName("sampled")] + public bool? Sampled { get; set; } /// - public class Marks { } + [JsonPropertyName("span_count")] + public SpanCount SpanCount { get; set; } + + /// + /// Hex encoded 128 random bits ID of the correlated trace. + /// + [JsonPropertyName("trace_id")] + public string TraceId { get; set; } + + /// + /// Keyword of specific relevance in the service's domain (eg: 'request', 'backgroundjob', + /// etc) + /// + [JsonPropertyName("type")] + public string Type { get; set; } +} + +/// +public class Marks { } + +/// +public class SpanCount +{ + /// + /// Number of spans that have been dropped by the agent recording the transaction. + /// + [JsonPropertyName("dropped")] + public long? Dropped { get; set; } + + /// + /// Number of correlated spans that are recorded. + /// + [JsonPropertyName("started")] + public long Started { get; set; } +} +/// + public class Span : IIntakeObject +{ /// - public class SpanCount + public Span(string type, string name, string id, string traceId, string parentId, long timestamp) { - /// - /// Number of spans that have been dropped by the agent recording the transaction. - /// - [JsonPropertyName("dropped")] - public long? Dropped { get; set; } - - /// - /// Number of correlated spans that are recorded. - /// - [JsonPropertyName("started")] - public long Started { get; set; } + Type = type; + Name = name; + Id = id; + TraceId = traceId; + ParentId = parentId; + Timestamp = timestamp; } - /// - public class Span : IIntakeObject - { - /// - public Span(string type, string name, string id, string traceId, string parentId, long timestamp) - { - Type = type; - Name = name; - Id = id; - TraceId = traceId; - ParentId = parentId; - Timestamp = timestamp; - } - - /// - /// Recorded time of the event, UTC based and formatted as microseconds since Unix epoch - /// - public long Timestamp { get; set; } + /// + /// Recorded time of the event, UTC based and formatted as microseconds since Unix epoch + /// + public long Timestamp { get; set; } /// /// The specific kind of event within the sub-type represented by the span (e.g. query, @@ -140,17 +140,17 @@ public Span(string type, string name, string id, string traceId, string parentId /// public string? Action { get; set; } - /// + /// /// List of successor transactions and/or spans. /// public List? ChildIds { get; set; } - /// + /// /// Any other arbitrary data captured by the agent, optionally provided by the user /// public Context? Context { get; set; } - /// + /// /// Duration of the span in milliseconds /// public double? Duration { get; set; } @@ -180,7 +180,7 @@ public Span(string type, string name, string id, string traceId, string parentId /// public List? Stacktrace { get; set; } - /// + /// /// Offset relative to the transaction's timestamp identifying the start of the span, in /// milliseconds /// @@ -191,7 +191,7 @@ public Span(string type, string name, string id, string traceId, string parentId /// public string? Subtype { get; set; } - /// + /// /// Indicates whether the span was executed synchronously or asynchronously. /// public bool? Sync { get; set; } @@ -206,13 +206,13 @@ public Span(string type, string name, string id, string traceId, string parentId /// public string? TransactionId { get; set; } - /// + /// /// Keyword of specific relevance in the service's domain (eg: 'db', 'template', etc) /// public string Type { get; set; } } - /// +/// public class Context { /// @@ -238,7 +238,7 @@ public class Context public ContextService? Service { get; set; } } - /// +/// public class Db { /// @@ -273,7 +273,7 @@ public class Db public string? User { get; set; } } - /// +/// public class Destination { /// @@ -293,7 +293,7 @@ public class Destination public DestinationService? Service { get; set; } } - /// +/// public class DestinationService { /// @@ -315,7 +315,7 @@ public class DestinationService public string? Type { get; set; } } - /// +/// public class Http { /// @@ -334,7 +334,7 @@ public class Http public string? Url { get; set; } } - /// +/// public class ContextService { /// @@ -348,7 +348,7 @@ public class ContextService public string? Name { get; set; } } - /// +/// public class Agent { /// @@ -366,4 +366,3 @@ public class Agent /// public string? Version { get; set; } } -} diff --git a/src/Elastic.Ingest.Elasticsearch/DataStreams/DataStreamChannel.cs b/src/Elastic.Ingest.Elasticsearch/DataStreams/DataStreamChannel.cs index da2e6bd..9ae956a 100644 --- a/src/Elastic.Ingest.Elasticsearch/DataStreams/DataStreamChannel.cs +++ b/src/Elastic.Ingest.Elasticsearch/DataStreams/DataStreamChannel.cs @@ -9,41 +9,41 @@ using Elastic.Ingest.Elasticsearch.Serialization; using Elastic.Ingest.Transport; -namespace Elastic.Ingest.Elasticsearch.DataStreams +namespace Elastic.Ingest.Elasticsearch.DataStreams; + +/// A channel to push messages to Elasticsearch data streams +public class DataStreamChannel : ElasticsearchChannelBase> { - /// A channel to push messages to Elasticsearch data streams - public class DataStreamChannel : ElasticsearchChannelBase> - { - private readonly CreateOperation _fixedHeader; + private readonly CreateOperation _fixedHeader; - /// - public DataStreamChannel(DataStreamChannelOptions options) : this(options, null) { } + /// + public DataStreamChannel(DataStreamChannelOptions options) : this(options, null) { } - /// - public DataStreamChannel(DataStreamChannelOptions options, ICollection>? callbackListeners) : base(options, callbackListeners) - { - var target = Options.DataStream.ToString(); - _fixedHeader = new CreateOperation { Index = target }; - } + /// + public DataStreamChannel(DataStreamChannelOptions options, ICollection>? callbackListeners) : base(options, callbackListeners) + { + var target = Options.DataStream.ToString(); + _fixedHeader = new CreateOperation { Index = target }; + } - /// - protected override BulkOperationHeader CreateBulkOperationHeader(TEvent @event) => _fixedHeader; + /// + protected override BulkOperationHeader CreateBulkOperationHeader(TEvent @event) => _fixedHeader; - /// - protected override string TemplateName => Options.DataStream.GetTemplateName(); - /// - protected override string TemplateWildcard => Options.DataStream.GetNamespaceWildcard(); + /// + protected override string TemplateName => Options.DataStream.GetTemplateName(); + /// + protected override string TemplateWildcard => Options.DataStream.GetNamespaceWildcard(); - /// - /// Gets a default index template for the current - /// - /// A tuple of (name, body) describing the index template - protected override (string, string) GetDefaultIndexTemplate(string name, string match, string mappingsName, string settingsName) - { - var additionalComponents = GetInferredComponentTemplates(); - var additionalComponentsJson = string.Join(", ", additionalComponents.Select(a => $"\"{a}\"")); + /// + /// Gets a default index template for the current + /// + /// A tuple of (name, body) describing the index template + protected override (string, string) GetDefaultIndexTemplate(string name, string match, string mappingsName, string settingsName) + { + var additionalComponents = GetInferredComponentTemplates(); + var additionalComponentsJson = string.Join(", ", additionalComponents.Select(a => $"\"{a}\"")); - var indexTemplateBody = @$"{{ + var indexTemplateBody = @$"{{ ""index_patterns"": [""{match}""], ""data_stream"": {{ }}, ""composed_of"": [ ""{mappingsName}"", ""{settingsName}"", {additionalComponentsJson} ], @@ -53,21 +53,20 @@ protected override (string, string) GetDefaultIndexTemplate(string name, string ""assembly_version"": ""{LibraryVersion.Current}"" }} }}"; - return (name, indexTemplateBody); - } + return (name, indexTemplateBody); + } - /// - /// Yields additional component templates to include in the index template based on the data stream naming scheme - /// - protected List GetInferredComponentTemplates() - { - var additionalComponents = new List { "data-streams-mappings" }; - // if we know the type of data is logs or metrics apply certain defaults that Elasticsearch ships with. - if (Options.DataStream.Type.ToLowerInvariant() == "logs") - additionalComponents.AddRange(new[] { "logs-settings", "logs-mappings" }); - else if (Options.DataStream.Type.ToLowerInvariant() == "metrics") - additionalComponents.AddRange(new[] { "metrics-settings", "metrics-mappings" }); - return additionalComponents; - } + /// + /// Yields additional component templates to include in the index template based on the data stream naming scheme + /// + protected List GetInferredComponentTemplates() + { + var additionalComponents = new List { "data-streams-mappings" }; + // if we know the type of data is logs or metrics apply certain defaults that Elasticsearch ships with. + if (Options.DataStream.Type.ToLowerInvariant() == "logs") + additionalComponents.AddRange(new[] { "logs-settings", "logs-mappings" }); + else if (Options.DataStream.Type.ToLowerInvariant() == "metrics") + additionalComponents.AddRange(new[] { "metrics-settings", "metrics-mappings" }); + return additionalComponents; } } diff --git a/src/Elastic.Ingest.Elasticsearch/DataStreams/DataStreamChannelOptions.cs b/src/Elastic.Ingest.Elasticsearch/DataStreams/DataStreamChannelOptions.cs index 4a731d3..8f6a5f2 100644 --- a/src/Elastic.Ingest.Elasticsearch/DataStreams/DataStreamChannelOptions.cs +++ b/src/Elastic.Ingest.Elasticsearch/DataStreams/DataStreamChannelOptions.cs @@ -3,16 +3,15 @@ // See the LICENSE file in the project root for more information using Elastic.Transport; -namespace Elastic.Ingest.Elasticsearch.DataStreams +namespace Elastic.Ingest.Elasticsearch.DataStreams; + +/// Controls which data stream the channel should write to +public class DataStreamChannelOptions : ElasticsearchChannelOptionsBase { - /// Controls which data stream the channel should write to - public class DataStreamChannelOptions : ElasticsearchChannelOptionsBase - { - /// - public DataStreamChannelOptions(HttpTransport transport) : base(transport) => - DataStream = new DataStreamName(typeof(TEvent).Name.ToLowerInvariant()); + /// + public DataStreamChannelOptions(HttpTransport transport) : base(transport) => + DataStream = new DataStreamName(typeof(TEvent).Name.ToLowerInvariant()); - /// - public DataStreamName DataStream { get; set; } - } + /// + public DataStreamName DataStream { get; set; } } diff --git a/src/Elastic.Ingest.Elasticsearch/DataStreams/DataStreamName.cs b/src/Elastic.Ingest.Elasticsearch/DataStreams/DataStreamName.cs index 60fe433..3ad3ece 100644 --- a/src/Elastic.Ingest.Elasticsearch/DataStreams/DataStreamName.cs +++ b/src/Elastic.Ingest.Elasticsearch/DataStreams/DataStreamName.cs @@ -4,56 +4,55 @@ using System; using System.Linq; -namespace Elastic.Ingest.Elasticsearch.DataStreams +namespace Elastic.Ingest.Elasticsearch.DataStreams; + +/// +/// Strongly types a reference to a data stream using Elastic's data stream naming scheme +/// +public record DataStreamName { - /// - /// Strongly types a reference to a data stream using Elastic's data stream naming scheme - /// - public record DataStreamName + /// Generic type describing the data + public string Type { get; init; } + + /// Describes the data ingested and its structure + public string DataSet { get; init; } + + /// User-configurable arbitrary grouping + public string Namespace { get; init; } + + private static readonly char[] BadCharacters = { '\\', '/', '*', '?', '"', '<', '>', '|', ' ', ',', '#' }; + private static readonly string BadCharactersError = string.Join(", ", BadCharacters.Select(c => $"'{c}'").ToArray()); + + /// + public DataStreamName(string type, string dataSet = "generic", string @namespace = "default") { - /// Generic type describing the data - public string Type { get; init; } - - /// Describes the data ingested and its structure - public string DataSet { get; init; } - - /// User-configurable arbitrary grouping - public string Namespace { get; init; } - - private static readonly char[] BadCharacters = { '\\', '/', '*', '?', '"', '<', '>', '|', ' ', ',', '#' }; - private static readonly string BadCharactersError = string.Join(", ", BadCharacters.Select(c => $"'{c}'").ToArray()); - - /// - public DataStreamName(string type, string dataSet = "generic", string @namespace = "default") - { - if (string.IsNullOrEmpty(type)) throw new ArgumentException($"{nameof(type)} can not be null or empty", nameof(type)); - if (string.IsNullOrEmpty(dataSet)) throw new ArgumentException($"{nameof(dataSet)} can not be null or empty", nameof(dataSet)); - if (string.IsNullOrEmpty(@namespace)) throw new ArgumentException($"{nameof(@namespace)} can not be null or empty", nameof(@namespace)); - if (type.IndexOfAny(BadCharacters) > 0) - throw new ArgumentException($"{nameof(type)} can not contain any of {BadCharactersError}", nameof(type)); - if (dataSet.IndexOfAny(BadCharacters) > 0) - throw new ArgumentException($"{nameof(dataSet)} can not contain any of {BadCharactersError}", nameof(type)); - if (@namespace.IndexOfAny(BadCharacters) > 0) - throw new ArgumentException($"{nameof(@namespace)} can not contain any of {BadCharactersError}", nameof(type)); - - Type = type.ToLowerInvariant(); - DataSet = dataSet.ToLowerInvariant(); - Namespace = @namespace.ToLowerInvariant(); - } - - /// Returns a good index template name for this data stream - public string GetTemplateName() => $"{Type}-{DataSet}"; - /// Returns a good index template wildcard match for this data stream - public string GetNamespaceWildcard() => $"{Type}-{DataSet}-*"; - - private string? _stringValue; - /// > - public override string ToString() - { - if (_stringValue != null) return _stringValue; - - _stringValue = $"{Type}-{DataSet}-{Namespace}"; - return _stringValue; - } + if (string.IsNullOrEmpty(type)) throw new ArgumentException($"{nameof(type)} can not be null or empty", nameof(type)); + if (string.IsNullOrEmpty(dataSet)) throw new ArgumentException($"{nameof(dataSet)} can not be null or empty", nameof(dataSet)); + if (string.IsNullOrEmpty(@namespace)) throw new ArgumentException($"{nameof(@namespace)} can not be null or empty", nameof(@namespace)); + if (type.IndexOfAny(BadCharacters) > 0) + throw new ArgumentException($"{nameof(type)} can not contain any of {BadCharactersError}", nameof(type)); + if (dataSet.IndexOfAny(BadCharacters) > 0) + throw new ArgumentException($"{nameof(dataSet)} can not contain any of {BadCharactersError}", nameof(type)); + if (@namespace.IndexOfAny(BadCharacters) > 0) + throw new ArgumentException($"{nameof(@namespace)} can not contain any of {BadCharactersError}", nameof(type)); + + Type = type.ToLowerInvariant(); + DataSet = dataSet.ToLowerInvariant(); + Namespace = @namespace.ToLowerInvariant(); + } + + /// Returns a good index template name for this data stream + public string GetTemplateName() => $"{Type}-{DataSet}"; + /// Returns a good index template wildcard match for this data stream + public string GetNamespaceWildcard() => $"{Type}-{DataSet}-*"; + + private string? _stringValue; + /// > + public override string ToString() + { + if (_stringValue != null) return _stringValue; + + _stringValue = $"{Type}-{DataSet}-{Namespace}"; + return _stringValue; } } diff --git a/src/Elastic.Ingest.Elasticsearch/ElasticsearchChannelBase.Bootstrap.cs b/src/Elastic.Ingest.Elasticsearch/ElasticsearchChannelBase.Bootstrap.cs index 2775ea2..aaadd54 100644 --- a/src/Elastic.Ingest.Elasticsearch/ElasticsearchChannelBase.Bootstrap.cs +++ b/src/Elastic.Ingest.Elasticsearch/ElasticsearchChannelBase.Bootstrap.cs @@ -7,160 +7,160 @@ using Elastic.Ingest.Transport; using Elastic.Transport; -namespace Elastic.Ingest.Elasticsearch +namespace Elastic.Ingest.Elasticsearch; + +public abstract partial class ElasticsearchChannelBase { - public abstract partial class ElasticsearchChannelBase + /// The index template name should register. + protected abstract string TemplateName { get; } + /// The index template wildcard the should register for its index template. + protected abstract string TemplateWildcard { get; } + + /// + /// Returns a minimal default index template for an implementation + /// + /// A tuple of (name, body) describing the index template + protected abstract (string, string) GetDefaultIndexTemplate(string name, string match, string mappingsName, string settingsName); + + + /// + /// Bootstrap the target data stream. Will register the appropriate index and component templates + /// + /// Either None (no bootstrapping), Silent (quiet exit), Failure (throw exceptions) + /// Registers a component template that ensures the template is managed by this ilm policy + /// + public virtual async Task BootstrapElasticsearchAsync(BootstrapMethod bootstrapMethod, string? ilmPolicy = null, CancellationToken ctx = default) + { + if (bootstrapMethod == BootstrapMethod.None) return true; + + var name = TemplateName; + var match = TemplateWildcard; + if (await IndexTemplateExistsAsync(name, ctx).ConfigureAwait(false)) return false; + + var (settingsName, settingsBody) = GetDefaultComponentSettings(name, ilmPolicy); + if (!await PutComponentTemplateAsync(bootstrapMethod, settingsName, settingsBody, ctx).ConfigureAwait(false)) + return false; + + var (mappingsName, mappingsBody) = GetDefaultComponentMappings(name); + if (!await PutComponentTemplateAsync(bootstrapMethod, mappingsName, mappingsBody, ctx).ConfigureAwait(false)) + return false; + + var (indexTemplateName, indexTemplateBody) = GetDefaultIndexTemplate(name, match, mappingsName, settingsName); + if (!await PutIndexTemplateAsync(bootstrapMethod, indexTemplateName, indexTemplateBody, ctx).ConfigureAwait(false)) + return false; + + return true; + } + + /// + /// Bootstrap the target data stream. Will register the appropriate index and component templates + /// + /// Either None (no bootstrapping), Silent (quiet exit), Failure (throw exceptions) + /// Registers a component template that ensures the template is managed by this ilm policy + public virtual bool BootstrapElasticsearch(BootstrapMethod bootstrapMethod, string? ilmPolicy = null) + { + if (bootstrapMethod == BootstrapMethod.None) return true; + + var name = TemplateName; + var match = TemplateWildcard; + if (IndexTemplateExists(name)) return false; + + var (settingsName, settingsBody) = GetDefaultComponentSettings(name, ilmPolicy); + if (!PutComponentTemplate(bootstrapMethod, settingsName, settingsBody)) + return false; + + var (mappingsName, mappingsBody) = GetDefaultComponentMappings(name); + if (!PutComponentTemplate(bootstrapMethod, mappingsName, mappingsBody)) + return false; + + var (indexTemplateName, indexTemplateBody) = GetDefaultIndexTemplate(name, match, mappingsName, settingsName); + if (!PutIndexTemplate(bootstrapMethod, indexTemplateName, indexTemplateBody)) + return false; + + return true; + } + + /// + protected bool IndexTemplateExists(string name) + { + var templateExists = Options.Transport.Request(HttpMethod.HEAD, $"_index_template/{name}"); + var statusCode = templateExists.ApiCallDetails.HttpStatusCode; + return statusCode is 200; + } + + /// + protected async Task IndexTemplateExistsAsync(string name, CancellationToken ctx = default) { - /// The index template name should register. - protected abstract string TemplateName { get; } - /// The index template wildcard the should register for its index template. - protected abstract string TemplateWildcard { get; } - - /// - /// Returns a minimal default index template for an implementation - /// - /// A tuple of (name, body) describing the index template - protected abstract (string, string) GetDefaultIndexTemplate(string name, string match, string mappingsName, string settingsName); - - - /// - /// Bootstrap the target data stream. Will register the appropriate index and component templates - /// - /// Either None (no bootstrapping), Silent (quiet exit), Failure (throw exceptions) - /// Registers a component template that ensures the template is managed by this ilm policy - /// - public virtual async Task BootstrapElasticsearchAsync(BootstrapMethod bootstrapMethod, string? ilmPolicy = null, CancellationToken ctx = default) - { - if (bootstrapMethod == BootstrapMethod.None) return true; - - var name = TemplateName; - var match = TemplateWildcard; - if (await IndexTemplateExistsAsync(name, ctx).ConfigureAwait(false)) return false; - - var (settingsName, settingsBody) = GetDefaultComponentSettings(name, ilmPolicy); - if (!await PutComponentTemplateAsync(bootstrapMethod, settingsName, settingsBody, ctx).ConfigureAwait(false)) - return false; - - var (mappingsName, mappingsBody) = GetDefaultComponentMappings(name); - if (!await PutComponentTemplateAsync(bootstrapMethod, mappingsName, mappingsBody, ctx).ConfigureAwait(false)) - return false; - - var (indexTemplateName, indexTemplateBody) = GetDefaultIndexTemplate(name, match, mappingsName, settingsName); - if (!await PutIndexTemplateAsync(bootstrapMethod, indexTemplateName, indexTemplateBody, ctx).ConfigureAwait(false)) - return false; - - return true; - } - - /// - /// Bootstrap the target data stream. Will register the appropriate index and component templates - /// - /// Either None (no bootstrapping), Silent (quiet exit), Failure (throw exceptions) - /// Registers a component template that ensures the template is managed by this ilm policy - public virtual bool BootstrapElasticsearch(BootstrapMethod bootstrapMethod, string? ilmPolicy = null) - { - if (bootstrapMethod == BootstrapMethod.None) return true; - - var name = TemplateName; - var match = TemplateWildcard; - if (IndexTemplateExists(name)) return false; - - var (settingsName, settingsBody) = GetDefaultComponentSettings(name, ilmPolicy); - if (!PutComponentTemplate(bootstrapMethod, settingsName, settingsBody)) - return false; - - var (mappingsName, mappingsBody) = GetDefaultComponentMappings(name); - if (!PutComponentTemplate(bootstrapMethod, mappingsName, mappingsBody)) - return false; - - var (indexTemplateName, indexTemplateBody) = GetDefaultIndexTemplate(name, match, mappingsName, settingsName); - if (!PutIndexTemplate(bootstrapMethod, indexTemplateName, indexTemplateBody)) - return false; - - return true; - } - - /// - protected bool IndexTemplateExists(string name) - { - var templateExists = Options.Transport.Request(HttpMethod.HEAD, $"_index_template/{name}"); - var statusCode = templateExists.ApiCallDetails.HttpStatusCode; - return statusCode is 200; - } - - /// - protected async Task IndexTemplateExistsAsync(string name, CancellationToken ctx = default) - { - var templateExists = await Options.Transport.RequestAsync - (HttpMethod.HEAD, $"_index_template/{name}", cancellationToken: ctx) - .ConfigureAwait(false); - var statusCode = templateExists.ApiCallDetails.HttpStatusCode; - return statusCode is 200; - } - - /// - protected bool PutIndexTemplate(BootstrapMethod bootstrapMethod, string name, string body) - { - var putIndexTemplateResponse = Options.Transport.Request - (HttpMethod.PUT, $"_index_template/{name}", PostData.String(body)); - if (putIndexTemplateResponse.ApiCallDetails.HasSuccessfulStatusCode) return true; - - return bootstrapMethod == BootstrapMethod.Silent - ? false - : throw new Exception( - $"Failure to create index templates for {TemplateWildcard}: {putIndexTemplateResponse}"); - } - - /// - protected async Task PutIndexTemplateAsync(BootstrapMethod bootstrapMethod, string name, string body, CancellationToken ctx = default) - { - var putIndexTemplateResponse = await Options.Transport.RequestAsync - (HttpMethod.PUT, $"_index_template/{name}", PostData.String(body), cancellationToken: ctx) - .ConfigureAwait(false); - if (putIndexTemplateResponse.ApiCallDetails.HasSuccessfulStatusCode) return true; - - return bootstrapMethod == BootstrapMethod.Silent - ? false - : throw new Exception( - $"Failure to create index templates for {TemplateWildcard}: {putIndexTemplateResponse}"); - } - - /// - protected bool PutComponentTemplate(BootstrapMethod bootstrapMethod, string name, string body) - { - var putComponentTemplate = Options.Transport.Request - (HttpMethod.PUT, $"_component_template/{name}", PostData.String(body)); - if (putComponentTemplate.ApiCallDetails.HasSuccessfulStatusCode) return true; - - return bootstrapMethod == BootstrapMethod.Silent - ? false - : throw new Exception( - $"Failure to create component template `${name}` for {TemplateWildcard}: {putComponentTemplate}"); - } - - /// - protected async Task PutComponentTemplateAsync(BootstrapMethod bootstrapMethod, string name, string body, CancellationToken ctx = default) - { - var putComponentTemplate = await Options.Transport.RequestAsync - (HttpMethod.PUT, $"_component_template/{name}", PostData.String(body), cancellationToken: ctx) - .ConfigureAwait(false); - if (putComponentTemplate.ApiCallDetails.HasSuccessfulStatusCode) return true; - - return bootstrapMethod == BootstrapMethod.Silent - ? false - : throw new Exception( - $"Failure to create component template `${name}` for {TemplateWildcard}: {putComponentTemplate}"); - } - - /// - /// Returns default component settings template for a - /// - /// A tuple of (name, body) describing the default component template settings - protected (string, string) GetDefaultComponentSettings(string indexTemplateName, string? ilmPolicy = null) - { - if (string.IsNullOrWhiteSpace(ilmPolicy)) ilmPolicy = "logs"; - var settingsName = $"{indexTemplateName}-settings"; - var settingsBody = $@"{{ + var templateExists = await Options.Transport.RequestAsync + (HttpMethod.HEAD, $"_index_template/{name}", cancellationToken: ctx) + .ConfigureAwait(false); + var statusCode = templateExists.ApiCallDetails.HttpStatusCode; + return statusCode is 200; + } + + /// + protected bool PutIndexTemplate(BootstrapMethod bootstrapMethod, string name, string body) + { + var putIndexTemplateResponse = Options.Transport.Request + (HttpMethod.PUT, $"_index_template/{name}", PostData.String(body)); + if (putIndexTemplateResponse.ApiCallDetails.HasSuccessfulStatusCode) return true; + + return bootstrapMethod == BootstrapMethod.Silent + ? false + : throw new Exception( + $"Failure to create index templates for {TemplateWildcard}: {putIndexTemplateResponse}"); + } + + /// + protected async Task PutIndexTemplateAsync(BootstrapMethod bootstrapMethod, string name, string body, CancellationToken ctx = default) + { + var putIndexTemplateResponse = await Options.Transport.RequestAsync + (HttpMethod.PUT, $"_index_template/{name}", PostData.String(body), cancellationToken: ctx) + .ConfigureAwait(false); + if (putIndexTemplateResponse.ApiCallDetails.HasSuccessfulStatusCode) return true; + + return bootstrapMethod == BootstrapMethod.Silent + ? false + : throw new Exception( + $"Failure to create index templates for {TemplateWildcard}: {putIndexTemplateResponse}"); + } + + /// + protected bool PutComponentTemplate(BootstrapMethod bootstrapMethod, string name, string body) + { + var putComponentTemplate = Options.Transport.Request + (HttpMethod.PUT, $"_component_template/{name}", PostData.String(body)); + if (putComponentTemplate.ApiCallDetails.HasSuccessfulStatusCode) return true; + + return bootstrapMethod == BootstrapMethod.Silent + ? false + : throw new Exception( + $"Failure to create component template `${name}` for {TemplateWildcard}: {putComponentTemplate}"); + } + + /// + protected async Task PutComponentTemplateAsync(BootstrapMethod bootstrapMethod, string name, string body, CancellationToken ctx = default) + { + var putComponentTemplate = await Options.Transport.RequestAsync + (HttpMethod.PUT, $"_component_template/{name}", PostData.String(body), cancellationToken: ctx) + .ConfigureAwait(false); + if (putComponentTemplate.ApiCallDetails.HasSuccessfulStatusCode) return true; + + return bootstrapMethod == BootstrapMethod.Silent + ? false + : throw new Exception( + $"Failure to create component template `${name}` for {TemplateWildcard}: {putComponentTemplate}"); + } + + /// + /// Returns default component settings template for a + /// + /// A tuple of (name, body) describing the default component template settings + protected (string, string) GetDefaultComponentSettings(string indexTemplateName, string? ilmPolicy = null) + { + if (string.IsNullOrWhiteSpace(ilmPolicy)) ilmPolicy = "logs"; + var settingsName = $"{indexTemplateName}-settings"; + var settingsBody = $@"{{ ""template"": {{ ""settings"": {{ ""index.lifecycle.name"": ""{ilmPolicy}"" @@ -171,17 +171,17 @@ protected async Task PutComponentTemplateAsync(BootstrapMethod bootstrapMe ""assembly_version"": ""{LibraryVersion.Current}"" }} }}"; - return (settingsName, settingsBody); - } - - /// - /// Returns a minimal default mapping component settings template for a - /// - /// A tuple of (name, body) describing the default component template mappings - protected (string, string) GetDefaultComponentMappings(string indexTemplateName) - { - var settingsName = $"{indexTemplateName}-mappings"; - var settingsBody = $@"{{ + return (settingsName, settingsBody); + } + + /// + /// Returns a minimal default mapping component settings template for a + /// + /// A tuple of (name, body) describing the default component template mappings + protected (string, string) GetDefaultComponentMappings(string indexTemplateName) + { + var settingsName = $"{indexTemplateName}-mappings"; + var settingsBody = $@"{{ ""template"": {{ ""mappings"": {{ }} @@ -191,8 +191,7 @@ protected async Task PutComponentTemplateAsync(BootstrapMethod bootstrapMe ""assembly_version"": ""{LibraryVersion.Current}"" }} }}"; - return (settingsName, settingsBody); - } - + return (settingsName, settingsBody); } + } diff --git a/src/Elastic.Ingest.Elasticsearch/ElasticsearchChannelBase.cs b/src/Elastic.Ingest.Elasticsearch/ElasticsearchChannelBase.cs index f711178..2bff2eb 100644 --- a/src/Elastic.Ingest.Elasticsearch/ElasticsearchChannelBase.cs +++ b/src/Elastic.Ingest.Elasticsearch/ElasticsearchChannelBase.cs @@ -19,160 +19,161 @@ using Elastic.Transport; using static Elastic.Ingest.Elasticsearch.ElasticsearchChannelStatics; -namespace Elastic.Ingest.Elasticsearch +namespace Elastic.Ingest.Elasticsearch; + +/// +/// An abstract base class for both and +/// Coordinates most of the sending to- and bootstrapping of Elasticsearch +/// +public abstract partial class ElasticsearchChannelBase + : TransportChannelBase + where TChannelOptions : ElasticsearchChannelOptionsBase { - /// - /// An abstract base class for both and - /// Coordinates most of the sending to- and bootstrapping of Elasticsearch - /// - public abstract partial class ElasticsearchChannelBase - : TransportChannelBase - where TChannelOptions : ElasticsearchChannelOptionsBase - { - /// - protected ElasticsearchChannelBase(TChannelOptions options, ICollection>? callbackListeners) - : base(options, callbackListeners) { } + /// + protected ElasticsearchChannelBase(TChannelOptions options, ICollection>? callbackListeners) + : base(options, callbackListeners) { } - /// - protected ElasticsearchChannelBase(TChannelOptions options) - : base(options) { } + /// + protected ElasticsearchChannelBase(TChannelOptions options) + : base(options) { } - /// - protected override bool Retry(BulkResponse response) - { - var details = response.ApiCallDetails; - if (!details.HasSuccessfulStatusCode) - Options.ExportExceptionCallback?.Invoke(new Exception(details.ToString(), details.OriginalException)); - return details.HasSuccessfulStatusCode; - } + /// + protected override bool Retry(BulkResponse response) + { + var details = response.ApiCallDetails; + if (!details.HasSuccessfulStatusCode) + Options.ExportExceptionCallback?.Invoke(new Exception(details.ToString(), details.OriginalException)); + return details.HasSuccessfulStatusCode; + } - /// - protected override bool RetryAllItems(BulkResponse response) => response.ApiCallDetails.HttpStatusCode == 429; + /// + protected override bool RetryAllItems(BulkResponse response) => response.ApiCallDetails.HttpStatusCode == 429; - /// - protected override List<(TEvent, BulkResponseItem)> Zip(BulkResponse response, IReadOnlyCollection page) => - page.Zip(response.Items, (doc, item) => (doc, item)).ToList(); + /// + protected override List<(TEvent, BulkResponseItem)> Zip(BulkResponse response, IReadOnlyCollection page) => + page.Zip(response.Items, (doc, item) => (doc, item)).ToList(); - /// - protected override bool RetryEvent((TEvent, BulkResponseItem) @event) => - RetryStatusCodes.Contains(@event.Item2.Status); + /// + protected override bool RetryEvent((TEvent, BulkResponseItem) @event) => + RetryStatusCodes.Contains(@event.Item2.Status); - /// - protected override bool RejectEvent((TEvent, BulkResponseItem) @event) => - @event.Item2.Status < 200 || @event.Item2.Status > 300; + /// + protected override bool RejectEvent((TEvent, BulkResponseItem) @event) => + @event.Item2.Status < 200 || @event.Item2.Status > 300; - /// - protected override Task Export(HttpTransport transport, ArraySegment page, CancellationToken ctx = default) - { + /// + protected override Task ExportAsync(HttpTransport transport, ArraySegment page, CancellationToken ctx = default) + { #if NETSTANDARD2_1 - // Option is obsolete to prevent external users to set it. + // Option is obsolete to prevent external users to set it. #pragma warning disable CS0618 - if (Options.UseReadOnlyMemory) + if (Options.UseReadOnlyMemory) #pragma warning restore CS0618 - { - var bytes = GetBytes(page); - return transport.RequestAsync(HttpMethod.POST, "/_bulk", PostData.ReadOnlyMemory(bytes), RequestParams, ctx); - } -#endif - return transport.RequestAsync(HttpMethod.POST, "/_bulk", - PostData.StreamHandler(page, - (_, _) => - { - /* NOT USED */ - }, - async (b, stream, ctx) => { await WriteBufferToStreamAsync(b, stream, ctx).ConfigureAwait(false); }) - , RequestParams, ctx); + { + var bytes = GetBytes(page); + return transport.RequestAsync(HttpMethod.POST, "/_bulk", PostData.ReadOnlyMemory(bytes), RequestParams, ctx); } +#endif +#pragma warning disable IDE0022 // Use expression body for method + return transport.RequestAsync(HttpMethod.POST, "/_bulk", + PostData.StreamHandler(page, + (_, _) => + { + /* NOT USED */ + }, + async (b, stream, ctx) => { await WriteBufferToStreamAsync(b, stream, ctx).ConfigureAwait(false); }) + , RequestParams, ctx); +#pragma warning restore IDE0022 // Use expression body for method + } - /// - /// Asks implementations to create a based on the being exported. - /// - protected abstract BulkOperationHeader CreateBulkOperationHeader(TEvent @event); + /// + /// Asks implementations to create a based on the being exported. + /// + protected abstract BulkOperationHeader CreateBulkOperationHeader(TEvent @event); #if NETSTANDARD2_1_OR_GREATER - private ReadOnlyMemory GetBytes(ArraySegment page) + private ReadOnlyMemory GetBytes(ArraySegment page) + { + // ArrayBufferWriter inserts comma's when serializing multiple times + // Hence the multiple writer.Resets() as advised on this feature request + // https://github.com/dotnet/runtime/issues/82314 + var bufferWriter = new ArrayBufferWriter(); + using var writer = new Utf8JsonWriter(bufferWriter, WriterOptions); + foreach (var @event in page.AsSpan()) { - // ArrayBufferWriter inserts comma's when serializing multiple times - // Hence the multiple writer.Resets() as advised on this feature request - // https://github.com/dotnet/runtime/issues/82314 - var bufferWriter = new ArrayBufferWriter(); - using var writer = new Utf8JsonWriter(bufferWriter, WriterOptions); - foreach (var @event in page.AsSpan()) + var indexHeader = CreateBulkOperationHeader(@event); + JsonSerializer.Serialize(writer, indexHeader, indexHeader.GetType(), SerializerOptions); + bufferWriter.Write(LineFeed); + writer.Reset(); + + if (indexHeader is UpdateOperation) { - var indexHeader = CreateBulkOperationHeader(@event); - JsonSerializer.Serialize(writer, indexHeader, indexHeader.GetType(), SerializerOptions); - bufferWriter.Write(LineFeed); + bufferWriter.Write(DocUpdateHeaderStart); writer.Reset(); + } - if (indexHeader is UpdateOperation) - { - bufferWriter.Write(DocUpdateHeaderStart); - writer.Reset(); - } - - if (Options.EventWriter?.WriteToArrayBuffer != null) - Options.EventWriter.WriteToArrayBuffer(bufferWriter, @event); - else - JsonSerializer.Serialize(writer, @event, SerializerOptions); - writer.Reset(); + if (Options.EventWriter?.WriteToArrayBuffer != null) + Options.EventWriter.WriteToArrayBuffer(bufferWriter, @event); + else + JsonSerializer.Serialize(writer, @event, SerializerOptions); + writer.Reset(); - if (indexHeader is UpdateOperation) - { - bufferWriter.Write(DocUpdateHeaderEnd); - writer.Reset(); - } - - bufferWriter.Write(LineFeed); + if (indexHeader is UpdateOperation) + { + bufferWriter.Write(DocUpdateHeaderEnd); writer.Reset(); } - return bufferWriter.WrittenMemory; + + bufferWriter.Write(LineFeed); + writer.Reset(); } + return bufferWriter.WrittenMemory; + } #endif - private async Task WriteBufferToStreamAsync(ArraySegment b, Stream stream, CancellationToken ctx) - { + private async Task WriteBufferToStreamAsync(ArraySegment b, Stream stream, CancellationToken ctx) + { #if NETSTANDARD2_1_OR_GREATER - var items = b; + var items = b; #else - // needs cast prior to netstandard2.0 - IReadOnlyList items = b; + // needs cast prior to netstandard2.0 + IReadOnlyList items = b; #endif - // for is okay on ArraySegment, foreach performs bad: - // https://antao-almada.medium.com/how-to-use-span-t-and-memory-t-c0b126aae652 - // ReSharper disable once ForCanBeConvertedToForeach - for (var i = 0; i < items.Count; i++) - { - var @event = items[i]; - if (@event == null) continue; + // for is okay on ArraySegment, foreach performs bad: + // https://antao-almada.medium.com/how-to-use-span-t-and-memory-t-c0b126aae652 + // ReSharper disable once ForCanBeConvertedToForeach + for (var i = 0; i < items.Count; i++) + { + var @event = items[i]; + if (@event == null) continue; - var indexHeader = CreateBulkOperationHeader(@event); - await JsonSerializer.SerializeAsync(stream, indexHeader, indexHeader.GetType(), SerializerOptions, ctx) - .ConfigureAwait(false); - await stream.WriteAsync(LineFeed, 0, 1, ctx).ConfigureAwait(false); + var indexHeader = CreateBulkOperationHeader(@event); + await JsonSerializer.SerializeAsync(stream, indexHeader, indexHeader.GetType(), SerializerOptions, ctx) + .ConfigureAwait(false); + await stream.WriteAsync(LineFeed, 0, 1, ctx).ConfigureAwait(false); - if (indexHeader is UpdateOperation) - await stream.WriteAsync(DocUpdateHeaderStart, 0, DocUpdateHeaderStart.Length, ctx).ConfigureAwait(false); + if (indexHeader is UpdateOperation) + await stream.WriteAsync(DocUpdateHeaderStart, 0, DocUpdateHeaderStart.Length, ctx).ConfigureAwait(false); - if (Options.EventWriter?.WriteToStreamAsync != null) - await Options.EventWriter.WriteToStreamAsync(stream, @event, ctx).ConfigureAwait(false); - else - await JsonSerializer.SerializeAsync(stream, @event, SerializerOptions, ctx) - .ConfigureAwait(false); + if (Options.EventWriter?.WriteToStreamAsync != null) + await Options.EventWriter.WriteToStreamAsync(stream, @event, ctx).ConfigureAwait(false); + else + await JsonSerializer.SerializeAsync(stream, @event, SerializerOptions, ctx) + .ConfigureAwait(false); - if (indexHeader is UpdateOperation) - await stream.WriteAsync(DocUpdateHeaderEnd, 0, DocUpdateHeaderEnd.Length, ctx).ConfigureAwait(false); + if (indexHeader is UpdateOperation) + await stream.WriteAsync(DocUpdateHeaderEnd, 0, DocUpdateHeaderEnd.Length, ctx).ConfigureAwait(false); - await stream.WriteAsync(LineFeed, 0, 1, ctx).ConfigureAwait(false); - } + await stream.WriteAsync(LineFeed, 0, 1, ctx).ConfigureAwait(false); } + } - /// - protected class HeadIndexTemplateResponse : TransportResponse { } + /// + protected class HeadIndexTemplateResponse : TransportResponse { } - /// - protected class PutIndexTemplateResponse : TransportResponse { } + /// + protected class PutIndexTemplateResponse : TransportResponse { } - /// - protected class PutComponentTemplateResponse : TransportResponse { } - } + /// + protected class PutComponentTemplateResponse : TransportResponse { } } diff --git a/src/Elastic.Ingest.Elasticsearch/ElasticsearchChannelOptionsBase.cs b/src/Elastic.Ingest.Elasticsearch/ElasticsearchChannelOptionsBase.cs index 0f87f44..ea21867 100644 --- a/src/Elastic.Ingest.Elasticsearch/ElasticsearchChannelOptionsBase.cs +++ b/src/Elastic.Ingest.Elasticsearch/ElasticsearchChannelOptionsBase.cs @@ -7,34 +7,33 @@ using Elastic.Ingest.Transport; using Elastic.Transport; -namespace Elastic.Ingest.Elasticsearch +namespace Elastic.Ingest.Elasticsearch; + +/// +/// Base options implementation for implementations +/// +public abstract class ElasticsearchChannelOptionsBase : TransportChannelOptionsBase { + /// + protected ElasticsearchChannelOptionsBase(HttpTransport transport) : base(transport) { } + /// - /// Base options implementation for implementations + /// Export option, Optionally provide a custom write implementation for /// - public abstract class ElasticsearchChannelOptionsBase : TransportChannelOptionsBase - { - /// - protected ElasticsearchChannelOptionsBase(HttpTransport transport) : base(transport) { } + public IElasticsearchEventWriter? EventWriter { get; set; } - /// - /// Export option, Optionally provide a custom write implementation for - /// - public IElasticsearchEventWriter? EventWriter { get; set; } - - #if NETSTANDARD2_1_OR_GREATER - /// - /// Expert option, - /// This will eagerly serialize to and use . - /// If false (default) the channel will use to directly write to the stream. - /// - #else - /// - /// Expert option, only available in netstandard2.1+ compatible runtimes to evaluate serialization approaches - /// - #endif - [Obsolete("Temporary exposed expert option, used to evaluate two different approaches to serialization")] - public bool UseReadOnlyMemory { get; set; } + #if NETSTANDARD2_1_OR_GREATER + /// + /// Expert option, + /// This will eagerly serialize to and use . + /// If false (default) the channel will use to directly write to the stream. + /// + #else + /// + /// Expert option, only available in netstandard2.1+ compatible runtimes to evaluate serialization approaches + /// + #endif + [Obsolete("Temporary exposed expert option, used to evaluate two different approaches to serialization")] + public bool UseReadOnlyMemory { get; set; } - } } diff --git a/src/Elastic.Ingest.Elasticsearch/ElasticsearchChannelStatics.cs b/src/Elastic.Ingest.Elasticsearch/ElasticsearchChannelStatics.cs index 56bbacc..13a431a 100644 --- a/src/Elastic.Ingest.Elasticsearch/ElasticsearchChannelStatics.cs +++ b/src/Elastic.Ingest.Elasticsearch/ElasticsearchChannelStatics.cs @@ -9,29 +9,28 @@ using System.Text.Json.Serialization; using Elastic.Transport; -namespace Elastic.Ingest.Elasticsearch +namespace Elastic.Ingest.Elasticsearch; + +internal class ElasticsearchRequestParameters : RequestParameters { } +internal static class ElasticsearchChannelStatics { - internal class ElasticsearchRequestParameters : RequestParameters { } - internal static class ElasticsearchChannelStatics - { - public static readonly byte[] LineFeed = { (byte)'\n' }; + public static readonly byte[] LineFeed = { (byte)'\n' }; - public static readonly byte[] DocUpdateHeaderStart = Encoding.UTF8.GetBytes("{\"doc_as_upsert\": true, \"doc\": "); - public static readonly byte[] DocUpdateHeaderEnd = Encoding.UTF8.GetBytes(" }"); + public static readonly byte[] DocUpdateHeaderStart = Encoding.UTF8.GetBytes("{\"doc_as_upsert\": true, \"doc\": "); + public static readonly byte[] DocUpdateHeaderEnd = Encoding.UTF8.GetBytes(" }"); - public static readonly ElasticsearchRequestParameters RequestParams = - new() { QueryString = { { "filter_path", "error, items.*.status,items.*.error" } } }; + public static readonly ElasticsearchRequestParameters RequestParams = + new() { QueryString = { { "filter_path", "error, items.*.status,items.*.error" } } }; - public static readonly HashSet RetryStatusCodes = new(new[] { 502, 503, 504, 429 }); + public static readonly HashSet RetryStatusCodes = new(new[] { 502, 503, 504, 429 }); - public static readonly JsonSerializerOptions SerializerOptions = new () - { - DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingDefault, - Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping, - }; + public static readonly JsonSerializerOptions SerializerOptions = new () + { + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingDefault, + Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping, + }; - public static readonly JsonWriterOptions WriterOptions = - // SkipValidation as we write ndjson - new() { Encoder = SerializerOptions.Encoder, Indented = SerializerOptions.WriteIndented, SkipValidation = true}; - } + public static readonly JsonWriterOptions WriterOptions = + // SkipValidation as we write ndjson + new() { Encoder = SerializerOptions.Encoder, Indented = SerializerOptions.WriteIndented, SkipValidation = true}; } diff --git a/src/Elastic.Ingest.Elasticsearch/Indices/IndexChannel.cs b/src/Elastic.Ingest.Elasticsearch/Indices/IndexChannel.cs index 7ae2fe3..06c3f4d 100644 --- a/src/Elastic.Ingest.Elasticsearch/Indices/IndexChannel.cs +++ b/src/Elastic.Ingest.Elasticsearch/Indices/IndexChannel.cs @@ -8,51 +8,51 @@ using Elastic.Ingest.Elasticsearch.Serialization; using Elastic.Ingest.Transport; -namespace Elastic.Ingest.Elasticsearch.Indices +namespace Elastic.Ingest.Elasticsearch.Indices; + +/// A channel to push messages to an Elasticsearch index +/// If unsure prefer to use +/// +public class IndexChannel : ElasticsearchChannelBase> { - /// A channel to push messages to an Elasticsearch index - /// If unsure prefer to use - /// - public class IndexChannel : ElasticsearchChannelBase> - { - /// - public IndexChannel(IndexChannelOptions options) : this(options, null) { } + /// + public IndexChannel(IndexChannelOptions options) : this(options, null) { } - /// - public IndexChannel(IndexChannelOptions options, ICollection>? callbackListeners) : base(options, callbackListeners) - { - TemplateName = string.Format(Options.IndexFormat, "template"); - TemplateWildcard = string.Format(Options.IndexFormat, "*"); - } + /// + public IndexChannel(IndexChannelOptions options, ICollection>? callbackListeners) : base(options, callbackListeners) + { + TemplateName = string.Format(Options.IndexFormat, "template"); + TemplateWildcard = string.Format(Options.IndexFormat, "*"); + } - /// - protected override BulkOperationHeader CreateBulkOperationHeader(TEvent @event) - { - var indexTime = Options.TimestampLookup?.Invoke(@event) ?? DateTimeOffset.Now; - if (Options.IndexOffset.HasValue) indexTime = indexTime.ToOffset(Options.IndexOffset.Value); + /// + protected override BulkOperationHeader CreateBulkOperationHeader(TEvent @event) + { + var indexTime = Options.TimestampLookup?.Invoke(@event) ?? DateTimeOffset.Now; + if (Options.IndexOffset.HasValue) indexTime = indexTime.ToOffset(Options.IndexOffset.Value); - var index = string.Format(Options.IndexFormat, indexTime); - var id = Options.BulkOperationIdLookup?.Invoke(@event); - if (!string.IsNullOrWhiteSpace(id) && id != null && (Options.BulkUpsertLookup?.Invoke(@event, id) ?? false)) - return new UpdateOperation { Id = id, Index = index }; - return - !string.IsNullOrWhiteSpace(id) - ? new IndexOperation { Index = index, Id = id } - : new CreateOperation { Index = index }; - } + var index = string.Format(Options.IndexFormat, indexTime); + var id = Options.BulkOperationIdLookup?.Invoke(@event); + if (!string.IsNullOrWhiteSpace(id) && id != null && (Options.BulkUpsertLookup?.Invoke(@event, id) ?? false)) + return new UpdateOperation { Id = id, Index = index }; + return + !string.IsNullOrWhiteSpace(id) + ? new IndexOperation { Index = index, Id = id } + : new CreateOperation { Index = index }; + } - /// - protected override string TemplateName { get; } - /// - protected override string TemplateWildcard { get; } + /// + protected override string TemplateName { get; } + /// + protected override string TemplateWildcard { get; } - /// - /// Gets a default index template for the current - /// - /// A tuple of (name, body) describing the index template - protected override (string, string) GetDefaultIndexTemplate(string name, string match, string mappingsName, string settingsName) - { - var indexTemplateBody = @$"{{ + /// + /// Gets a default index template for the current + /// + /// A tuple of (name, body) describing the index template + protected override (string, string) GetDefaultIndexTemplate(string name, string match, string mappingsName, string settingsName) + { + var indexTemplateBody = @$"{{ ""index_patterns"": [""{match}""], ""composed_of"": [ ""{mappingsName}"", ""{settingsName}"" ], ""priority"": 201, @@ -61,7 +61,6 @@ protected override (string, string) GetDefaultIndexTemplate(string name, string ""assembly_version"": ""{LibraryVersion.Current}"" }} }}"; - return (name, indexTemplateBody); - } + return (name, indexTemplateBody); } } diff --git a/src/Elastic.Ingest.Elasticsearch/Indices/IndexChannelOptions.cs b/src/Elastic.Ingest.Elasticsearch/Indices/IndexChannelOptions.cs index 7c8d73e..5175b17 100644 --- a/src/Elastic.Ingest.Elasticsearch/Indices/IndexChannelOptions.cs +++ b/src/Elastic.Ingest.Elasticsearch/Indices/IndexChannelOptions.cs @@ -4,52 +4,51 @@ using System; using Elastic.Transport; -namespace Elastic.Ingest.Elasticsearch.Indices +namespace Elastic.Ingest.Elasticsearch.Indices; + +/// +/// Provides options to to control how and where data gets written to Elasticsearch +/// +/// +public class IndexChannelOptions : ElasticsearchChannelOptionsBase { + /// + public IndexChannelOptions(HttpTransport transport) : base(transport) { } + /// - /// Provides options to to control how and where data gets written to Elasticsearch + /// Gets or sets the format string for the Elastic search index. The current DateTimeOffset is passed as parameter + /// 0. + /// Defaults to "dotnet-{0:yyyy.MM.dd}" + /// If no {0} parameter is defined the index name is effectively fixed /// - /// - public class IndexChannelOptions : ElasticsearchChannelOptionsBase - { - /// - public IndexChannelOptions(HttpTransport transport) : base(transport) { } - - /// - /// Gets or sets the format string for the Elastic search index. The current DateTimeOffset is passed as parameter - /// 0. - /// Defaults to "dotnet-{0:yyyy.MM.dd}" - /// If no {0} parameter is defined the index name is effectively fixed - /// - public string IndexFormat { get; set; } = "dotnet-{0:yyyy.MM.dd}"; + public string IndexFormat { get; set; } = "dotnet-{0:yyyy.MM.dd}"; - /// - /// Gets or sets the offset to use for the index DateTimeOffset. Default value is null, which uses the system local - /// offset. Use "00:00" for UTC. - /// - public TimeSpan? IndexOffset { get; set; } + /// + /// Gets or sets the offset to use for the index DateTimeOffset. Default value is null, which uses the system local + /// offset. Use "00:00" for UTC. + /// + public TimeSpan? IndexOffset { get; set; } - /// - /// Provide a per document DateTimeOffset to be used as the date passed as parameter 0 to - /// - public Func? TimestampLookup { get; set; } + /// + /// Provide a per document DateTimeOffset to be used as the date passed as parameter 0 to + /// + public Func? TimestampLookup { get; set; } - /// - /// If the document provides an Id this allows you to set a per document `_id`. - /// If an `_id` is defined an `_index` bulk operation will be created. - /// Otherwise (the default) `_create` bulk operation will be issued for the document. - /// Read more about bulk operations here: - /// https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html#bulk-api-request-body - /// - public Func? BulkOperationIdLookup { get; set; } + /// + /// If the document provides an Id this allows you to set a per document `_id`. + /// If an `_id` is defined an `_index` bulk operation will be created. + /// Otherwise (the default) `_create` bulk operation will be issued for the document. + /// Read more about bulk operations here: + /// https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html#bulk-api-request-body + /// + public Func? BulkOperationIdLookup { get; set; } - /// - /// Uses the callback provided to to determine if this is in fact an update operation - /// If this returns true the document will be sent as an upsert operation - /// Otherwise (the default) `index` bulk operation will be issued for the document. - /// Read more about bulk operations here: - /// https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html#bulk-api-request-body - /// - public Func? BulkUpsertLookup { get; set; } - } + /// + /// Uses the callback provided to to determine if this is in fact an update operation + /// If this returns true the document will be sent as an upsert operation + /// Otherwise (the default) `index` bulk operation will be issued for the document. + /// Read more about bulk operations here: + /// https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html#bulk-api-request-body + /// + public Func? BulkUpsertLookup { get; set; } } diff --git a/src/Elastic.Ingest.Elasticsearch/IsExternalInit.cs b/src/Elastic.Ingest.Elasticsearch/IsExternalInit.cs index 15850eb..3db5b78 100644 --- a/src/Elastic.Ingest.Elasticsearch/IsExternalInit.cs +++ b/src/Elastic.Ingest.Elasticsearch/IsExternalInit.cs @@ -2,7 +2,6 @@ // Elasticsearch B.V licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information // ReSharper disable once CheckNamespace -namespace System.Runtime.CompilerServices -{ - internal static class IsExternalInit {} -} +namespace System.Runtime.CompilerServices; + +internal static class IsExternalInit {} diff --git a/src/Elastic.Ingest.Elasticsearch/Serialization/BulkOperationHeader.cs b/src/Elastic.Ingest.Elasticsearch/Serialization/BulkOperationHeader.cs index bae4fbb..b5dd325 100644 --- a/src/Elastic.Ingest.Elasticsearch/Serialization/BulkOperationHeader.cs +++ b/src/Elastic.Ingest.Elasticsearch/Serialization/BulkOperationHeader.cs @@ -6,94 +6,93 @@ using System.Text.Json; using System.Text.Json.Serialization; -namespace Elastic.Ingest.Elasticsearch.Serialization +namespace Elastic.Ingest.Elasticsearch.Serialization; + +/// Represents the _bulk operation meta header +public abstract class BulkOperationHeader { - /// Represents the _bulk operation meta header - public abstract class BulkOperationHeader - { - /// The index or data stream to write to - [JsonPropertyName("_index")] - public string? Index { get; init; } + /// The index or data stream to write to + [JsonPropertyName("_index")] + public string? Index { get; init; } - /// The id of the object being written - [JsonPropertyName("_id")] - public string? Id { get; init; } + /// The id of the object being written + [JsonPropertyName("_id")] + public string? Id { get; init; } - /// Require to point to an alias - [JsonPropertyName("require_alias")] - public bool? RequireAlias { get; init; } - } + /// Require to point to an alias + [JsonPropertyName("require_alias")] + public bool? RequireAlias { get; init; } +} - /// Represents the _bulk create operation meta header - [JsonConverter(typeof(BulkOperationHeaderConverter))] - public class CreateOperation : BulkOperationHeader - { - /// - [JsonPropertyName("dynamic_templates")] - public Dictionary? DynamicTemplates { get; init; } - } +/// Represents the _bulk create operation meta header +[JsonConverter(typeof(BulkOperationHeaderConverter))] +public class CreateOperation : BulkOperationHeader +{ + /// + [JsonPropertyName("dynamic_templates")] + public Dictionary? DynamicTemplates { get; init; } +} - /// Represents the _bulk index operation meta header - [JsonConverter(typeof(BulkOperationHeaderConverter))] - public class IndexOperation : BulkOperationHeader - { - /// - [JsonPropertyName("dynamic_templates")] - public Dictionary? DynamicTemplates { get; init; } - } +/// Represents the _bulk index operation meta header +[JsonConverter(typeof(BulkOperationHeaderConverter))] +public class IndexOperation : BulkOperationHeader +{ + /// + [JsonPropertyName("dynamic_templates")] + public Dictionary? DynamicTemplates { get; init; } +} - /// Represents the _bulk delete operation meta header - [JsonConverter(typeof(BulkOperationHeaderConverter))] - public class DeleteOperation : BulkOperationHeader - { - } +/// Represents the _bulk delete operation meta header +[JsonConverter(typeof(BulkOperationHeaderConverter))] +public class DeleteOperation : BulkOperationHeader +{ +} - /// Represents the _bulk update operation meta header - [JsonConverter(typeof(BulkOperationHeaderConverter))] - public class UpdateOperation : BulkOperationHeader - { - } +/// Represents the _bulk update operation meta header +[JsonConverter(typeof(BulkOperationHeaderConverter))] +public class UpdateOperation : BulkOperationHeader +{ +} - internal class BulkOperationHeaderConverter : JsonConverter - where THeader : BulkOperationHeader - { - public override THeader Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) => - throw new NotImplementedException(); +internal class BulkOperationHeaderConverter : JsonConverter + where THeader : BulkOperationHeader +{ + public override THeader Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) => + throw new NotImplementedException(); - public override void Write(Utf8JsonWriter writer, THeader value, JsonSerializerOptions options) + public override void Write(Utf8JsonWriter writer, THeader value, JsonSerializerOptions options) + { + var op = value switch { - var op = value switch - { - CreateOperation _ => "create", - DeleteOperation _ => "delete", - IndexOperation _ => "index", - UpdateOperation _ => "update", - _ => throw new ArgumentOutOfRangeException(nameof(value), value, null) - }; - writer.WriteStartObject(); - writer.WritePropertyName(op); - writer.WriteStartObject(); - if (!string.IsNullOrWhiteSpace(value.Index)) - writer.WriteString("_index", value.Index); - if (!string.IsNullOrWhiteSpace(value.Id)) - writer.WriteString("_id", value.Id); - if (value.RequireAlias == true) - writer.WriteBoolean("require_alias", true); - if (value is CreateOperation c) - WriteDynamicTemplates(writer, options, c.DynamicTemplates); - if (value is IndexOperation i) - WriteDynamicTemplates(writer, options, i.DynamicTemplates); + CreateOperation _ => "create", + DeleteOperation _ => "delete", + IndexOperation _ => "index", + UpdateOperation _ => "update", + _ => throw new ArgumentOutOfRangeException(nameof(value), value, null) + }; + writer.WriteStartObject(); + writer.WritePropertyName(op); + writer.WriteStartObject(); + if (!string.IsNullOrWhiteSpace(value.Index)) + writer.WriteString("_index", value.Index); + if (!string.IsNullOrWhiteSpace(value.Id)) + writer.WriteString("_id", value.Id); + if (value.RequireAlias == true) + writer.WriteBoolean("require_alias", true); + if (value is CreateOperation c) + WriteDynamicTemplates(writer, options, c.DynamicTemplates); + if (value is IndexOperation i) + WriteDynamicTemplates(writer, options, i.DynamicTemplates); - writer.WriteEndObject(); - writer.WriteEndObject(); - } + writer.WriteEndObject(); + writer.WriteEndObject(); + } - private static void WriteDynamicTemplates(Utf8JsonWriter writer, JsonSerializerOptions options, Dictionary? templates) - { - if (templates is not { Count: > 0 }) return; + private static void WriteDynamicTemplates(Utf8JsonWriter writer, JsonSerializerOptions options, Dictionary? templates) + { + if (templates is not { Count: > 0 }) return; - writer.WritePropertyName("dynamic_templates"); - JsonSerializer.Serialize(writer, templates, options); - } + writer.WritePropertyName("dynamic_templates"); + JsonSerializer.Serialize(writer, templates, options); } } diff --git a/src/Elastic.Ingest.Elasticsearch/Serialization/BulkResponse.cs b/src/Elastic.Ingest.Elasticsearch/Serialization/BulkResponse.cs index 8418a9a..7f94a44 100644 --- a/src/Elastic.Ingest.Elasticsearch/Serialization/BulkResponse.cs +++ b/src/Elastic.Ingest.Elasticsearch/Serialization/BulkResponse.cs @@ -9,59 +9,57 @@ using Elastic.Transport; using Elastic.Transport.Products.Elasticsearch; -namespace Elastic.Ingest.Elasticsearch.Serialization -{ +namespace Elastic.Ingest.Elasticsearch.Serialization; - /// Represents the _bulk response from Elasticsearch - public class BulkResponse : TransportResponse - { - /// - /// Individual bulk response items information - /// - [JsonPropertyName("items")] - [JsonConverter(typeof(ResponseItemsConverter))] - public IReadOnlyCollection Items { get; set; } = null!; - /// Overall bulk error from Elasticsearch if any - [JsonPropertyName("error")] - public ErrorCause? Error { get; set; } +/// Represents the _bulk response from Elasticsearch +public class BulkResponse : TransportResponse +{ + /// + /// Individual bulk response items information + /// + [JsonPropertyName("items")] + [JsonConverter(typeof(ResponseItemsConverter))] + public IReadOnlyCollection Items { get; set; } = null!; - /// - /// Tries and get the error from Elasticsearch as string - /// - /// True if Elasticsearch contained an overall bulk error - public bool TryGetServerErrorReason(out string? reason) - { - reason = Error?.Reason; - return !string.IsNullOrWhiteSpace(reason); - } + /// Overall bulk error from Elasticsearch if any + [JsonPropertyName("error")] + public ErrorCause? Error { get; set; } - /// - public override string ToString() => ApiCallDetails.DebugInformation; + /// + /// Tries and get the error from Elasticsearch as string + /// + /// True if Elasticsearch contained an overall bulk error + public bool TryGetServerErrorReason(out string? reason) + { + reason = Error?.Reason; + return !string.IsNullOrWhiteSpace(reason); } - internal class ResponseItemsConverter : JsonConverter> + /// + public override string ToString() => ApiCallDetails.DebugInformation; +} + +internal class ResponseItemsConverter : JsonConverter> +{ + public static readonly IReadOnlyCollection EmptyBulkItems = + new ReadOnlyCollection(new List()); + + public override IReadOnlyCollection Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { - public static readonly IReadOnlyCollection EmptyBulkItems = - new ReadOnlyCollection(new List()); + if (reader.TokenType != JsonTokenType.StartArray) return EmptyBulkItems; - public override IReadOnlyCollection Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + var list = new List(); + var depth = reader.CurrentDepth; + while (reader.Read() && reader.CurrentDepth > depth) { - if (reader.TokenType != JsonTokenType.StartArray) return EmptyBulkItems; - - var list = new List(); - var depth = reader.CurrentDepth; - while (reader.Read() && reader.CurrentDepth > depth) - { - var item = JsonSerializer.Deserialize(ref reader, options); - if (item != null) - list.Add(item); - } - return new ReadOnlyCollection(list); + var item = JsonSerializer.Deserialize(ref reader, options); + if (item != null) + list.Add(item); } - - public override void Write(Utf8JsonWriter writer, IReadOnlyCollection value, JsonSerializerOptions options) => - throw new NotImplementedException(); + return new ReadOnlyCollection(list); } + public override void Write(Utf8JsonWriter writer, IReadOnlyCollection value, JsonSerializerOptions options) => + throw new NotImplementedException(); } diff --git a/src/Elastic.Ingest.Elasticsearch/Serialization/BulkResponseItem.cs b/src/Elastic.Ingest.Elasticsearch/Serialization/BulkResponseItem.cs index 1490af3..bc30649 100644 --- a/src/Elastic.Ingest.Elasticsearch/Serialization/BulkResponseItem.cs +++ b/src/Elastic.Ingest.Elasticsearch/Serialization/BulkResponseItem.cs @@ -6,80 +6,79 @@ using System.Text.Json.Serialization; using Elastic.Transport.Products.Elasticsearch; -namespace Elastic.Ingest.Elasticsearch.Serialization +namespace Elastic.Ingest.Elasticsearch.Serialization; + +/// Represents a bulk response item +[JsonConverter(typeof(ItemConverter))] +public class BulkResponseItem { - /// Represents a bulk response item - [JsonConverter(typeof(ItemConverter))] - public class BulkResponseItem - { - /// The action that was used for the event (create/index) - public string Action { get; internal set; } = null!; - /// Elasticsearch error if any - public ErrorCause? Error { get; internal set; } - /// Status code from Elasticsearch writing the event - public int Status { get; internal set; } - } + /// The action that was used for the event (create/index) + public string Action { get; internal set; } = null!; + /// Elasticsearch error if any + public ErrorCause? Error { get; internal set; } + /// Status code from Elasticsearch writing the event + public int Status { get; internal set; } +} + +internal class ItemConverter : JsonConverter +{ + private static readonly BulkResponseItem OkayBulkResponseItem = new BulkResponseItem { Status = 200, Action = "index" }; - internal class ItemConverter : JsonConverter + public override BulkResponseItem Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { - private static readonly BulkResponseItem OkayBulkResponseItem = new BulkResponseItem { Status = 200, Action = "index" }; + //TODO nasty null return + if (reader.TokenType != JsonTokenType.StartObject) return null!; - public override BulkResponseItem Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + reader.Read(); + var depth = reader.CurrentDepth; + var status = 0; + ErrorCause? error = null; + var action = reader.GetString()!; + while (reader.Read() && reader.CurrentDepth >= depth) { - //TODO nasty null return - if (reader.TokenType != JsonTokenType.StartObject) return null!; + if (reader.TokenType != JsonTokenType.PropertyName) continue; - reader.Read(); - var depth = reader.CurrentDepth; - var status = 0; - ErrorCause? error = null; - var action = reader.GetString()!; - while (reader.Read() && reader.CurrentDepth >= depth) + var text = reader.GetString(); + switch (text) { - if (reader.TokenType != JsonTokenType.PropertyName) continue; - - var text = reader.GetString(); - switch (text) - { - case "status": - reader.Read(); - status = reader.GetInt32(); - break; - case "error": - reader.Read(); - error = JsonSerializer.Deserialize(ref reader, options); - break; - } + case "status": + reader.Read(); + status = reader.GetInt32(); + break; + case "error": + reader.Read(); + error = JsonSerializer.Deserialize(ref reader, options); + break; } - var r = status == 200 - ? OkayBulkResponseItem - : new BulkResponseItem { Action = action, Status = status, Error = error }; - - return r; } + var r = status == 200 + ? OkayBulkResponseItem + : new BulkResponseItem { Action = action, Status = status, Error = error }; - public override void Write(Utf8JsonWriter writer, BulkResponseItem value, JsonSerializerOptions options) - { - // ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract - if (value is null) - { - writer.WriteNullValue(); - return; - } + return r; + } - writer.WriteStartObject(); - writer.WritePropertyName(value.Action); - writer.WriteStartObject(); + public override void Write(Utf8JsonWriter writer, BulkResponseItem value, JsonSerializerOptions options) + { + // ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract + if (value is null) + { + writer.WriteNullValue(); + return; + } - if (value.Error != null) - { - writer.WritePropertyName("error"); - JsonSerializer.Serialize(writer, value.Error, options); - } + writer.WriteStartObject(); + writer.WritePropertyName(value.Action); + writer.WriteStartObject(); - writer.WriteNumber("status", value.Status); - writer.WriteEndObject(); - writer.WriteEndObject(); + if (value.Error != null) + { + writer.WritePropertyName("error"); + JsonSerializer.Serialize(writer, value.Error, options); } + + writer.WriteNumber("status", value.Status); + writer.WriteEndObject(); + writer.WriteEndObject(); } } diff --git a/src/Elastic.Ingest.OpenTelemetry/CustomActivityExporter.cs b/src/Elastic.Ingest.OpenTelemetry/CustomActivityExporter.cs index 26d6895..7cfcfb7 100644 --- a/src/Elastic.Ingest.OpenTelemetry/CustomActivityExporter.cs +++ b/src/Elastic.Ingest.OpenTelemetry/CustomActivityExporter.cs @@ -4,27 +4,26 @@ using System.Diagnostics; using OpenTelemetry; -namespace Elastic.Ingest.OpenTelemetry +namespace Elastic.Ingest.OpenTelemetry; + +/// +public class CustomActivityProcessor : BatchActivityExportProcessor { /// - public class CustomActivityProcessor : BatchActivityExportProcessor + public CustomActivityProcessor( + BaseExporter exporter, + int maxQueueSize = 2048, + int scheduledDelayMilliseconds = 5000, + int exporterTimeoutMilliseconds = 30000, + int maxExportBatchSize = 512 + ) + : base(exporter, maxQueueSize, scheduledDelayMilliseconds, exporterTimeoutMilliseconds, maxExportBatchSize) { - /// - public CustomActivityProcessor( - BaseExporter exporter, - int maxQueueSize = 2048, - int scheduledDelayMilliseconds = 5000, - int exporterTimeoutMilliseconds = 30000, - int maxExportBatchSize = 512 - ) - : base(exporter, maxQueueSize, scheduledDelayMilliseconds, exporterTimeoutMilliseconds, maxExportBatchSize) - { - Activity.DefaultIdFormat = ActivityIdFormat.W3C; - Activity.ForceDefaultIdFormat = true; - } + Activity.DefaultIdFormat = ActivityIdFormat.W3C; + Activity.ForceDefaultIdFormat = true; + } - /// - public void Add(Activity a) => OnExport(a); + /// + public void Add(Activity a) => OnExport(a); - } } diff --git a/src/Elastic.Ingest.OpenTelemetry/CustomOtlpTraceExporter.cs b/src/Elastic.Ingest.OpenTelemetry/CustomOtlpTraceExporter.cs index 251866e..03d8538 100644 --- a/src/Elastic.Ingest.OpenTelemetry/CustomOtlpTraceExporter.cs +++ b/src/Elastic.Ingest.OpenTelemetry/CustomOtlpTraceExporter.cs @@ -14,107 +14,108 @@ using Elastic.Channels; using Elastic.Channels.Diagnostics; -namespace Elastic.Ingest.OpenTelemetry +namespace Elastic.Ingest.OpenTelemetry; + +/// +public class CustomOtlpTraceExporter : OtlpTraceExporter { /// - public class CustomOtlpTraceExporter : OtlpTraceExporter + public CustomOtlpTraceExporter(OtlpExporterOptions options, TraceChannelOptions channelOptions) : base(options) { - /// - public CustomOtlpTraceExporter(OtlpExporterOptions options, TraceChannelOptions channelOptions) : base(options) - { - var type = GetType(); - var attrbutes = new[] { new KeyValuePair("telemetry.sdk.language", "dotnet") }; - var resource = ResourceBuilder.CreateDefault(); - if (!string.IsNullOrWhiteSpace(channelOptions.ServiceName)) - resource.AddService(channelOptions.ServiceName); - - var buildResource = resource.AddAttributes(attrbutes).Build(); - // hack but there is no other way to set a resource without spinning up the world - // through SDK. - // internal void SetResource(Resource resource) - var prop = type.BaseType?.GetMethod("SetResource", BindingFlags.Instance | BindingFlags.NonPublic); - prop?.Invoke(this, new object?[]{ buildResource }); - } + var type = GetType(); + var attrbutes = new[] { new KeyValuePair("telemetry.sdk.language", "dotnet") }; + var resource = ResourceBuilder.CreateDefault(); + if (!string.IsNullOrWhiteSpace(channelOptions.ServiceName)) + resource.AddService(channelOptions.ServiceName); + + var buildResource = resource.AddAttributes(attrbutes).Build(); + // hack but there is no other way to set a resource without spinning up the world + // through SDK. + // internal void SetResource(Resource resource) + var prop = type.BaseType?.GetMethod("SetResource", BindingFlags.Instance | BindingFlags.NonPublic); + prop?.Invoke(this, new object?[]{ buildResource }); } +} +/// +public class TraceChannelOptions : ChannelOptionsBase +{ /// - public class TraceChannelOptions : ChannelOptionsBase - { - /// - public string? ServiceName { get; set; } - /// - public Uri? Endpoint { get; set; } - /// - public string? SecretToken { get; set; } - } + public string? ServiceName { get; set; } + /// + public Uri? Endpoint { get; set; } + /// + public string? SecretToken { get; set; } +} +/// +public class TraceExportResult +{ /// - public class TraceExportResult - { - /// - public ExportResult Result { get; internal set; } - } + public ExportResult Result { get; internal set; } +} +/// +public class TraceChannel : BufferedChannelBase +{ /// - public class TraceChannel : BufferedChannelBase - { - /// - public TraceChannel(TraceChannelOptions options) : this(options, null) { } - - /// - public TraceChannel(TraceChannelOptions options, ICollection>? callbackListeners) - : base(options, callbackListeners) { - var o = new OtlpExporterOptions(); - o.Endpoint = options.Endpoint; - o.Headers = $"Authorization=Bearer {options.SecretToken}"; - TraceExporter = new CustomOtlpTraceExporter(o, options); + public TraceChannel(TraceChannelOptions options) : this(options, null) { } + + /// + public TraceChannel(TraceChannelOptions options, ICollection>? callbackListeners) + : base(options, callbackListeners) { + var o = new OtlpExporterOptions + { + Endpoint = options.Endpoint, + Headers = $"Authorization=Bearer {options.SecretToken}" + }; + TraceExporter = new CustomOtlpTraceExporter(o, options); Processor = new CustomActivityProcessor(TraceExporter, - maxExportBatchSize: options.BufferOptions.OutboundBufferMaxSize, - maxQueueSize: options.BufferOptions.InboundBufferMaxSize, - scheduledDelayMilliseconds: (int)options.BufferOptions.OutboundBufferMaxLifetime.TotalMilliseconds, - exporterTimeoutMilliseconds: (int)options.BufferOptions.OutboundBufferMaxLifetime.TotalMilliseconds - ); - var bufferType = typeof(BaseExporter<>).Assembly.GetTypes().First(t=>t.Name == "CircularBuffer`1"); - var activityBuffer = bufferType.GetGenericTypeDefinition().MakeGenericType(typeof(Activity)); - var bufferTypeConstructor = activityBuffer.GetConstructors().First(); - var bufferAddMethod = bufferType.GetMethod("Add"); - - var batchType = typeof(Batch); - var batchConstructor = batchType.GetConstructors(BindingFlags.Instance | BindingFlags.NonPublic).First(c=>c.GetParameters().Length == 2); - - BatchCreator = (page) => - { - var buffer = bufferTypeConstructor.Invoke(new object[] {options.BufferOptions.OutboundBufferMaxSize }); - bufferAddMethod.Invoke(buffer, new[] { page }); - var batch = (Batch)batchConstructor.Invoke(new[] {buffer, options.BufferOptions.OutboundBufferMaxSize }); - return batch; - }; - - } - - private Func, Batch> BatchCreator { get; } - - /// - public CustomOtlpTraceExporter TraceExporter { get; } - - /// - public CustomActivityProcessor Processor { get; } - - /// - protected override Task Export(ArraySegment page, CancellationToken ctx = default) + maxExportBatchSize: options.BufferOptions.OutboundBufferMaxSize, + maxQueueSize: options.BufferOptions.InboundBufferMaxSize, + scheduledDelayMilliseconds: (int)options.BufferOptions.OutboundBufferMaxLifetime.TotalMilliseconds, + exporterTimeoutMilliseconds: (int)options.BufferOptions.OutboundBufferMaxLifetime.TotalMilliseconds + ); + var bufferType = typeof(BaseExporter<>).Assembly.GetTypes().First(t=>t.Name == "CircularBuffer`1"); + var activityBuffer = bufferType.GetGenericTypeDefinition().MakeGenericType(typeof(Activity)); + var bufferTypeConstructor = activityBuffer.GetConstructors().First(); + var bufferAddMethod = bufferType.GetMethod("Add"); + + var batchType = typeof(Batch); + var batchConstructor = batchType.GetConstructors(BindingFlags.Instance | BindingFlags.NonPublic).First(c=>c.GetParameters().Length == 2); + + BatchCreator = (page) => { - var batch = BatchCreator(page); - var result = TraceExporter.Export(batch); - return Task.FromResult(new TraceExportResult { Result = result }); - } + var buffer = bufferTypeConstructor.Invoke(new object[] {options.BufferOptions.OutboundBufferMaxSize }); + bufferAddMethod.Invoke(buffer, new[] { page }); + var batch = (Batch)batchConstructor.Invoke(new[] {buffer, options.BufferOptions.OutboundBufferMaxSize }); + return batch; + }; - /// - public override void Dispose() - { - base.Dispose(); - Processor.Dispose(); - } + } + + private Func, Batch> BatchCreator { get; } + + /// + public CustomOtlpTraceExporter TraceExporter { get; } + + /// + public CustomActivityProcessor Processor { get; } + + /// + protected override Task ExportAsync(ArraySegment page, CancellationToken ctx = default) + { + var batch = BatchCreator(page); + var result = TraceExporter.Export(batch); + return Task.FromResult(new TraceExportResult { Result = result }); + } + /// + public override void Dispose() + { + base.Dispose(); + Processor.Dispose(); } + } diff --git a/src/Elastic.Ingest.Transport/TransportChannelBase.cs b/src/Elastic.Ingest.Transport/TransportChannelBase.cs index 4d1de80..cb6cb37 100644 --- a/src/Elastic.Ingest.Transport/TransportChannelBase.cs +++ b/src/Elastic.Ingest.Transport/TransportChannelBase.cs @@ -10,34 +10,32 @@ using Elastic.Channels.Diagnostics; using Elastic.Transport; -namespace Elastic.Ingest.Transport -{ - /// - /// A implementation that provides a common base for channels - /// looking to data - /// over - /// - public abstract class TransportChannelBase : - ResponseItemsBufferedChannelBase - where TChannelOptions : TransportChannelOptionsBase - where TResponse : TransportResponse, new() +namespace Elastic.Ingest.Transport; - { - /// - protected TransportChannelBase(TChannelOptions options, ICollection>? callbackListeners) - : base(options, callbackListeners) { } +/// +/// A implementation that provides a common base for channels +/// looking to data +/// over +/// +public abstract class TransportChannelBase : + ResponseItemsBufferedChannelBase + where TChannelOptions : TransportChannelOptionsBase + where TResponse : TransportResponse, new() +{ + /// + protected TransportChannelBase(TChannelOptions options, ICollection>? callbackListeners) + : base(options, callbackListeners) { } - /// - protected TransportChannelBase(TChannelOptions options) - : base(options) { } + /// + protected TransportChannelBase(TChannelOptions options) + : base(options) { } - /// Implement sending the current of the buffer to the output. - /// - /// Active page of the buffer that needs to be send to the output - /// - protected abstract Task Export(HttpTransport transport, ArraySegment page, CancellationToken ctx = default); + /// Implement sending the current of the buffer to the output. + /// + /// Active page of the buffer that needs to be send to the output + /// + protected abstract Task ExportAsync(HttpTransport transport, ArraySegment page, CancellationToken ctx = default); - /// > - protected override Task Export(ArraySegment buffer, CancellationToken ctx = default) => Export(Options.Transport, buffer, ctx); - } + /// > + protected override Task ExportAsync(ArraySegment buffer, CancellationToken ctx = default) => ExportAsync(Options.Transport, buffer, ctx); } diff --git a/src/Elastic.Ingest.Transport/TransportChannelOptionsBase.cs b/src/Elastic.Ingest.Transport/TransportChannelOptionsBase.cs index b760808..896792e 100644 --- a/src/Elastic.Ingest.Transport/TransportChannelOptionsBase.cs +++ b/src/Elastic.Ingest.Transport/TransportChannelOptionsBase.cs @@ -5,20 +5,19 @@ using Elastic.Channels; using Elastic.Transport; -namespace Elastic.Ingest.Transport +namespace Elastic.Ingest.Transport; + +/// +/// Provides channel options to implementation. +/// +public abstract class TransportChannelOptionsBase + : ResponseItemsChannelOptionsBase { + /// + protected TransportChannelOptionsBase(HttpTransport transport) => Transport = transport; + /// - /// Provides channel options to implementation. + /// The implementation to be used by the channel /// - public abstract class TransportChannelOptionsBase - : ResponseItemsChannelOptionsBase - { - /// - protected TransportChannelOptionsBase(HttpTransport transport) => Transport = transport; - - /// - /// The implementation to be used by the channel - /// - public HttpTransport Transport { get; } - } + public HttpTransport Transport { get; } } diff --git a/tests/Elastic.Channels.Tests/BehaviorTests.cs b/tests/Elastic.Channels.Tests/BehaviorTests.cs index 13b7b48..5af17e4 100644 --- a/tests/Elastic.Channels.Tests/BehaviorTests.cs +++ b/tests/Elastic.Channels.Tests/BehaviorTests.cs @@ -11,168 +11,167 @@ using Xunit; using Xunit.Abstractions; -namespace Elastic.Channels.Tests +namespace Elastic.Channels.Tests; + +public class BehaviorTests : IDisposable { - public class BehaviorTests : IDisposable - { - public BehaviorTests(ITestOutputHelper testOutput) => XunitContext.Register(testOutput); + public BehaviorTests(ITestOutputHelper testOutput) => XunitContext.Register(testOutput); - void IDisposable.Dispose() => XunitContext.Flush(); + void IDisposable.Dispose() => XunitContext.Flush(); - [Fact] public async Task RespectsPagination() + [Fact] public async Task RespectsPagination() + { + int totalEvents = 500_000, maxInFlight = totalEvents / 5, bufferSize = maxInFlight / 10; + var expectedSentBuffers = totalEvents / bufferSize; + var bufferOptions = new BufferOptions { - int totalEvents = 500_000, maxInFlight = totalEvents / 5, bufferSize = maxInFlight / 10; - var expectedSentBuffers = totalEvents / bufferSize; - var bufferOptions = new BufferOptions - { - WaitHandle = new CountdownEvent(expectedSentBuffers), InboundBufferMaxSize = maxInFlight, OutboundBufferMaxSize = bufferSize, - }; - var channel = new NoopBufferedChannel(bufferOptions); - - var written = 0; - for (var i = 0; i < totalEvents; i++) - { - var e = new NoopBufferedChannel.NoopEvent(); - if (await channel.WaitToWriteAsync(e)) - written++; - } - var signalled = bufferOptions.WaitHandle.Wait(TimeSpan.FromSeconds(5)); - signalled.Should().BeTrue("The channel was not drained in the expected time"); - written.Should().Be(totalEvents); - channel.ExportedBuffers.Should().Be(expectedSentBuffers); - } + WaitHandle = new CountdownEvent(expectedSentBuffers), InboundBufferMaxSize = maxInFlight, OutboundBufferMaxSize = bufferSize, + }; + var channel = new NoopBufferedChannel(bufferOptions); - /// - /// If we are feeding data slowly e.g smaller than - /// we don't want this data equally distributed over multiple calls to export the data. - /// Instead we want the smaller buffer to go out over a single export to the external system - /// - [Fact] public async Task MessagesAreSequentiallyDistributedOverWorkers() + var written = 0; + for (var i = 0; i < totalEvents; i++) { - int totalEvents = 500_000, maxInFlight = totalEvents / 5, bufferSize = maxInFlight / 10; - var bufferOptions = new BufferOptions - { - WaitHandle = new CountdownEvent(1), - InboundBufferMaxSize = maxInFlight, - OutboundBufferMaxSize = bufferSize, - OutboundBufferMaxLifetime = TimeSpan.FromMilliseconds(500) - }; - - var channel = new NoopBufferedChannel(bufferOptions); - var written = 0; - for (var i = 0; i < 100; i++) - { - var e = new NoopBufferedChannel.NoopEvent(); - if (await channel.WaitToWriteAsync(e)) - written++; - } - var signalled = bufferOptions.WaitHandle.Wait(TimeSpan.FromSeconds(1)); - signalled.Should().BeTrue("The channel was not drained in the expected time"); - written.Should().Be(100); - channel.ExportedBuffers.Should().Be(1); + var e = new NoopBufferedChannel.NoopEvent(); + if (await channel.WaitToWriteAsync(e)) + written++; } + var signalled = bufferOptions.WaitHandle.Wait(TimeSpan.FromSeconds(5)); + signalled.Should().BeTrue("The channel was not drained in the expected time"); + written.Should().Be(totalEvents); + channel.ExportedBuffers.Should().Be(expectedSentBuffers); + } - [Fact] public async Task ConcurrencyIsApplied() + /// + /// If we are feeding data slowly e.g smaller than + /// we don't want this data equally distributed over multiple calls to export the data. + /// Instead we want the smaller buffer to go out over a single export to the external system + /// + [Fact] public async Task MessagesAreSequentiallyDistributedOverWorkers() + { + int totalEvents = 500_000, maxInFlight = totalEvents / 5, bufferSize = maxInFlight / 10; + var bufferOptions = new BufferOptions { - int totalEvents = 5_000, maxInFlight = 5_000, bufferSize = 500; - var expectedPages = totalEvents / bufferSize; - var bufferOptions = new BufferOptions - { - WaitHandle = new CountdownEvent(expectedPages), - InboundBufferMaxSize = maxInFlight, - OutboundBufferMaxSize = bufferSize, - ExportMaxConcurrency = 4 - }; - - var channel = new NoopBufferedChannel(bufferOptions, observeConcurrency: true); - - var written = 0; - for (var i = 0; i < totalEvents; i++) - { - var e = new NoopBufferedChannel.NoopEvent(); - if (await channel.WaitToWriteAsync(e)) - written++; - } - var signalled = bufferOptions.WaitHandle.Wait(TimeSpan.FromSeconds(5)); - signalled.Should().BeTrue("The channel was not drained in the expected time"); - written.Should().Be(totalEvents); - channel.ExportedBuffers.Should().Be(expectedPages); - channel.ObservedConcurrency.Should().BeGreaterThan(1); + WaitHandle = new CountdownEvent(1), + InboundBufferMaxSize = maxInFlight, + OutboundBufferMaxSize = bufferSize, + OutboundBufferMaxLifetime = TimeSpan.FromMilliseconds(500) + }; + + var channel = new NoopBufferedChannel(bufferOptions); + var written = 0; + for (var i = 0; i < 100; i++) + { + var e = new NoopBufferedChannel.NoopEvent(); + if (await channel.WaitToWriteAsync(e)) + written++; } + var signalled = bufferOptions.WaitHandle.Wait(TimeSpan.FromSeconds(1)); + signalled.Should().BeTrue("The channel was not drained in the expected time"); + written.Should().Be(100); + channel.ExportedBuffers.Should().Be(1); + } - [Fact] public async Task ManyChannelsContinueToDoWork() + [Fact] public async Task ConcurrencyIsApplied() + { + int totalEvents = 5_000, maxInFlight = 5_000, bufferSize = 500; + var expectedPages = totalEvents / bufferSize; + var bufferOptions = new BufferOptions { - int totalEvents = 50_000_000, maxInFlight = totalEvents / 5, bufferSize = maxInFlight / 10; - int closedThread = 0, maxFor = Environment.ProcessorCount * 2; - var expectedSentBuffers = totalEvents / bufferSize; + WaitHandle = new CountdownEvent(expectedPages), + InboundBufferMaxSize = maxInFlight, + OutboundBufferMaxSize = bufferSize, + ExportMaxConcurrency = 4 + }; - Task StartChannel(int taskNumber) - { - var bufferOptions = new BufferOptions - { - WaitHandle = new CountdownEvent(expectedSentBuffers), - InboundBufferMaxSize = maxInFlight, - OutboundBufferMaxSize = 1000, - OutboundBufferMaxLifetime = TimeSpan.FromMilliseconds(20) - }; - using var channel = new DiagnosticsBufferedChannel(bufferOptions, name: $"Task {taskNumber}"); - var written = 0; - var t = Task.Factory.StartNew(async () => - { - for (var i = 0; i < totalEvents; i++) - { - var e = new NoopBufferedChannel.NoopEvent(); - if (await channel.WaitToWriteAsync(e)) - written++; - } - }, TaskCreationOptions.LongRunning | TaskCreationOptions.PreferFairness); - // wait for some work to have progressed - bufferOptions.WaitHandle.Wait(TimeSpan.FromMilliseconds(500)); - - written.Should().BeGreaterThan(0).And.BeLessThan(totalEvents); - channel.ExportedBuffers.Should().BeGreaterThan(0, "Parallel invocation: {0} channel: {1}", taskNumber, channel); - Interlocked.Increment(ref closedThread); - return t; - } + var channel = new NoopBufferedChannel(bufferOptions, observeConcurrency: true); - var tasks = Enumerable.Range(0, maxFor).Select(i => Task.Factory.StartNew(() => StartChannel(i), TaskCreationOptions.LongRunning | TaskCreationOptions.PreferFairness)).ToArray(); - - await Task.WhenAll(tasks); - - closedThread.Should().BeGreaterThan(0).And.Be(maxFor); + var written = 0; + for (var i = 0; i < totalEvents; i++) + { + var e = new NoopBufferedChannel.NoopEvent(); + if (await channel.WaitToWriteAsync(e)) + written++; } + var signalled = bufferOptions.WaitHandle.Wait(TimeSpan.FromSeconds(5)); + signalled.Should().BeTrue("The channel was not drained in the expected time"); + written.Should().Be(totalEvents); + channel.ExportedBuffers.Should().Be(expectedPages); + channel.ObservedConcurrency.Should().BeGreaterThan(1); + } - [Fact] public async Task SlowlyPushEvents() + [Fact] public async Task ManyChannelsContinueToDoWork() + { + int totalEvents = 50_000_000, maxInFlight = totalEvents / 5, bufferSize = maxInFlight / 10; + int closedThread = 0, maxFor = Environment.ProcessorCount * 2; + var expectedSentBuffers = totalEvents / bufferSize; + + Task StartChannel(int taskNumber) { - int totalEvents = 50_000_000, maxInFlight = totalEvents / 5, bufferSize = maxInFlight / 10; - var expectedSentBuffers = totalEvents / bufferSize; var bufferOptions = new BufferOptions { WaitHandle = new CountdownEvent(expectedSentBuffers), InboundBufferMaxSize = maxInFlight, - OutboundBufferMaxSize = 10_000, - OutboundBufferMaxLifetime = TimeSpan.FromMilliseconds(100) + OutboundBufferMaxSize = 1000, + OutboundBufferMaxLifetime = TimeSpan.FromMilliseconds(20) }; - using var channel = new DiagnosticsBufferedChannel(bufferOptions, name: $"Slow push channel"); - await Task.Delay(TimeSpan.FromMilliseconds(200)); + using var channel = new DiagnosticsBufferedChannel(bufferOptions, name: $"Task {taskNumber}"); var written = 0; - var _ = Task.Factory.StartNew(async () => + var t = Task.Factory.StartNew(async () => { - for (var i = 0; i < totalEvents && !channel.Options.BufferOptions.WaitHandle.IsSet; i++) + for (var i = 0; i < totalEvents; i++) { var e = new NoopBufferedChannel.NoopEvent(); - if (await channel.WaitToWriteAsync(e).ConfigureAwait(false)) + if (await channel.WaitToWriteAsync(e)) written++; - await Task.Delay(TimeSpan.FromMilliseconds(40)).ConfigureAwait(false); } - }, TaskCreationOptions.LongRunning); + }, TaskCreationOptions.LongRunning | TaskCreationOptions.PreferFairness); // wait for some work to have progressed bufferOptions.WaitHandle.Wait(TimeSpan.FromMilliseconds(500)); - //Ensure we written to the channel but not enough to satisfy OutboundBufferMaxSize - written.Should().BeGreaterThan(0).And.BeLessThan(10_000); - //even though OutboundBufferMaxSize was not hit we should still observe an invocation to Export() - //because OutboundBufferMaxLifetime was hit - channel.ExportedBuffers.Should().BeGreaterThan(0, "{0}", channel); + + written.Should().BeGreaterThan(0).And.BeLessThan(totalEvents); + channel.ExportedBuffers.Should().BeGreaterThan(0, "Parallel invocation: {0} channel: {1}", taskNumber, channel); + Interlocked.Increment(ref closedThread); + return t; } + + var tasks = Enumerable.Range(0, maxFor).Select(i => Task.Factory.StartNew(() => StartChannel(i), TaskCreationOptions.LongRunning | TaskCreationOptions.PreferFairness)).ToArray(); + + await Task.WhenAll(tasks); + + closedThread.Should().BeGreaterThan(0).And.Be(maxFor); + } + + [Fact] public async Task SlowlyPushEvents() + { + int totalEvents = 50_000_000, maxInFlight = totalEvents / 5, bufferSize = maxInFlight / 10; + var expectedSentBuffers = totalEvents / bufferSize; + var bufferOptions = new BufferOptions + { + WaitHandle = new CountdownEvent(expectedSentBuffers), + InboundBufferMaxSize = maxInFlight, + OutboundBufferMaxSize = 10_000, + OutboundBufferMaxLifetime = TimeSpan.FromMilliseconds(100) + }; + using var channel = new DiagnosticsBufferedChannel(bufferOptions, name: $"Slow push channel"); + await Task.Delay(TimeSpan.FromMilliseconds(200)); + var written = 0; + var _ = Task.Factory.StartNew(async () => + { + for (var i = 0; i < totalEvents && !channel.Options.BufferOptions.WaitHandle.IsSet; i++) + { + var e = new NoopBufferedChannel.NoopEvent(); + if (await channel.WaitToWriteAsync(e).ConfigureAwait(false)) + written++; + await Task.Delay(TimeSpan.FromMilliseconds(40)).ConfigureAwait(false); + } + }, TaskCreationOptions.LongRunning); + // wait for some work to have progressed + bufferOptions.WaitHandle.Wait(TimeSpan.FromMilliseconds(500)); + //Ensure we written to the channel but not enough to satisfy OutboundBufferMaxSize + written.Should().BeGreaterThan(0).And.BeLessThan(10_000); + //even though OutboundBufferMaxSize was not hit we should still observe an invocation to Export() + //because OutboundBufferMaxLifetime was hit + channel.ExportedBuffers.Should().BeGreaterThan(0, "{0}", channel); } } diff --git a/tests/Elastic.Channels.Tests/TroubleshootTests.cs b/tests/Elastic.Channels.Tests/TroubleshootTests.cs index cf9d6f2..706717c 100644 --- a/tests/Elastic.Channels.Tests/TroubleshootTests.cs +++ b/tests/Elastic.Channels.Tests/TroubleshootTests.cs @@ -11,83 +11,82 @@ using Xunit; using Xunit.Abstractions; -namespace Elastic.Channels.Tests +namespace Elastic.Channels.Tests; + +public class TroubleshootTests : IDisposable { - public class TroubleshootTests : IDisposable - { - public TroubleshootTests(ITestOutputHelper testOutput) => XunitContext.Register(testOutput); - void IDisposable.Dispose() => XunitContext.Flush(); + public TroubleshootTests(ITestOutputHelper testOutput) => XunitContext.Register(testOutput); + void IDisposable.Dispose() => XunitContext.Flush(); - [Fact] public async Task CanDisableDiagnostics() + [Fact] public async Task CanDisableDiagnostics() + { + var (totalEvents, expectedSentBuffers, bufferOptions) = Setup(); + var channel = new NoopBufferedChannel(new NoopBufferedChannel.NoopChannelOptions() { - var (totalEvents, expectedSentBuffers, bufferOptions) = Setup(); - var channel = new NoopBufferedChannel(new NoopBufferedChannel.NoopChannelOptions() - { - DisableDiagnostics = true, - BufferOptions = bufferOptions - }); + DisableDiagnostics = true, + BufferOptions = bufferOptions + }); - await WriteExpectedEvents(totalEvents, channel, bufferOptions, expectedSentBuffers); + await WriteExpectedEvents(totalEvents, channel, bufferOptions, expectedSentBuffers); - channel.ToString().Should().Contain("Diagnostics.NoopBufferedChannel"); - channel.ToString().Should().NotContain("Successful publish over channel: NoopBufferedChannel."); - channel.ToString().Should().NotContain($"Exported Buffers: {expectedSentBuffers:N0}"); - } + channel.ToString().Should().Contain("Diagnostics.NoopBufferedChannel"); + channel.ToString().Should().NotContain("Successful publish over channel: NoopBufferedChannel."); + channel.ToString().Should().NotContain($"Exported Buffers: {expectedSentBuffers:N0}"); + } - [Fact] public async Task DefaultIncludesDiagnostics() + [Fact] public async Task DefaultIncludesDiagnostics() + { + var (totalEvents, expectedSentBuffers, bufferOptions) = Setup(); + var channel = new NoopBufferedChannel(new NoopBufferedChannel.NoopChannelOptions() { - var (totalEvents, expectedSentBuffers, bufferOptions) = Setup(); - var channel = new NoopBufferedChannel(new NoopBufferedChannel.NoopChannelOptions() - { - BufferOptions = bufferOptions - }); + BufferOptions = bufferOptions + }); - await WriteExpectedEvents(totalEvents, channel, bufferOptions, expectedSentBuffers); + await WriteExpectedEvents(totalEvents, channel, bufferOptions, expectedSentBuffers); - channel.ToString().Should().NotContain("Diagnostics.NoopBufferedChannel"); - channel.ToString().Should().Contain("Successful publish over channel: NoopBufferedChannel."); - channel.ToString().Should().Contain($"Exported Buffers:"); - } + channel.ToString().Should().NotContain("Diagnostics.NoopBufferedChannel"); + channel.ToString().Should().Contain("Successful publish over channel: NoopBufferedChannel."); + channel.ToString().Should().Contain($"Exported Buffers:"); + } - [Fact] public async Task DiagnosticsChannelAlwaysIncludesDiagnosticsInToString() - { - var (totalEvents, expectedSentBuffers, bufferOptions) = Setup(); - var channel = new DiagnosticsBufferedChannel(bufferOptions); + [Fact] public async Task DiagnosticsChannelAlwaysIncludesDiagnosticsInToString() + { + var (totalEvents, expectedSentBuffers, bufferOptions) = Setup(); + var channel = new DiagnosticsBufferedChannel(bufferOptions); - await WriteExpectedEvents(totalEvents, channel, bufferOptions, expectedSentBuffers); + await WriteExpectedEvents(totalEvents, channel, bufferOptions, expectedSentBuffers); - channel.ToString().Should().NotContain("Diagnostics.DiagnosticsBufferedChannel"); - channel.ToString().Should().Contain("Successful publish over channel: DiagnosticsBufferedChannel."); - channel.ToString().Should().Contain($"Exported Buffers: {expectedSentBuffers:N0}"); - } + channel.ToString().Should().NotContain("Diagnostics.DiagnosticsBufferedChannel"); + channel.ToString().Should().Contain("Successful publish over channel: DiagnosticsBufferedChannel."); + channel.ToString().Should().Contain($"Exported Buffers: {expectedSentBuffers:N0}"); + } - private static async Task WriteExpectedEvents(int totalEvents, NoopBufferedChannel channel, BufferOptions bufferOptions, int expectedSentBuffers) + private static async Task WriteExpectedEvents(int totalEvents, NoopBufferedChannel channel, BufferOptions bufferOptions, int expectedSentBuffers) + { + var written = 0; + for (var i = 0; i < totalEvents; i++) { - var written = 0; - for (var i = 0; i < totalEvents; i++) - { - var e = new NoopBufferedChannel.NoopEvent(); - if (await channel.WaitToWriteAsync(e)) - written++; - } - var signalled = bufferOptions.WaitHandle.Wait(TimeSpan.FromSeconds(5)); - signalled.Should().BeTrue("The channel was not drained in the expected time"); - written.Should().Be(totalEvents); - channel.ExportedBuffers.Should().Be(expectedSentBuffers); + var e = new NoopBufferedChannel.NoopEvent(); + if (await channel.WaitToWriteAsync(e)) + written++; } + var signalled = bufferOptions.WaitHandle.Wait(TimeSpan.FromSeconds(5)); + signalled.Should().BeTrue("The channel was not drained in the expected time"); + written.Should().Be(totalEvents); + channel.ExportedBuffers.Should().Be(expectedSentBuffers); + } - private static (int totalEvents, int expectedSentBuffers, BufferOptions bufferOptions) Setup() + private static (int totalEvents, int expectedSentBuffers, BufferOptions bufferOptions) Setup() + { + int totalEvents = 5000, maxInFlight = totalEvents / 5, bufferSize = maxInFlight / 10; + var expectedSentBuffers = totalEvents / bufferSize; + var bufferOptions = new BufferOptions { - int totalEvents = 5000, maxInFlight = totalEvents / 5, bufferSize = maxInFlight / 10; - var expectedSentBuffers = totalEvents / bufferSize; - var bufferOptions = new BufferOptions - { - WaitHandle = new CountdownEvent(expectedSentBuffers), - InboundBufferMaxSize = maxInFlight, - OutboundBufferMaxSize = bufferSize, - }; - return (totalEvents, expectedSentBuffers, bufferOptions); - } - + WaitHandle = new CountdownEvent(expectedSentBuffers), + InboundBufferMaxSize = maxInFlight, + OutboundBufferMaxSize = bufferSize, + }; + return (totalEvents, expectedSentBuffers, bufferOptions); } + } diff --git a/tests/Elastic.Ingest.Elasticsearch.IntegrationTests/DataStreamIngestionTests.cs b/tests/Elastic.Ingest.Elasticsearch.IntegrationTests/DataStreamIngestionTests.cs index 1bbb3b5..c8aedc6 100644 --- a/tests/Elastic.Ingest.Elasticsearch.IntegrationTests/DataStreamIngestionTests.cs +++ b/tests/Elastic.Ingest.Elasticsearch.IntegrationTests/DataStreamIngestionTests.cs @@ -13,67 +13,66 @@ using Xunit; using Xunit.Abstractions; -namespace Elastic.Ingest.Elasticsearch.IntegrationTests +namespace Elastic.Ingest.Elasticsearch.IntegrationTests; + +public class DataStreamIngestionTests : IntegrationTestBase { - public class DataStreamIngestionTests : IntegrationTestBase + public DataStreamIngestionTests(IngestionCluster cluster, ITestOutputHelper output) : base(cluster, output) { - public DataStreamIngestionTests(IngestionCluster cluster, ITestOutputHelper output) : base(cluster, output) - { - } + } - [Fact] - public async Task EnsureDocumentsEndUpInDataStream() + [Fact] + public async Task EnsureDocumentsEndUpInDataStream() + { + var targetDataStream = new DataStreamName("timeseriesdocs", "dotnet"); + var slim = new CountdownEvent(1); + var options = new DataStreamChannelOptions(Client.Transport) { - var targetDataStream = new DataStreamName("timeseriesdocs", "dotnet"); - var slim = new CountdownEvent(1); - var options = new DataStreamChannelOptions(Client.Transport) - { #pragma warning disable CS0618 - UseReadOnlyMemory = true, + UseReadOnlyMemory = true, #pragma warning restore CS0618 - DataStream = targetDataStream, - BufferOptions = new BufferOptions { WaitHandle = slim, OutboundBufferMaxSize = 1 } - }; - var channel = new DataStreamChannel(options); + DataStream = targetDataStream, + BufferOptions = new BufferOptions { WaitHandle = slim, OutboundBufferMaxSize = 1 } + }; + var channel = new DataStreamChannel(options); - var bootstrapped = await channel.BootstrapElasticsearchAsync(BootstrapMethod.Failure, "7-days-default"); - bootstrapped.Should().BeTrue("Expected to be able to bootstrap data stream channel"); + var bootstrapped = await channel.BootstrapElasticsearchAsync(BootstrapMethod.Failure, "7-days-default"); + bootstrapped.Should().BeTrue("Expected to be able to bootstrap data stream channel"); - var dataStream = - await Client.Indices.GetDataStreamAsync(new GetDataStreamRequest(targetDataStream.ToString())); - dataStream.DataStreams.Should().BeNullOrEmpty(); + var dataStream = + await Client.Indices.GetDataStreamAsync(new GetDataStreamRequest(targetDataStream.ToString())); + dataStream.DataStreams.Should().BeNullOrEmpty(); - channel.TryWrite(new TimeSeriesDocument { Timestamp = DateTimeOffset.Now, Message = "hello-world" }); - if (!slim.WaitHandle.WaitOne(TimeSpan.FromSeconds(10))) - throw new Exception($"document was not persisted within 10 seconds: {channel}"); + channel.TryWrite(new TimeSeriesDocument { Timestamp = DateTimeOffset.Now, Message = "hello-world" }); + if (!slim.WaitHandle.WaitOne(TimeSpan.FromSeconds(10))) + throw new Exception($"document was not persisted within 10 seconds: {channel}"); - var refreshResult = await Client.Indices.RefreshAsync(targetDataStream.ToString()); - refreshResult.IsValidResponse.Should().BeTrue("{0}", refreshResult.DebugInformation); - var searchResult = await Client.SearchAsync(s => s.Indices(targetDataStream.ToString())); - searchResult.Total.Should().Be(1); + var refreshResult = await Client.Indices.RefreshAsync(targetDataStream.ToString()); + refreshResult.IsValidResponse.Should().BeTrue("{0}", refreshResult.DebugInformation); + var searchResult = await Client.SearchAsync(s => s.Indices(targetDataStream.ToString())); + searchResult.Total.Should().Be(1); - var storedDocument = searchResult.Documents.First(); - storedDocument.Message.Should().Be("hello-world"); + var storedDocument = searchResult.Documents.First(); + storedDocument.Message.Should().Be("hello-world"); - var hit = searchResult.Hits.First(); - hit.Index.Should().StartWith($".ds-{targetDataStream}-"); + var hit = searchResult.Hits.First(); + hit.Index.Should().StartWith($".ds-{targetDataStream}-"); - // the following throws in the 8.0.4 version of the client - // The JSON value could not be converted to Elastic.Clients.Elasticsearch.HealthStatus. Path: $.data_stre... - // await Client.Indices.GetDataStreamAsync(new GetDataStreamRequest(targetDataStream.ToString()) - var getDataStream = - await Client.Transport.RequestAsync(HttpMethod.GET, $"/_data_stream/{targetDataStream}"); + // the following throws in the 8.0.4 version of the client + // The JSON value could not be converted to Elastic.Clients.Elasticsearch.HealthStatus. Path: $.data_stre... + // await Client.Indices.GetDataStreamAsync(new GetDataStreamRequest(targetDataStream.ToString()) + var getDataStream = + await Client.Transport.RequestAsync(HttpMethod.GET, $"/_data_stream/{targetDataStream}"); - getDataStream.ApiCallDetails.HttpStatusCode.Should() - .Be(200, "{0}", getDataStream.ApiCallDetails.DebugInformation); + getDataStream.ApiCallDetails.HttpStatusCode.Should() + .Be(200, "{0}", getDataStream.ApiCallDetails.DebugInformation); - //this ensures the data stream was setup using the expected bootstrapped template - getDataStream.ApiCallDetails.DebugInformation.Should() - .Contain(@$"""template"" : ""{targetDataStream.GetTemplateName()}"""); + //this ensures the data stream was setup using the expected bootstrapped template + getDataStream.ApiCallDetails.DebugInformation.Should() + .Contain(@$"""template"" : ""{targetDataStream.GetTemplateName()}"""); - //this ensures the data stream is managed by the expected ilm_policy - getDataStream.ApiCallDetails.DebugInformation.Should() - .Contain(@"""ilm_policy"" : ""7-days-default"""); - } + //this ensures the data stream is managed by the expected ilm_policy + getDataStream.ApiCallDetails.DebugInformation.Should() + .Contain(@"""ilm_policy"" : ""7-days-default"""); } } diff --git a/tests/Elastic.Ingest.Elasticsearch.IntegrationTests/IndexIngestionTests.cs b/tests/Elastic.Ingest.Elasticsearch.IntegrationTests/IndexIngestionTests.cs index 8ea3345..79391e9 100644 --- a/tests/Elastic.Ingest.Elasticsearch.IntegrationTests/IndexIngestionTests.cs +++ b/tests/Elastic.Ingest.Elasticsearch.IntegrationTests/IndexIngestionTests.cs @@ -12,74 +12,73 @@ using Xunit; using Xunit.Abstractions; -namespace Elastic.Ingest.Elasticsearch.IntegrationTests +namespace Elastic.Ingest.Elasticsearch.IntegrationTests; + +public class IndexIngestionTests : IntegrationTestBase { - public class IndexIngestionTests : IntegrationTestBase + public IndexIngestionTests(IngestionCluster cluster, ITestOutputHelper output) : base(cluster, output) { - public IndexIngestionTests(IngestionCluster cluster, ITestOutputHelper output) : base(cluster, output) - { - } + } - [Fact] - public async Task EnsureDocumentsEndUpInIndex() + [Fact] + public async Task EnsureDocumentsEndUpInIndex() + { + var indexPrefix = "catalog-data-"; + var slim = new CountdownEvent(1); + var options = new IndexChannelOptions(Client.Transport) { - var indexPrefix = "catalog-data-"; - var slim = new CountdownEvent(1); - var options = new IndexChannelOptions(Client.Transport) + IndexFormat = indexPrefix + "{0:yyyy.MM.dd}", + BulkOperationIdLookup = c => c.Id, + BulkUpsertLookup = (c, id) => id == "hello-world-2", + TimestampLookup = c => c.Created, + BufferOptions = new BufferOptions { - IndexFormat = indexPrefix + "{0:yyyy.MM.dd}", - BulkOperationIdLookup = c => c.Id, - BulkUpsertLookup = (c, id) => id == "hello-world-2", - TimestampLookup = c => c.Created, - BufferOptions = new BufferOptions - { - WaitHandle = slim, OutboundBufferMaxSize = 2, - } - }; - var channel = new IndexChannel(options); - var bootstrapped = await channel.BootstrapElasticsearchAsync(BootstrapMethod.Failure, "7-days-default"); - bootstrapped.Should().BeTrue("Expected to be able to bootstrap index channel"); + WaitHandle = slim, OutboundBufferMaxSize = 2, + } + }; + var channel = new IndexChannel(options); + var bootstrapped = await channel.BootstrapElasticsearchAsync(BootstrapMethod.Failure, "7-days-default"); + bootstrapped.Should().BeTrue("Expected to be able to bootstrap index channel"); - var date = DateTimeOffset.Now; - var indexName = string.Format(options.IndexFormat, date); + var date = DateTimeOffset.Now; + var indexName = string.Format(options.IndexFormat, date); - var index = await Client.Indices.GetAsync(new GetIndexRequest(indexName)); - index.Indices.Should().BeNullOrEmpty(); + var index = await Client.Indices.GetAsync(new GetIndexRequest(indexName)); + index.Indices.Should().BeNullOrEmpty(); - channel.TryWrite(new CatalogDocument { Created = date, Title = "Hello World!", Id = "hello-world" }); - channel.TryWrite(new CatalogDocument { Created = date, Title = "Hello World!", Id = "hello-world-2" }); - if (!slim.WaitHandle.WaitOne(TimeSpan.FromSeconds(10))) - throw new Exception($"ecs document was not persisted within 10 seconds: {channel}"); + channel.TryWrite(new CatalogDocument { Created = date, Title = "Hello World!", Id = "hello-world" }); + channel.TryWrite(new CatalogDocument { Created = date, Title = "Hello World!", Id = "hello-world-2" }); + if (!slim.WaitHandle.WaitOne(TimeSpan.FromSeconds(10))) + throw new Exception($"ecs document was not persisted within 10 seconds: {channel}"); - var refreshResult = await Client.Indices.RefreshAsync(indexName); - refreshResult.IsValidResponse.Should().BeTrue("{0}", refreshResult.DebugInformation); - var searchResult = await Client.SearchAsync(s => s.Indices(indexName)); - searchResult.Total.Should().Be(2); + var refreshResult = await Client.Indices.RefreshAsync(indexName); + refreshResult.IsValidResponse.Should().BeTrue("{0}", refreshResult.DebugInformation); + var searchResult = await Client.SearchAsync(s => s.Indices(indexName)); + searchResult.Total.Should().Be(2); - var storedDocument = searchResult.Documents.First(); - storedDocument.Id.Should().Be("hello-world"); - storedDocument.Title.Should().Be("Hello World!"); + var storedDocument = searchResult.Documents.First(); + storedDocument.Id.Should().Be("hello-world"); + storedDocument.Title.Should().Be("Hello World!"); - var hit = searchResult.Hits.First(); - hit.Index.Should().Be(indexName); + var hit = searchResult.Hits.First(); + hit.Index.Should().Be(indexName); - index = await Client.Indices.GetAsync(new GetIndexRequest(indexName)); - index.Indices.Should().NotBeNullOrEmpty(); + index = await Client.Indices.GetAsync(new GetIndexRequest(indexName)); + index.Indices.Should().NotBeNullOrEmpty(); - index.Indices[indexName].Settings?.Index?.Lifecycle?.Name?.Should().NotBeNull().And.Be("7-days-default"); + index.Indices[indexName].Settings?.Index?.Lifecycle?.Name?.Should().NotBeNull().And.Be("7-days-default"); - // Bug in client, for now assume template was applied because the ILM policy is set on the index. - /* - // The JSON value could not be converted to Elastic.Clients.Elasticsearch.Names. Path: $.index_templates[0].index_template.index_patterns | LineNumber: 5 | BytePositionInLine: 28. + // Bug in client, for now assume template was applied because the ILM policy is set on the index. + /* + // The JSON value could not be converted to Elastic.Clients.Elasticsearch.Names. Path: $.index_templates[0].index_template.index_patterns | LineNumber: 5 | BytePositionInLine: 28. - var templateName = string.Format(options.IndexFormat, "template"); - var template = await Client.Indices.GetIndexTemplateAsync(new GetIndexTemplateRequest(templateName)); - template.IsValidResponse.Should().BeTrue("{0}", template.DebugInformation); - template.IndexTemplates.First().Should().NotBeNull(); - template.IndexTemplates.First().Name.Should().Be(templateName); - //template.IndexTemplates.First().IndexTemplate.Template..Should().Be(templateName); - */ + var templateName = string.Format(options.IndexFormat, "template"); + var template = await Client.Indices.GetIndexTemplateAsync(new GetIndexTemplateRequest(templateName)); + template.IsValidResponse.Should().BeTrue("{0}", template.DebugInformation); + template.IndexTemplates.First().Should().NotBeNull(); + template.IndexTemplates.First().Name.Should().Be(templateName); + //template.IndexTemplates.First().IndexTemplate.Template..Should().Be(templateName); + */ - } } } diff --git a/tests/Elastic.Ingest.Elasticsearch.IntegrationTests/IngestionCluster.cs b/tests/Elastic.Ingest.Elasticsearch.IntegrationTests/IngestionCluster.cs index 89de9b7..4ca6571 100644 --- a/tests/Elastic.Ingest.Elasticsearch.IntegrationTests/IngestionCluster.cs +++ b/tests/Elastic.Ingest.Elasticsearch.IntegrationTests/IngestionCluster.cs @@ -11,34 +11,33 @@ [assembly: TestFramework("Elastic.Elasticsearch.Xunit.Sdk.ElasticTestFramework", "Elastic.Elasticsearch.Xunit")] -namespace Elastic.Ingest.Elasticsearch.IntegrationTests +namespace Elastic.Ingest.Elasticsearch.IntegrationTests; + +/// Declare our cluster that we want to inject into our test classes +public class IngestionCluster : XunitClusterBase { - /// Declare our cluster that we want to inject into our test classes - public class IngestionCluster : XunitClusterBase - { - public IngestionCluster() : base(new XunitClusterConfiguration("8.7.0") { StartingPortNumber = 9202 }) { } + public IngestionCluster() : base(new XunitClusterConfiguration("8.7.0") { StartingPortNumber = 9202 }) { } - public ElasticsearchClient CreateClient(ITestOutputHelper output) => - this.GetOrAddClient(_ => - { - var hostName = (System.Diagnostics.Process.GetProcessesByName("mitmproxy").Any() - ? "ipv4.fiddler" - : "localhost"); - var nodes = NodesUris(hostName); - var connectionPool = new StaticNodePool(nodes); - var settings = new ElasticsearchClientSettings(connectionPool) - .Proxy(new Uri("http://ipv4.fiddler:8080"), null!, null!) - .RequestTimeout(TimeSpan.FromSeconds(5)) - .OnRequestCompleted(d => + public ElasticsearchClient CreateClient(ITestOutputHelper output) => + this.GetOrAddClient(_ => + { + var hostName = (System.Diagnostics.Process.GetProcessesByName("mitmproxy").Any() + ? "ipv4.fiddler" + : "localhost"); + var nodes = NodesUris(hostName); + var connectionPool = new StaticNodePool(nodes); + var settings = new ElasticsearchClientSettings(connectionPool) + .Proxy(new Uri("http://ipv4.fiddler:8080"), null!, null!) + .RequestTimeout(TimeSpan.FromSeconds(5)) + .OnRequestCompleted(d => + { + try { output.WriteLine(d.DebugInformation);} + catch { - try { output.WriteLine(d.DebugInformation);} - catch - { - // ignored - } - }) - .EnableDebugMode(); - return new ElasticsearchClient(settings); - }); - } + // ignored + } + }) + .EnableDebugMode(); + return new ElasticsearchClient(settings); + }); } diff --git a/tests/Elastic.Ingest.Elasticsearch.Tests/BulkResponseBuilder.cs b/tests/Elastic.Ingest.Elasticsearch.Tests/BulkResponseBuilder.cs index 4444dc2..34f3400 100644 --- a/tests/Elastic.Ingest.Elasticsearch.Tests/BulkResponseBuilder.cs +++ b/tests/Elastic.Ingest.Elasticsearch.Tests/BulkResponseBuilder.cs @@ -4,33 +4,32 @@ using System.Linq; -namespace Elastic.Ingest.Elasticsearch.Tests +namespace Elastic.Ingest.Elasticsearch.Tests; + +public static class BulkResponseBuilder { - public static class BulkResponseBuilder + public static object CreateResponse(params int[] statusCodes) => new { - public static object CreateResponse(params int[] statusCodes) => new + items = statusCodes.Select(status => new { index = CreateItemResponse(status) }).ToArray() + }; + + private static object CreateItemResponse(int statusCode) => + statusCode switch { - items = statusCodes.Select(status => new { index = CreateItemResponse(status) }).ToArray() + 429 => new { status = statusCode, error = CreateErrorObject(statusCode) }, + 400 => new { status = statusCode, error = CreateErrorObject(statusCode) }, + _ => new { status = statusCode } }; - private static object CreateItemResponse(int statusCode) => - statusCode switch - { - 429 => new { status = statusCode, error = CreateErrorObject(statusCode) }, - 400 => new { status = statusCode, error = CreateErrorObject(statusCode) }, - _ => new { status = statusCode } - }; - - private static object CreateErrorObject(in int statusCode) => - statusCode switch - { - 500 => new { index = "index", reason = "bad request 500", type = "some_exception" }, - 502 => new { index = "index", reason = "bad request 502", type = "some_exception" }, - 503 => new { index = "index", reason = "bad request 503", type = "some_exception" }, - 504 => new { index = "index", reason = "bad request 504", type = "some_exception" }, - 429 => new { index = "index", reason = "rejected execution of org.", type = "es_rejected_execution_exception" }, - 400 => new { index = "BADINDEX", reason = "invalid index name", type = "invalid_index_name_exception" }, - _ => new { status = statusCode } - }; - } + private static object CreateErrorObject(in int statusCode) => + statusCode switch + { + 500 => new { index = "index", reason = "bad request 500", type = "some_exception" }, + 502 => new { index = "index", reason = "bad request 502", type = "some_exception" }, + 503 => new { index = "index", reason = "bad request 503", type = "some_exception" }, + 504 => new { index = "index", reason = "bad request 504", type = "some_exception" }, + 429 => new { index = "index", reason = "rejected execution of org.", type = "es_rejected_execution_exception" }, + 400 => new { index = "BADINDEX", reason = "invalid index name", type = "invalid_index_name_exception" }, + _ => new { status = statusCode } + }; } diff --git a/tests/Elastic.Ingest.Elasticsearch.Tests/ElasticsearchChannelTests.cs b/tests/Elastic.Ingest.Elasticsearch.Tests/ElasticsearchChannelTests.cs index 70131a8..85d8047 100644 --- a/tests/Elastic.Ingest.Elasticsearch.Tests/ElasticsearchChannelTests.cs +++ b/tests/Elastic.Ingest.Elasticsearch.Tests/ElasticsearchChannelTests.cs @@ -7,120 +7,119 @@ using FluentAssertions; using Xunit; -namespace Elastic.Ingest.Elasticsearch.Tests +namespace Elastic.Ingest.Elasticsearch.Tests; + +public class ElasticsearchChannelTests { - public class ElasticsearchChannelTests + [Fact] + public void RejectionsAreReportedAndNotRetried() + { + var client = TestSetup.CreateClient(v => v + .ClientCalls(c => c.BulkResponse(400, 400)) + .ClientCalls(c => c.BulkResponse(200, 200)) + ); + using var session = TestSetup.CreateTestSession(client); + session.WriteAndWait(events: 2); + + session.LastException.Should().BeNull(); + + session.Rejections.Should().Be(1); + session.TotalBulkRequests.Should().Be(1); + session.TotalRetries.Should().Be(0); + + session.WriteAndWait(events: 2); + + session.Rejections.Should().Be(1); + session.TotalBulkRequests.Should().Be(2); + session.TotalRetries.Should().Be(0); + } + + [Fact] + public void BackoffRetries() { - [Fact] - public void RejectionsAreReportedAndNotRetried() - { - var client = TestSetup.CreateClient(v => v - .ClientCalls(c => c.BulkResponse(400, 400)) - .ClientCalls(c => c.BulkResponse(200, 200)) - ); - using var session = TestSetup.CreateTestSession(client); - session.WriteAndWait(events: 2); - - session.LastException.Should().BeNull(); - - session.Rejections.Should().Be(1); - session.TotalBulkRequests.Should().Be(1); - session.TotalRetries.Should().Be(0); - - session.WriteAndWait(events: 2); - - session.Rejections.Should().Be(1); - session.TotalBulkRequests.Should().Be(2); - session.TotalRetries.Should().Be(0); - } - - [Fact] - public void BackoffRetries() - { - var client = TestSetup.CreateClient(v => v - // first two events keep bouncing - .ClientCalls(c => c.BulkResponse(429, 429)) - .ClientCalls(c => c.BulkResponse(429, 429)) //retry 1 - .ClientCalls(c => c.BulkResponse(429, 429)) //retry 2 - // finally succeeds - .ClientCalls(c => c.BulkResponse(200, 200)) //retry 3 - // next two succeed straight away - .ClientCalls(c => c.BulkResponse(200, 200)) //next batch - ); - using var session = TestSetup.CreateTestSession(client); - session.WriteAndWait(events: 2); - - session.LastException.Should().BeNull(); - - session.Rejections.Should().Be(0); - session.TotalBulkRequests.Should().Be(4); - session.TotalRetries.Should().Be(3); - - session.WriteAndWait(events: 2); - - session.TotalBulkRequests.Should().Be(5); - session.TotalRetries.Should().Be(3); - session.Rejections.Should().Be(0); - - } - - [Fact] - public void BackoffTooMuchEndsUpOnDLQ() - { - var client = TestSetup.CreateClient(v => v - // first two events keep bouncing - .ClientCalls(c => c.BulkResponse(429, 429)) - .ClientCalls(c => c.BulkResponse(429, 429)) //retry 1 - .ClientCalls(c => c.BulkResponse(429, 429)) //retry 2 - .ClientCalls(c => c.BulkResponse(429, 429)) //retry 3 - // next two succeed straight away - .ClientCalls(c => c.BulkResponse(200, 200)) //next batch - ); - using var session = TestSetup.CreateTestSession(client); - session.WriteAndWait(events: 2); - - session.LastException.Should().BeNull(); - - session.TotalBulkRequests.Should().Be(4); - session.TotalRetries.Should().Be(3); - session.MaxRetriesExceeded.Should().Be(1); - session.Rejections.Should().Be(0); - - session.WriteAndWait(events: 2); - - session.TotalBulkRequests.Should().Be(5); - session.TotalRetries.Should().Be(3); - session.Rejections.Should().Be(0); - } - - [Fact] - public void ExceptionDoesNotHaltProcessingAndIsReported() - { - var client = TestSetup.CreateClient(v => v - // first two events throws an exception in the client call - .ClientCalls(c => c.Fails(TimesHelper.Once, new Exception("boom!"))) - // next two succeed straight away - .ClientCalls(c => c.BulkResponse(200, 200)) //next batch - ); - using var session = TestSetup.CreateTestSession(client); - session.WriteAndWait(events: 2); - - session.LastException.Should().NotBeNull(); - session.LastException.Message.Should().Be("boom!"); - - session.TotalBulkRequests.Should().Be(1); - session.TotalBulkResponses.Should().Be(0); - session.TotalRetries.Should().Be(0); - session.MaxRetriesExceeded.Should().Be(0); - session.Rejections.Should().Be(0); - - session.WriteAndWait(events: 2); - - session.TotalBulkRequests.Should().Be(2); - session.TotalBulkResponses.Should().Be(1); - session.TotalRetries.Should().Be(0); - session.Rejections.Should().Be(0); - - } + var client = TestSetup.CreateClient(v => v + // first two events keep bouncing + .ClientCalls(c => c.BulkResponse(429, 429)) + .ClientCalls(c => c.BulkResponse(429, 429)) //retry 1 + .ClientCalls(c => c.BulkResponse(429, 429)) //retry 2 + // finally succeeds + .ClientCalls(c => c.BulkResponse(200, 200)) //retry 3 + // next two succeed straight away + .ClientCalls(c => c.BulkResponse(200, 200)) //next batch + ); + using var session = TestSetup.CreateTestSession(client); + session.WriteAndWait(events: 2); + + session.LastException.Should().BeNull(); + + session.Rejections.Should().Be(0); + session.TotalBulkRequests.Should().Be(4); + session.TotalRetries.Should().Be(3); + + session.WriteAndWait(events: 2); + + session.TotalBulkRequests.Should().Be(5); + session.TotalRetries.Should().Be(3); + session.Rejections.Should().Be(0); + + } + + [Fact] + public void BackoffTooMuchEndsUpOnDLQ() + { + var client = TestSetup.CreateClient(v => v + // first two events keep bouncing + .ClientCalls(c => c.BulkResponse(429, 429)) + .ClientCalls(c => c.BulkResponse(429, 429)) //retry 1 + .ClientCalls(c => c.BulkResponse(429, 429)) //retry 2 + .ClientCalls(c => c.BulkResponse(429, 429)) //retry 3 + // next two succeed straight away + .ClientCalls(c => c.BulkResponse(200, 200)) //next batch + ); + using var session = TestSetup.CreateTestSession(client); + session.WriteAndWait(events: 2); + + session.LastException.Should().BeNull(); + + session.TotalBulkRequests.Should().Be(4); + session.TotalRetries.Should().Be(3); + session.MaxRetriesExceeded.Should().Be(1); + session.Rejections.Should().Be(0); + + session.WriteAndWait(events: 2); + + session.TotalBulkRequests.Should().Be(5); + session.TotalRetries.Should().Be(3); + session.Rejections.Should().Be(0); + } + + [Fact] + public void ExceptionDoesNotHaltProcessingAndIsReported() + { + var client = TestSetup.CreateClient(v => v + // first two events throws an exception in the client call + .ClientCalls(c => c.Fails(TimesHelper.Once, new Exception("boom!"))) + // next two succeed straight away + .ClientCalls(c => c.BulkResponse(200, 200)) //next batch + ); + using var session = TestSetup.CreateTestSession(client); + session.WriteAndWait(events: 2); + + session.LastException.Should().NotBeNull(); + session.LastException.Message.Should().Be("boom!"); + + session.TotalBulkRequests.Should().Be(1); + session.TotalBulkResponses.Should().Be(0); + session.TotalRetries.Should().Be(0); + session.MaxRetriesExceeded.Should().Be(0); + session.Rejections.Should().Be(0); + + session.WriteAndWait(events: 2); + + session.TotalBulkRequests.Should().Be(2); + session.TotalBulkResponses.Should().Be(1); + session.TotalRetries.Should().Be(0); + session.Rejections.Should().Be(0); + } } diff --git a/tests/Elastic.Ingest.Elasticsearch.Tests/SerializationTests.cs b/tests/Elastic.Ingest.Elasticsearch.Tests/SerializationTests.cs index 1f164e1..5fedabb 100644 --- a/tests/Elastic.Ingest.Elasticsearch.Tests/SerializationTests.cs +++ b/tests/Elastic.Ingest.Elasticsearch.Tests/SerializationTests.cs @@ -7,21 +7,20 @@ using FluentAssertions; using Xunit; -namespace Elastic.Ingest.Elasticsearch.Tests +namespace Elastic.Ingest.Elasticsearch.Tests; + +public class SerializationTests { - public class SerializationTests + [Fact] + public void CanSerializeBulkResponseItem() { - [Fact] - public void CanSerializeBulkResponseItem() - { - var json = "{\"index\":{\"status\":200}}"; - var item = JsonSerializer.Deserialize(json); + var json = "{\"index\":{\"status\":200}}"; + var item = JsonSerializer.Deserialize(json); - item.Should().NotBeNull(); + item.Should().NotBeNull(); - var actual = JsonSerializer.Serialize(item); + var actual = JsonSerializer.Serialize(item); - actual.Should().Be(json); - } + actual.Should().Be(json); } } diff --git a/tests/Elastic.Ingest.Elasticsearch.Tests/Setup.cs b/tests/Elastic.Ingest.Elasticsearch.Tests/Setup.cs index f4a14c5..7581d0c 100644 --- a/tests/Elastic.Ingest.Elasticsearch.Tests/Setup.cs +++ b/tests/Elastic.Ingest.Elasticsearch.Tests/Setup.cs @@ -12,103 +12,102 @@ using Elastic.Transport.VirtualizedCluster.Components; using Elastic.Transport.VirtualizedCluster.Rules; -namespace Elastic.Ingest.Elasticsearch.Tests +namespace Elastic.Ingest.Elasticsearch.Tests; + +public static class TestSetup { - public static class TestSetup + public static HttpTransport CreateClient(Func setup) { - public static HttpTransport CreateClient(Func setup) - { - var cluster = Virtual.Elasticsearch.Bootstrap(numberOfNodes: 1).Ping(c=>c.SucceedAlways()); - var virtualSettings = setup(cluster) - .StaticNodePool() - .Settings(s=>s.DisablePing()); - - //var audit = new Auditor(() => virtualSettings); - //audit.VisualizeCalls(cluster.ClientCallRules.Count); - - var settings = new TransportConfiguration(virtualSettings.ConnectionPool, virtualSettings.Connection) - .DisablePing() - .EnableDebugMode(); - return new DefaultHttpTransport(settings); - } + var cluster = Virtual.Elasticsearch.Bootstrap(numberOfNodes: 1).Ping(c=>c.SucceedAlways()); + var virtualSettings = setup(cluster) + .StaticNodePool() + .Settings(s=>s.DisablePing()); + + //var audit = new Auditor(() => virtualSettings); + //audit.VisualizeCalls(cluster.ClientCallRules.Count); + + var settings = new TransportConfiguration(virtualSettings.ConnectionPool, virtualSettings.Connection) + .DisablePing() + .EnableDebugMode(); + return new DefaultHttpTransport(settings); + } - public static ClientCallRule BulkResponse(this ClientCallRule rule, params int[] statusCodes) => - rule.Succeeds(TimesHelper.Once).ReturnResponse(BulkResponseBuilder.CreateResponse(statusCodes)); + public static ClientCallRule BulkResponse(this ClientCallRule rule, params int[] statusCodes) => + rule.Succeeds(TimesHelper.Once).ReturnResponse(BulkResponseBuilder.CreateResponse(statusCodes)); - public class TestSession : IDisposable - { - private int _rejections; - private int _requests; - private int _responses; - private int _retries; - private int _maxRetriesExceeded; + public class TestSession : IDisposable + { + private int _rejections; + private int _requests; + private int _responses; + private int _retries; + private int _maxRetriesExceeded; - public TestSession(HttpTransport transport) + public TestSession(HttpTransport transport) + { + Transport = transport; + BufferOptions = new BufferOptions { - Transport = transport; - BufferOptions = new BufferOptions - { - ExportMaxConcurrency = 1, - OutboundBufferMaxSize = 2, - OutboundBufferMaxLifetime = TimeSpan.FromSeconds(10), - WaitHandle = WaitHandle, - ExportMaxRetries = 3, - ExportBackoffPeriod = _ => TimeSpan.FromMilliseconds(1), - }; - ChannelOptions = new IndexChannelOptions(transport) - { - BufferOptions = BufferOptions, + ExportMaxConcurrency = 1, + OutboundBufferMaxSize = 2, + OutboundBufferMaxLifetime = TimeSpan.FromSeconds(10), + WaitHandle = WaitHandle, + ExportMaxRetries = 3, + ExportBackoffPeriod = _ => TimeSpan.FromMilliseconds(1), + }; + ChannelOptions = new IndexChannelOptions(transport) + { + BufferOptions = BufferOptions, #pragma warning disable CS0618 - UseReadOnlyMemory = true, + UseReadOnlyMemory = true, #pragma warning restore CS0618 - ServerRejectionCallback = (_) => Interlocked.Increment(ref _rejections), - ExportItemsAttemptCallback = (_, _) => Interlocked.Increment(ref _requests), - ExportResponseCallback = (_, _) => Interlocked.Increment(ref _responses), - ExportMaxRetriesCallback = (_) => Interlocked.Increment(ref _maxRetriesExceeded), - ExportRetryCallback = (_) => Interlocked.Increment(ref _retries), - ExportExceptionCallback= (e) => LastException = e - }; - Channel = new IndexChannel(ChannelOptions); - } - - public IndexChannel Channel { get; } + ServerRejectionCallback = (_) => Interlocked.Increment(ref _rejections), + ExportItemsAttemptCallback = (_, _) => Interlocked.Increment(ref _requests), + ExportResponseCallback = (_, _) => Interlocked.Increment(ref _responses), + ExportMaxRetriesCallback = (_) => Interlocked.Increment(ref _maxRetriesExceeded), + ExportRetryCallback = (_) => Interlocked.Increment(ref _retries), + ExportExceptionCallback= (e) => LastException = e + }; + Channel = new IndexChannel(ChannelOptions); + } - public HttpTransport Transport { get; } + public IndexChannel Channel { get; } - public IndexChannelOptions ChannelOptions { get; } + public HttpTransport Transport { get; } - public BufferOptions BufferOptions { get; } + public IndexChannelOptions ChannelOptions { get; } - public CountdownEvent WaitHandle { get; } = new CountdownEvent(1); + public BufferOptions BufferOptions { get; } - public int Rejections => _rejections; - public int TotalBulkRequests => _requests; - public int TotalBulkResponses => _responses; - public int TotalRetries => _retries; - public int MaxRetriesExceeded => _maxRetriesExceeded; - public Exception LastException { get; private set; } + public CountdownEvent WaitHandle { get; } = new CountdownEvent(1); - public void Wait() - { - WaitHandle.Wait(TimeSpan.FromSeconds(10)); - WaitHandle.Reset(); - } + public int Rejections => _rejections; + public int TotalBulkRequests => _requests; + public int TotalBulkResponses => _responses; + public int TotalRetries => _retries; + public int MaxRetriesExceeded => _maxRetriesExceeded; + public Exception LastException { get; private set; } - public void Dispose() - { - Channel?.Dispose(); - WaitHandle?.Dispose(); - } + public void Wait() + { + WaitHandle.Wait(TimeSpan.FromSeconds(10)); + WaitHandle.Reset(); } - public static TestSession CreateTestSession(HttpTransport transport) => - new TestSession(transport); - - public static void WriteAndWait(this TestSession session, int events = 1) + public void Dispose() { - foreach (var _ in Enumerable.Range(0, events)) - session.Channel.TryWrite(new TestDocument { Timestamp = DateTimeOffset.UtcNow }); - session.Wait(); + Channel?.Dispose(); + WaitHandle?.Dispose(); } } + + public static TestSession CreateTestSession(HttpTransport transport) => + new TestSession(transport); + + public static void WriteAndWait(this TestSession session, int events = 1) + { + foreach (var _ in Enumerable.Range(0, events)) + session.Channel.TryWrite(new TestDocument { Timestamp = DateTimeOffset.UtcNow }); + session.Wait(); + } }