From 86ce2bc65bb61a82537ffd2fb5fce767fc46246f Mon Sep 17 00:00:00 2001 From: Ledjon Behluli Date: Thu, 26 Sep 2024 00:30:14 +0200 Subject: [PATCH] addressed remaning comments --- .../Options/ActivationRebalancerOptions.cs | 33 ++++++++++++++--- .../Rebalancing/ActivationRebalancerWorker.cs | 31 +++++----------- .../RebalancingOptionsTests.cs | 35 ++++++++++--------- 3 files changed, 56 insertions(+), 43 deletions(-) diff --git a/src/Orleans.Runtime/Configuration/Options/ActivationRebalancerOptions.cs b/src/Orleans.Runtime/Configuration/Options/ActivationRebalancerOptions.cs index ae2d708ce2..b2dc296abe 100644 --- a/src/Orleans.Runtime/Configuration/Options/ActivationRebalancerOptions.cs +++ b/src/Orleans.Runtime/Configuration/Options/ActivationRebalancerOptions.cs @@ -68,17 +68,37 @@ public sealed class ActivationRebalancerOptions /// /// Determines whether should be scaled dynamically /// based on the total number of activations. When set to , the allowed entropy - /// deviation will increase logarithmically after reaching an activation threshold (10,000 activations), - /// and will cap at the maximum (0.1 deviation). + /// deviation will increase logarithmically after reaching , + /// and will cap at . /// /// This is in place because a deviation of say 10 activations has far lesser /// impact on a total of 100,000 activations than it does for say 1,000 activations. - public bool ScaleAllowedEntropyDeviation { get; set; } = DEFAULT_SCALE_DEFAULT_ALLOWED_ENTROPY_DEVIATION; + public bool ScaleAllowedEntropyDeviation { get; set; } = DEFAULT_SCALE_ALLOWED_ENTROPY_DEVIATION; /// /// The default value of . /// - public const bool DEFAULT_SCALE_DEFAULT_ALLOWED_ENTROPY_DEVIATION = true; + public const bool DEFAULT_SCALE_ALLOWED_ENTROPY_DEVIATION = true; + + /// + /// The maximum value allowed when is . + /// + public const double MAX_SCALED_ENTROPY_DEVIATION = 0.1d; + + /// + /// Determines the number of activations that must be active during any rebalancing cycle, in order for + /// (if, and only if, its ) to begin scaling the . + /// + /// + /// Allowed range: [1000-∞) + /// Values lower than 5000 are highly discouraged. + /// + public int ScaledEntropyDeviationActivationThreshold { get; set; } + + /// + /// The default value of . + /// + public const int DEFAULT_SCALED_ENTROPY_DEVIATION_ACTIVATION_THRESHOLD = 10_000; /// /// Represents the weight that is given to the number of rebalancing cycles that have passed during a rebalancing session. @@ -163,5 +183,10 @@ public void ValidateConfiguration() { throw new OrleansConfigurationException($"{nameof(ActivationRebalancerOptions.ActivationMigrationCountLimit)} must be greater than 0"); } + + if (_options.ScaledEntropyDeviationActivationThreshold < 1_000) + { + throw new OrleansConfigurationException($"{nameof(ActivationRebalancerOptions.ScaledEntropyDeviationActivationThreshold)} must be greater than or equal to 1000"); + } } } \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancerWorker.cs b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancerWorker.cs index 514364336d..32a34e7cfb 100644 --- a/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancerWorker.cs +++ b/src/Orleans.Runtime/Placement/Rebalancing/ActivationRebalancerWorker.cs @@ -3,7 +3,6 @@ using System.Collections.Immutable; using System.Diagnostics; using System.Linq; -using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Threading; using System.Threading.Tasks; @@ -12,7 +11,6 @@ using Orleans.Configuration; using Orleans.Placement; using Orleans.Placement.Rebalancing; -using Orleans.Statistics; #nullable enable @@ -46,9 +44,9 @@ private enum StopReason /// SessionStarting, /// - /// Current session has failed. + /// Current session has stagnated. /// - SessionFailed, + SessionStagnated, /// /// Current session has completed successfully till end /// @@ -255,7 +253,7 @@ private async Task RunRebalancingCycle(CancellationToken cancellationToken) if (_stagnantCycles >= _options.MaxStagnantCycles) { LogMaxStagnantCyclesReached(_stagnantCycles); - StopSession(StopReason.SessionFailed); + StopSession(StopReason.SessionStagnated); return; } @@ -323,15 +321,12 @@ private async Task RunRebalancingCycle(CancellationToken cancellationToken) var addressPairs = FormSiloPairs(snapshot); var migrationTasks = new List(); + Debug.Assert(addressPairs.Count % 2 == 0); + for (var i = 0; i < addressPairs.Count; i++) { (var lowSilo, var highSilo) = addressPairs[i]; - if (lowSilo.IsSameLogicalSilo(highSilo)) - { - continue; - } - var difference = Math.Abs( (snapshot[lowSilo].ActivationCount - idealDistributions[lowSilo]) - (snapshot[highSilo].ActivationCount - idealDistributions[highSilo])); @@ -439,20 +434,17 @@ private static double ComputeHarmonicMean(Dictionary 0); - var logFactor = (int)Math.Log10(totalActivations / ActivationThreshold); + var logFactor = (int)Math.Log10(totalActivations / _options.ScaledEntropyDeviationActivationThreshold); var adjustedDeviation = _options.AllowedEntropyDeviation * Math.Pow(10, logFactor); - return Math.Min(adjustedDeviation, MaxAllowedEntropyDeviation); + return Math.Min(adjustedDeviation, ActivationRebalancerOptions.MAX_SCALED_ENTROPY_DEVIATION); } private double ComputeAdaptiveScaling(int siloCount, int rebalancingCycle) @@ -482,11 +474,6 @@ private double ComputeAdaptiveScaling(int siloCount, int rebalancingCycle) right--; } - if (left == right) // Odd number of silos - { - pairs.Add((sorted[left].Key, sorted[left].Key)); // Pair this silo with itself - } - return pairs; } @@ -519,7 +506,7 @@ private void StopSession(StopReason reason, TimeSpan? duration = null) _suspendedUntilTs = 0; } break; - case StopReason.SessionFailed: + case StopReason.SessionStagnated: { _failedSessions++; SuspendFor(backoffProvider.Next(_failedSessions)); diff --git a/test/TesterInternal/ActivationRebalancingTests/RebalancingOptionsTests.cs b/test/TesterInternal/ActivationRebalancingTests/RebalancingOptionsTests.cs index 13de0e4afd..a880398c91 100644 --- a/test/TesterInternal/ActivationRebalancingTests/RebalancingOptionsTests.cs +++ b/test/TesterInternal/ActivationRebalancingTests/RebalancingOptionsTests.cs @@ -17,32 +17,32 @@ public void ConstantsShouldNotChange() Assert.Equal(0.0001d, ActivationRebalancerOptions.DEFAULT_ALLOWED_ENTROPY_DEVIATION); Assert.Equal(0.1d, ActivationRebalancerOptions.DEFAULT_CYCLE_NUMBER_WEIGHT); Assert.Equal(0.1d, ActivationRebalancerOptions.DEFAULT_SILO_NUMBER_WEIGHT); + Assert.Equal(0.1d, ActivationRebalancerOptions.MAX_SCALED_ENTROPY_DEVIATION); + Assert.Equal(10_000, ActivationRebalancerOptions.DEFAULT_SCALED_ENTROPY_DEVIATION_ACTIVATION_THRESHOLD); Assert.Equal(int.MaxValue, ActivationRebalancerOptions.DEFAULT_ACTIVATION_MIGRATION_COUNT_LIMIT); - Assert.True(ActivationRebalancerOptions.DEFAULT_SCALE_DEFAULT_ALLOWED_ENTROPY_DEVIATION); + Assert.True(ActivationRebalancerOptions.DEFAULT_SCALE_ALLOWED_ENTROPY_DEVIATION); } [Theory] - [InlineData(999, 1, 0.1, 0.5, 0.5, 0.5, 2, 500)] - [InlineData(1000, 1, 0.1, 0.5, 0.5, 0.5, 2, 500)] - [InlineData(1000, 1, 0, 0.5, 0.5, 0.5, 2, 500)] - [InlineData(1000, 1, 0.1, -0.001, 0.5, 0.5, 2, 500)] - [InlineData(1000, 1, 0.1, 0.011, 0.5, 0.5, 2, 500)] - [InlineData(1000, 1, 0.1, 0.001, -0.1, 0.5, 1.1, 500)] - [InlineData(1000, 1, 0.1, 0.001, 1.1, 0.5, 2, 500)] - [InlineData(1000, 1, 0.1, 0.001, 0.5, 0, 2, 500)] - [InlineData(1000, 1, 0.1, 0.001, 0.5, 1.1, 2, 500)] - [InlineData(1000, 1, 0.1, 0.001, 0.5, 0.5, -0.1, 500)] - [InlineData(1000, 1, 0.1, 0.001, 0.5, 0.5, 1.1, 1)] - [InlineData(1000, 1, 0.1, 0.001, 0.5, 0.5, 1.1, 0)] + [InlineData(1000, 1, 0, 0.2, 0.2, 0, -0.1, 0, 999)] + [InlineData(2000, 2, -1, 0.05, 0.05, 0.5, 1.1, 10, 500)] + [InlineData(1000, 1, 2, 0, 0.05, 0.5, 0.5, 10, 999)] + [InlineData(1000, 1, 2, 0.05, 0, 0.5, 0.5, 10, 999)] + [InlineData(1000, 1, 2, 0.05, 0.05, -0.1, 0.5, 10, 999)] + [InlineData(1000, 1, 2, 0.05, 0.05, 0.5, 1.1, 10, 999)] + [InlineData(1000, 1, 2, 0.05, 0.05, 0.5, 0.5, 0, 999)] + [InlineData(1000, 1, 2, 0.05, 0.05, 0.5, 0.5, 10, 999)] + public void InvalidOptionsShouldThrow( - double sessionCyclePeriodMilliseconds, - double publisherRefreshTimeSeconds, + int sessionCyclePeriodMilliseconds, + int publisherRefreshTimeSeconds, int maxStagnantCycles, double entropyQuantum, double allowedEntropyDeviation, double cycleNumberWeight, double siloNumberWeight, - int activationMigrationCountLimit) + int activationMigrationCountLimit, + int scaledEntropyDeviationActivationThreshold) { var publisherOptions = new DeploymentLoadPublisherOptions { @@ -57,7 +57,8 @@ public void InvalidOptionsShouldThrow( AllowedEntropyDeviation = allowedEntropyDeviation, CycleNumberWeight = cycleNumberWeight, SiloNumberWeight = siloNumberWeight, - ActivationMigrationCountLimit = activationMigrationCountLimit + ActivationMigrationCountLimit = activationMigrationCountLimit, + ScaledEntropyDeviationActivationThreshold = scaledEntropyDeviationActivationThreshold }; var validator = new ActivationRebalancerOptionsValidator(Options.Create(options), Options.Create(publisherOptions));