Skip to content

Commit cdccd96

Browse files
RobertLucianRobertLucian
authored
Prevent redundant autoscaling events on the autoscaler (#2348)
1 parent a6366b7 commit cdccd96

File tree

5 files changed

+47
-37
lines changed

5 files changed

+47
-37
lines changed

‎pkg/autoscaler/async_scaler.go

+7-4
Original file line numberDiff line numberDiff line change
@@ -52,8 +52,7 @@ func (s *AsyncScaler) Scale(apiName string, request int32) error {
5252
return errors.ErrorUnexpected("unable to find k8s deployment", apiName)
5353
}
5454

55-
current := deployment.Status.Replicas
56-
if current == request {
55+
if deployment.Spec.Replicas != nil && *deployment.Spec.Replicas == request {
5756
return nil
5857
}
5958

@@ -116,7 +115,7 @@ func (s *AsyncScaler) GetAutoscalingSpec(apiName string) (*userconfig.Autoscalin
116115
return autoscalingSpec, nil
117116
}
118117

119-
func (s *AsyncScaler) CurrentReplicas(apiName string) (int32, error) {
118+
func (s *AsyncScaler) CurrentRequestedReplicas(apiName string) (int32, error) {
120119
deployment, err := s.k8s.GetDeployment(workloads.K8sName(apiName))
121120
if err != nil {
122121
return 0, err
@@ -126,5 +125,9 @@ func (s *AsyncScaler) CurrentReplicas(apiName string) (int32, error) {
126125
return 0, errors.ErrorUnexpected("unable to find k8s deployment", apiName)
127126
}
128127

129-
return deployment.Status.Replicas, nil
128+
if deployment.Spec.Replicas == nil {
129+
return 0, errors.ErrorUnexpected("k8s deployment doesn't have the replicas field set", apiName)
130+
}
131+
132+
return *deployment.Spec.Replicas, nil
130133
}

‎pkg/autoscaler/autoscaler.go

+19-19
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ type Scaler interface {
4040
Scale(apiName string, request int32) error
4141
GetInFlightRequests(apiName string, window time.Duration) (*float64, error)
4242
GetAutoscalingSpec(apiName string) (*userconfig.Autoscaling, error)
43-
CurrentReplicas(apiName string) (int32, error)
43+
CurrentRequestedReplicas(apiName string) (int32, error)
4444
}
4545

4646
type Autoscaler struct {
@@ -76,12 +76,12 @@ func (a *Autoscaler) Awaken(api userconfig.Resource) error {
7676
zap.String("apiKind", api.Kind.String()),
7777
)
7878

79-
currentReplicas, err := scaler.CurrentReplicas(api.Name)
79+
currentRequestedReplicas, err := scaler.CurrentRequestedReplicas(api.Name)
8080
if err != nil {
8181
return errors.Wrap(err, "failed to get current replicas")
8282
}
8383

84-
if currentReplicas > 0 {
84+
if currentRequestedReplicas > 0 {
8585
return nil
8686
}
8787

@@ -166,7 +166,7 @@ func (a *Autoscaler) autoscaleFn(api userconfig.Resource) (func() error, error)
166166
return errors.Wrap(err, "failed to get autoscaling spec")
167167
}
168168

169-
currentReplicas, err := scaler.CurrentReplicas(api.Name)
169+
currentRequestedReplicas, err := scaler.CurrentRequestedReplicas(api.Name)
170170
if err != nil {
171171
return errors.Wrap(err, "failed to get current replicas")
172172
}
@@ -187,22 +187,22 @@ func (a *Autoscaler) autoscaleFn(api userconfig.Resource) (func() error, error)
187187
rawRecommendation := *avgInFlight / *autoscalingSpec.TargetInFlight
188188
recommendation := int32(math.Ceil(rawRecommendation))
189189

190-
if rawRecommendation < float64(currentReplicas) && rawRecommendation > float64(currentReplicas)*(1-autoscalingSpec.DownscaleTolerance) {
191-
recommendation = currentReplicas
190+
if rawRecommendation < float64(currentRequestedReplicas) && rawRecommendation > float64(currentRequestedReplicas)*(1-autoscalingSpec.DownscaleTolerance) {
191+
recommendation = currentRequestedReplicas
192192
}
193193

194-
if rawRecommendation > float64(currentReplicas) && rawRecommendation < float64(currentReplicas)*(1+autoscalingSpec.UpscaleTolerance) {
195-
recommendation = currentReplicas
194+
if rawRecommendation > float64(currentRequestedReplicas) && rawRecommendation < float64(currentRequestedReplicas)*(1+autoscalingSpec.UpscaleTolerance) {
195+
recommendation = currentRequestedReplicas
196196
}
197197

198198
// always allow subtraction of 1
199-
downscaleFactorFloor := libmath.MinInt32(currentReplicas-1, int32(math.Ceil(float64(currentReplicas)*autoscalingSpec.MaxDownscaleFactor)))
199+
downscaleFactorFloor := libmath.MinInt32(currentRequestedReplicas-1, int32(math.Ceil(float64(currentRequestedReplicas)*autoscalingSpec.MaxDownscaleFactor)))
200200
if recommendation < downscaleFactorFloor {
201201
recommendation = downscaleFactorFloor
202202
}
203203

204204
// always allow addition of 1
205-
upscaleFactorCeil := libmath.MaxInt32(currentReplicas+1, int32(math.Ceil(float64(currentReplicas)*autoscalingSpec.MaxUpscaleFactor)))
205+
upscaleFactorCeil := libmath.MaxInt32(currentRequestedReplicas+1, int32(math.Ceil(float64(currentRequestedReplicas)*autoscalingSpec.MaxUpscaleFactor)))
206206
if recommendation > upscaleFactorCeil {
207207
recommendation = upscaleFactorCeil
208208
}
@@ -228,24 +228,24 @@ func (a *Autoscaler) autoscaleFn(api userconfig.Resource) (func() error, error)
228228
var downscaleStabilizationFloor *int32
229229
var upscaleStabilizationCeil *int32
230230

231-
if request < currentReplicas {
231+
if request < currentRequestedReplicas {
232232
downscaleStabilizationFloor = recs.maxSince(autoscalingSpec.DownscaleStabilizationPeriod)
233233
if downscaleStabilizationFloor != nil {
234-
downscaleStabilizationFloor = pointer.Int32(libmath.MinInt32(*downscaleStabilizationFloor, currentReplicas))
234+
downscaleStabilizationFloor = pointer.Int32(libmath.MinInt32(*downscaleStabilizationFloor, currentRequestedReplicas))
235235
}
236236
if time.Since(startTime) < autoscalingSpec.DownscaleStabilizationPeriod {
237-
request = currentReplicas
237+
request = currentRequestedReplicas
238238
} else if downscaleStabilizationFloor != nil && request < *downscaleStabilizationFloor {
239239
request = *downscaleStabilizationFloor
240240
}
241241
}
242-
if request > currentReplicas {
242+
if request > currentRequestedReplicas {
243243
upscaleStabilizationCeil = recs.minSince(autoscalingSpec.UpscaleStabilizationPeriod)
244244
if upscaleStabilizationCeil != nil {
245-
upscaleStabilizationCeil = pointer.Int32(libmath.MaxInt32(*upscaleStabilizationCeil, currentReplicas))
245+
upscaleStabilizationCeil = pointer.Int32(libmath.MaxInt32(*upscaleStabilizationCeil, currentRequestedReplicas))
246246
}
247247
if time.Since(startTime) < autoscalingSpec.UpscaleStabilizationPeriod {
248-
request = currentReplicas
248+
request = currentRequestedReplicas
249249
} else if upscaleStabilizationCeil != nil && request > *upscaleStabilizationCeil {
250250
request = *upscaleStabilizationCeil
251251
}
@@ -256,7 +256,7 @@ func (a *Autoscaler) autoscaleFn(api userconfig.Resource) (func() error, error)
256256
"avg_in_flight": *avgInFlight,
257257
"target_in_flight": *autoscalingSpec.TargetInFlight,
258258
"raw_recommendation": rawRecommendation,
259-
"current_replicas": currentReplicas,
259+
"current_replicas": currentRequestedReplicas,
260260
"downscale_tolerance": autoscalingSpec.DownscaleTolerance,
261261
"upscale_tolerance": autoscalingSpec.UpscaleTolerance,
262262
"max_downscale_factor": autoscalingSpec.MaxDownscaleFactor,
@@ -274,8 +274,8 @@ func (a *Autoscaler) autoscaleFn(api userconfig.Resource) (func() error, error)
274274
},
275275
)
276276

277-
if currentReplicas != request {
278-
log.Infof("autoscaling event: %d -> %d", currentReplicas, request)
277+
if currentRequestedReplicas != request {
278+
log.Infof("autoscaling event: %d -> %d", currentRequestedReplicas, request)
279279
if err = scaler.Scale(api.Name, request); err != nil {
280280
return err
281281
}

‎pkg/autoscaler/autoscaler_test.go

+4-4
Original file line numberDiff line numberDiff line change
@@ -208,7 +208,7 @@ func TestAutoscaler_AutoscaleFn(t *testing.T) {
208208
GetAutoscalingSpecFunc: func(apiName string) (*userconfig.Autoscaling, error) {
209209
return &cases[i].autoscalingSpec, nil
210210
},
211-
CurrentReplicasFunc: func(apiName string) (int32, error) {
211+
CurrentRequestedReplicasFunc: func(apiName string) (int32, error) {
212212
return tt.currentReplicas, nil
213213
},
214214
}
@@ -273,7 +273,7 @@ func TestAutoscaler_Awake(t *testing.T) {
273273
MaxUpscaleFactor: 1.5,
274274
}, nil
275275
},
276-
CurrentReplicasFunc: func(apiName string) (int32, error) {
276+
CurrentRequestedReplicasFunc: func(apiName string) (int32, error) {
277277
return 0, nil
278278
},
279279
}
@@ -348,7 +348,7 @@ func TestAutoscaler_MinReplicas(t *testing.T) {
348348
MaxUpscaleFactor: 1.5,
349349
}, nil
350350
},
351-
CurrentReplicasFunc: func(apiName string) (int32, error) {
351+
CurrentRequestedReplicasFunc: func(apiName string) (int32, error) {
352352
return minReplicas + 1, nil
353353
},
354354
}
@@ -420,7 +420,7 @@ func TestAutoscaler_MaxReplicas(t *testing.T) {
420420
MaxUpscaleFactor: 1.5,
421421
}, nil
422422
},
423-
CurrentReplicasFunc: func(apiName string) (int32, error) {
423+
CurrentRequestedReplicasFunc: func(apiName string) (int32, error) {
424424
return minReplicas, nil
425425
},
426426
}

‎pkg/autoscaler/realtime_scaler.go

+10-3
Original file line numberDiff line numberDiff line change
@@ -63,8 +63,11 @@ func (s *RealtimeScaler) Scale(apiName string, request int32) error {
6363
return errors.Wrap(err, "failed to get deployment")
6464
}
6565

66-
current := deployment.Status.Replicas
66+
if deployment.Spec.Replicas == nil {
67+
return errors.Wrap(err, "k8s deployment doesn't have the replicas field set")
68+
}
6769

70+
current := *deployment.Spec.Replicas
6871
if current == request {
6972
return nil
7073
}
@@ -150,7 +153,7 @@ func (s *RealtimeScaler) GetAutoscalingSpec(apiName string) (*userconfig.Autosca
150153
return autoscalingSpec, nil
151154
}
152155

153-
func (s *RealtimeScaler) CurrentReplicas(apiName string) (int32, error) {
156+
func (s *RealtimeScaler) CurrentRequestedReplicas(apiName string) (int32, error) {
154157
ctx := context.Background()
155158

156159
// we use the controller-runtime client to make use of the cache mechanism
@@ -163,7 +166,11 @@ func (s *RealtimeScaler) CurrentReplicas(apiName string) (int32, error) {
163166
return 0, errors.Wrap(err, "failed to get deployment")
164167
}
165168

166-
return deployment.Status.Replicas, nil
169+
if deployment.Spec.Replicas == nil {
170+
return 0, errors.Wrap(err, "k8s deployment doesn't have the replicas field set")
171+
}
172+
173+
return *deployment.Spec.Replicas, nil
167174
}
168175

169176
func (s *RealtimeScaler) routeToService(deployment *kapps.Deployment) error {

‎pkg/autoscaler/scaler_func.go

+7-7
Original file line numberDiff line numberDiff line change
@@ -23,10 +23,10 @@ import (
2323
)
2424

2525
type ScalerFunc struct {
26-
ScaleFunc func(apiName string, request int32) error
27-
GetInFlightRequestsFunc func(apiName string, window time.Duration) (*float64, error)
28-
GetAutoscalingSpecFunc func(apiName string) (*userconfig.Autoscaling, error)
29-
CurrentReplicasFunc func(apiName string) (int32, error)
26+
ScaleFunc func(apiName string, request int32) error
27+
GetInFlightRequestsFunc func(apiName string, window time.Duration) (*float64, error)
28+
GetAutoscalingSpecFunc func(apiName string) (*userconfig.Autoscaling, error)
29+
CurrentRequestedReplicasFunc func(apiName string) (int32, error)
3030
}
3131

3232
func (s *ScalerFunc) Scale(apiName string, request int32) error {
@@ -53,10 +53,10 @@ func (s *ScalerFunc) GetAutoscalingSpec(apiName string) (*userconfig.Autoscaling
5353
return s.GetAutoscalingSpecFunc(apiName)
5454
}
5555

56-
func (s *ScalerFunc) CurrentReplicas(apiName string) (int32, error) {
57-
if s.CurrentReplicasFunc == nil {
56+
func (s *ScalerFunc) CurrentRequestedReplicas(apiName string) (int32, error) {
57+
if s.CurrentRequestedReplicasFunc == nil {
5858
return 0, nil
5959
}
6060

61-
return s.CurrentReplicasFunc(apiName)
61+
return s.CurrentRequestedReplicasFunc(apiName)
6262
}

0 commit comments

Comments
 (0)