Skip to content

Commit

Permalink
Major refactor + introduction of duration policies (#12)
Browse files Browse the repository at this point in the history
* Added duraton policies + refactor of POD handling
* Abstracted StartupCPUBoost
* Added tests to POD handler
* Introduced documentation workflow
mikouaj authored Nov 15, 2023
1 parent 962245f commit 42508d4
Showing 41 changed files with 2,728 additions and 618 deletions.
46 changes: 46 additions & 0 deletions .github/workflows/documentation.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
# Copyright 2023 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: Documentation checks
on:
push:
branches:
- main
pull_request:
branches:
- main

jobs:
linting:
name: Documentation linting
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Setup Python
uses: actions/setup-python@v4
with:
python-version: 3.9
- name: Check license headers
run: python3 scripts/license_header_check.py $GITHUB_WORKSPACE
- name: Markdown linter
uses: avto-dev/markdown-lint@v1
with:
config: '.markdownlint.yml'
args: '**/*.md'
- name: Markdown links
uses: gaurav-nelson/github-action-markdown-link-check@v1
with:
config-file: '.mlc_config.json'
base-branch: main
17 changes: 17 additions & 0 deletions .markdownlint.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
# Copyright 2023 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

MD013:
line_length: 100
tables: false
10 changes: 10 additions & 0 deletions .mlc_config.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
{
"ignorePatterns": [
{
"pattern": "^http://example.net"
},
{
"pattern": "^https://docs.github.com"
}
]
}
9 changes: 7 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
@@ -83,12 +83,17 @@ gcloud container clusters create poc \
operator: In
values: ["app-001", "app-002"]
spec:
timePeriod: 60
boostPercent: 50
durationPolicy:
podCondition:
name: Ready
value: "True"
```

The above example will boost CPU requests and limits of all PODs with `app=app-001` and `app=app-002`
labels in `demo` namespace. The resources will be increased by 50% for 60 seconds.
labels in `demo` namespace. The resources will be increased by 50% until the
[POD Condition](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-conditions)
`Ready` becomes `True`.

2. Schedule your workloads and observe the results

51 changes: 47 additions & 4 deletions api/v1alpha1/startupcpuboost_types.go
Original file line number Diff line number Diff line change
@@ -15,17 +15,60 @@
package v1alpha1

import (
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

// FixedDurationPolicyUnit defines the unit of time for a fixed
// time duration policy
// +kubebuilder:validation:Enum=Seconds;Minutes
type FixedDurationPolicyUnit string

const (
FixedDurationPolicyUnitSec FixedDurationPolicyUnit = "Seconds"
FixedDurationPolicyUnitMin FixedDurationPolicyUnit = "Minutes"
)

// FixedDurationPolicy defines the fixed time duration policy
type FixedDurationPolicy struct {
// unit of time for a fixed time policy
// +kubebuilder:validation:Required
Unit FixedDurationPolicyUnit `json:"unit,omitempty"`
// duration value for a fixed time policy
// +kubebuilder:validation:Required
// +kubebuilder:validation:Minimum:=1
Value int64 `json:"value,omitempty"`
}

// PodConditionDurationPolicy defines the PodCondition based
// duration policy
type PodConditionDurationPolicy struct {
// type of a PODCondition to check in a policy
Type corev1.PodConditionType `json:"type,omitempty"`
// status of a PODCondition to match in a policy
Status corev1.ConditionStatus `json:"status,omitempty"`
}

// DurationPolicy defines the policy used to determine the duration
// time of a resource boost
type DurationPolicy struct {
// fixed time duration policy
// +kubebuilder:validation:Optional
Fixed *FixedDurationPolicy `json:"fixedDuration,omitempty"`
// podCondition based duration policy
// +kubebuilder:validation:Optional
PodCondition *PodConditionDurationPolicy `json:"podCondition,omitempty"`
}

// StartupCPUBoostSpec defines the desired state of StartupCPUBoost
type StartupCPUBoostSpec struct {
// TimePeriod defines the period of time, in seconds, that POD will be affected
// by the CPU Boost after the initialization
TimePeriod int64 `json:"timePeriod,omitempty"`

// DurationPolicy specifies policies for resource boost duration
// +kubebuilder:validation:Required
DurationPolicy DurationPolicy `json:"durationPolicy,omitempty"`
// BootPercent defines the percent of CPU request increase that POD will get
// during the CPU boost time period
// +kubebuilder:validation:Required
// +kubebuilder:validation:Minimum:=1
BoostPercent int64 `json:"boostPercent,omitempty"`
}

58 changes: 57 additions & 1 deletion api/v1alpha1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

36 changes: 31 additions & 5 deletions config/crd/bases/autoscaling.x-k8s.io_startupcpuboosts.yaml
Original file line number Diff line number Diff line change
@@ -82,12 +82,38 @@ spec:
description: BootPercent defines the percent of CPU request increase
that POD will get during the CPU boost time period
format: int64
minimum: 1
type: integer
timePeriod:
description: TimePeriod defines the period of time, in seconds, that
POD will be affected by the CPU Boost after the initialization
format: int64
type: integer
durationPolicy:
description: DurationPolicy specifies policies for resource boost
duration
properties:
fixedDuration:
description: fixed time duration policy
properties:
unit:
description: unit of time for a fixed time policy
enum:
- Seconds
- Minutes
type: string
value:
description: duration value for a fixed time policy
format: int64
minimum: 1
type: integer
type: object
podCondition:
description: podCondition based duration policy
properties:
status:
description: status of a PODCondition to match in a policy
type: string
type:
description: type of a PODCondition to check in a policy
type: string
type: object
type: object
type: object
status:
description: StartupCPUBoostStatus defines the observed state of StartupCPUBoost
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
@@ -20,7 +20,6 @@ require (
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/emicklei/go-restful/v3 v3.9.0 // indirect
github.com/evanphx/json-patch v4.12.0+incompatible // indirect
github.com/evanphx/json-patch/v5 v5.6.0 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/go-logr/zapr v1.2.4 // indirect
@@ -51,6 +50,7 @@ require (
github.com/prometheus/procfs v0.9.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
go.uber.org/atomic v1.11.0 // indirect
go.uber.org/mock v0.3.0
go.uber.org/multierr v1.6.0 // indirect
go.uber.org/zap v1.24.0 // indirect
golang.org/x/net v0.17.0 // indirect
3 changes: 2 additions & 1 deletion go.sum
Original file line number Diff line number Diff line change
@@ -21,7 +21,6 @@ github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84=
github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww=
github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4=
github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
@@ -147,6 +146,8 @@ go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A=
go.uber.org/mock v0.3.0 h1:3mUxI1No2/60yUYax92Pt8eNOEecx2D3lcXZh2NEZJo=
go.uber.org/mock v0.3.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc=
go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60=
16 changes: 15 additions & 1 deletion hack/kind-poc-cluster.yaml
Original file line number Diff line number Diff line change
@@ -1,3 +1,17 @@
# Copyright 2023 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
name: poc
@@ -6,4 +20,4 @@ nodes:
- role: worker
- role: worker
featureGates:
InPlacePodVerticalScaling: true
InPlacePodVerticalScaling: true
104 changes: 104 additions & 0 deletions internal/boost/boost_suite_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
// Copyright 2023 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package boost_test

import (
"testing"
"time"

autoscaling "github.com/google/kube-startup-cpu-boost/api/v1alpha1"
bpod "github.com/google/kube-startup-cpu-boost/internal/boost/pod"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
corev1 "k8s.io/api/core/v1"
apiResource "k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

func TestBoost(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Boost Suite")
}

var (
podTemplate *corev1.Pod
annotTemplate *bpod.BoostPodAnnotation
specTemplate *autoscaling.StartupCPUBoost
)

var _ = BeforeSuite(func() {
specTemplate = &autoscaling.StartupCPUBoost{
ObjectMeta: metav1.ObjectMeta{
Name: "boost-001",
Namespace: "demo",
},
Spec: autoscaling.StartupCPUBoostSpec{
BoostPercent: 55,
},
}
annotTemplate = &bpod.BoostPodAnnotation{
BoostTimestamp: time.Now(),
InitCPURequests: map[string]string{
"container-one": "500m",
"continer-two": "500m",
},
InitCPULimits: map[string]string{
"container-one": "1",
"continer-two": "1",
},
}
reqQuantity, err := apiResource.ParseQuantity("1")
Expect(err).ShouldNot(HaveOccurred())
limitQuantity, err := apiResource.ParseQuantity("2")
Expect(err).ShouldNot(HaveOccurred())
podTemplate = &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "test-pod",
Namespace: specTemplate.Namespace,
Labels: map[string]string{
bpod.BoostLabelKey: specTemplate.Name,
},
Annotations: map[string]string{
bpod.BoostAnnotationKey: annotTemplate.ToJSON(),
},
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "container-one",
Resources: corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceCPU: reqQuantity,
},
Limits: corev1.ResourceList{
corev1.ResourceCPU: limitQuantity,
},
},
},
{
Name: "container-two",
Resources: corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceCPU: reqQuantity,
},
Limits: corev1.ResourceList{
corev1.ResourceCPU: limitQuantity,
},
},
},
},
},
}
})
16 changes: 16 additions & 0 deletions internal/boost/doc.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
// Copyright 2023 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package boost contains logic for managing startup resource boosts
package boost
197 changes: 96 additions & 101 deletions internal/boost/manager.go
Original file line number Diff line number Diff line change
@@ -12,7 +12,6 @@
// See the License for the specific language governing permissions and
// limitations under the License.

// Package boost contains logic for managing startup resource boosts
package boost

import (
@@ -24,17 +23,13 @@ import (
corev1 "k8s.io/api/core/v1"
"sigs.k8s.io/controller-runtime/pkg/client"

autoscaling "github.com/google/kube-startup-cpu-boost/api/v1alpha1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/types"
"github.com/go-logr/logr"
"github.com/google/kube-startup-cpu-boost/internal/boost/policy"
ctrl "sigs.k8s.io/controller-runtime"

apiResource "k8s.io/apimachinery/pkg/api/resource"
)

var (
errStartupCPUBoostAlreadyExists = errors.New("startupCPUBoost already exists")
errInvalidStartupCPUBoostSpec = errors.New("invalid startupCPUBoost spec")
)

const (
@@ -44,165 +39,165 @@ const (
)

type Manager interface {
AddStartupCPUBoost(ctx context.Context, boost *autoscaling.StartupCPUBoost) error
DeleteStartupCPUBoost(boost *autoscaling.StartupCPUBoost)
GetStartupCPUBoostForPod(pod *corev1.Pod) (*startupCPUBoost, bool)
GetStartupCPUBoost(namespace string, name string) (*startupCPUBoost, bool)
AddStartupCPUBoost(ctx context.Context, boost StartupCPUBoost) error
RemoveStartupCPUBoost(ctx context.Context, namespace, name string)
StartupCPUBoostForPod(ctx context.Context, pod *corev1.Pod) (StartupCPUBoost, bool)
StartupCPUBoost(namespace, name string) (StartupCPUBoost, bool)
Start(ctx context.Context) error
}

type TimeTicker interface {
Tick() <-chan time.Time
Stop()
}

type timeTickerImpl struct {
t time.Ticker
}

func (t *timeTickerImpl) Tick() <-chan time.Time {
return t.t.C
}

func (t *timeTickerImpl) Stop() {
t.t.Stop()
}

func newTimeTickerImpl(d time.Duration) TimeTicker {
return &timeTickerImpl{
t: *time.NewTicker(d),
}
}

type managerImpl struct {
sync.RWMutex
client client.Client
startupCPUBoosts map[string]map[string]*startupCPUBoost
ticker TimeTicker
checkInterval time.Duration
startupCPUBoosts map[string]map[string]StartupCPUBoost
timePolicyBoosts map[boostKey]StartupCPUBoost
}

type boostKey struct {
name string
namespace string
}

func NewManager(client client.Client) Manager {
return NewManagerWithTicker(client, newTimeTickerImpl(DefaultManagerCheckInterval))
}

func NewManagerWithTicker(client client.Client, ticker TimeTicker) Manager {
return &managerImpl{
client: client,
startupCPUBoosts: make(map[string]map[string]*startupCPUBoost),
ticker: ticker,
checkInterval: DefaultManagerCheckInterval,
startupCPUBoosts: make(map[string]map[string]StartupCPUBoost),
timePolicyBoosts: make(map[boostKey]StartupCPUBoost),
}
}

func (m *managerImpl) AddStartupCPUBoost(ctx context.Context, boost *autoscaling.StartupCPUBoost) error {
func (m *managerImpl) AddStartupCPUBoost(ctx context.Context, boost StartupCPUBoost) error {
m.Lock()
defer m.Unlock()
if _, ok := m.getStartupCPUBoostWithNS(boost.Namespace, boost.Name); ok {
if _, ok := m.getStartupCPUBoost(boost.Namespace(), boost.Name()); ok {
return errStartupCPUBoostAlreadyExists
}
boostImpl, err := newStartupCPUBoost(boost)
if err != nil {
return errInvalidStartupCPUBoostSpec
}
m.addStartupCPUBoostWithNS(boostImpl)
log := m.loggerFromContext(ctx).WithValues("boost", boost.Name, "namespace", boost.Namespace)
log.V(5).Info("handling startup-cpu-boost create")
m.addStartupCPUBoost(boost)
return nil
}

func (m *managerImpl) DeleteStartupCPUBoost(boost *autoscaling.StartupCPUBoost) {
func (m *managerImpl) RemoveStartupCPUBoost(ctx context.Context, namespace, name string) {
m.Lock()
defer m.Unlock()
if _, ok := m.getStartupCPUBoostWithNS(boost.Namespace, boost.Name); !ok {
return
log := m.loggerFromContext(ctx).WithValues("boost", name, "namespace", namespace)
log.V(5).Info("handling startup-cpu-boost delete")
if boosts, ok := m.startupCPUBoosts[namespace]; ok {
delete(boosts, name)
}
m.deleteStartupCPUBoostWithNS(boost.Namespace, boost.Name)
key := boostKey{name: name, namespace: namespace}
delete(m.timePolicyBoosts, key)
}

func (m *managerImpl) StartupCPUBoost(namespace string, name string) (StartupCPUBoost, bool) {
m.RLock()
defer m.RUnlock()
return m.getStartupCPUBoost(namespace, name)
}

func (m *managerImpl) GetStartupCPUBoostForPod(pod *corev1.Pod) (*startupCPUBoost, bool) {
func (m *managerImpl) StartupCPUBoostForPod(ctx context.Context, pod *corev1.Pod) (StartupCPUBoost, bool) {
m.RLock()
defer m.RUnlock()
log := m.loggerFromContext(ctx).WithValues("pod", pod.Name, "namespace", pod.Namespace)
log.V(5).Info("handling startup-cpu-boost pod lookup")
nsBoosts, ok := m.startupCPUBoosts[pod.Namespace]
if !ok {
return nil, false
}
for _, boost := range nsBoosts {
if boost.selector.Matches(labels.Set(pod.Labels)) {
if boost.Matches(pod) {
return boost, true
}
}
return nil, false
}

func (m *managerImpl) GetStartupCPUBoost(namespace string, name string) (*startupCPUBoost, bool) {
m.RLock()
defer m.RUnlock()
return m.getStartupCPUBoostWithNS(namespace, name)
}

func (m *managerImpl) Start(ctx context.Context) error {
log := ctrl.LoggerFrom(ctx).WithName("boost-manager")
t := time.NewTicker(m.checkInterval)
defer t.Stop()
log.V(2).Info("Boost manager starting")
log := m.loggerFromContext(ctx)
//t := time.NewTicker(m.checkInterval)
//defer t.Stop()
defer m.ticker.Stop()
log.V(2).Info("Starting")
for {
select {
case <-t.C:
case <-m.ticker.Tick():
log.V(5).Info("tick...")
m.updateStartupCPUBoostPods(ctx)
m.updateTimePolicyBoosts(ctx)
case <-ctx.Done():
return nil
}
}
}

func (m *managerImpl) addStartupCPUBoostWithNS(boostImpl *startupCPUBoost) {
nsBoosts, ok := m.startupCPUBoosts[boostImpl.namespace]
func (m *managerImpl) addStartupCPUBoost(boost StartupCPUBoost) {
boosts, ok := m.startupCPUBoosts[boost.Namespace()]
if !ok {
nsBoosts = make(map[string]*startupCPUBoost)
m.startupCPUBoosts[boostImpl.namespace] = nsBoosts
boosts = make(map[string]StartupCPUBoost)
m.startupCPUBoosts[boost.Namespace()] = boosts
}
boosts[boost.Name()] = boost
if _, ok := boost.DurationPolicies()[policy.FixedDurationPolicyName]; ok {
key := boostKey{name: boost.Name(), namespace: boost.Namespace()}
m.timePolicyBoosts[key] = boost
}
nsBoosts[boostImpl.name] = boostImpl
}

func (m *managerImpl) getStartupCPUBoostWithNS(ns string, name string) (*startupCPUBoost, bool) {
if nsboosts, ok := m.startupCPUBoosts[ns]; ok {
boost, ok := nsboosts[name]
func (m *managerImpl) getStartupCPUBoost(namespace string, name string) (StartupCPUBoost, bool) {
if boosts, ok := m.startupCPUBoosts[namespace]; ok {
boost, ok := boosts[name]
return boost, ok
}
return nil, false
}

func (m *managerImpl) deleteStartupCPUBoostWithNS(ns string, name string) {
if nsBoosts, ok := m.startupCPUBoosts[ns]; ok {
delete(nsBoosts, name)
}
}

func (m *managerImpl) getAllStartupCPUBoosts() []*startupCPUBoost {
result := make([]*startupCPUBoost, 0)
for _, nsMap := range m.startupCPUBoosts {
for _, boost := range nsMap {
result = append(result, boost)
}
}
return result
}

func (m *managerImpl) updateStartupCPUBoostPods(ctx context.Context) {
func (m *managerImpl) updateTimePolicyBoosts(ctx context.Context) {
m.RLock()
defer m.RUnlock()
now := time.Now()
for _, boost := range m.getAllStartupCPUBoosts() {
for _, pod := range boost.pods {
if pod.boostTimestamp.Add(boost.time).Before(now) {
log := ctrl.LoggerFrom(ctx).WithName("boost-manager").WithValues("pod.boostTimestamp", pod.boostTimestamp,
"boost.time", boost.time, "time.now", now, "pod", pod.name, "namespace", pod.namespace)
log.V(2).Info("Reverting startup CPU boost for pod")
if err := m.podCleanup(ctx, pod); err != nil {
log.Error(err, "unable to update pod")
}
delete(boost.pods, pod.name)
log := m.loggerFromContext(ctx)
for _, boost := range m.timePolicyBoosts {
for _, pod := range boost.ValidatePolicy(ctx, policy.FixedDurationPolicyName) {
log = log.WithValues("boost", boost.Name(), "namespace", boost.Namespace(), "pod", pod.Name)
log.V(5).Info("updating pod with initial resources")
if err := boost.RevertResources(ctx, pod); err != nil {
log.Error(err, "failed to revert resources for pod")
}
}
}
}

func (m *managerImpl) podCleanup(ctx context.Context, pod *startupCPUBoostPod) error {
podObj := &corev1.Pod{}
if err := m.client.Get(ctx, types.NamespacedName{Namespace: pod.namespace, Name: pod.name}, podObj); err != nil {
return err
}
for _, container := range podObj.Spec.Containers {
if request, ok := pod.initCPURequests[container.Name]; ok {
if reqQuantity, err := apiResource.ParseQuantity(request); err == nil {
container.Resources.Requests[corev1.ResourceCPU] = reqQuantity
} else {
return errors.New("unparsable init CPU request: " + err.Error())
}
}
if limit, ok := pod.initCPULimits[container.Name]; ok {
if limitQuantity, err := apiResource.ParseQuantity(limit); err == nil {
container.Resources.Limits[corev1.ResourceCPU] = limitQuantity
} else {
return errors.New("unparsable init CPU limit: " + err.Error())
}
}
}
delete(podObj.Labels, StartupCPUBoostPodLabelKey)
delete(podObj.Annotations, StartupCPUBoostPodAnnotationKey)
if err := m.client.Update(ctx, podObj); err != nil {
return err
}
return nil
func (m *managerImpl) loggerFromContext(ctx context.Context) logr.Logger {
return ctrl.LoggerFrom(ctx).
WithName("boost-manager")
}
293 changes: 212 additions & 81 deletions internal/boost/manager_test.go
Original file line number Diff line number Diff line change
@@ -12,94 +12,225 @@
// See the License for the specific language governing permissions and
// limitations under the License.

package boost
package boost_test

import (
"context"
"testing"
"time"

autoscaling "github.com/google/kube-startup-cpu-boost/api/v1alpha1"
cpuboost "github.com/google/kube-startup-cpu-boost/internal/boost"
"github.com/google/kube-startup-cpu-boost/internal/mock"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"go.uber.org/mock/gomock"
corev1 "k8s.io/api/core/v1"
apiResource "k8s.io/apimachinery/pkg/api/resource"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

func TestPodCleanup(t *testing.T) {
containerOneInitCPUReq := "100m"
containerOneInitCPULimit := "1000m"
containerTwoInitCPUReq := "200m"
containerTwoInitCPULimit := "2000m"
client := fake.NewClientBuilder().
WithInterceptorFuncs(interceptor.Funcs{
Get: func(ctx context.Context, client client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
podObj, ok := obj.(*corev1.Pod)
if !ok {
t.Fatalf("client get received non pod object")
}
containerOne := corev1.Container{
Name: "container-001",
Resources: corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceCPU: mustParseQuantity("800m"),
},
Limits: corev1.ResourceList{
corev1.ResourceCPU: mustParseQuantity("1800m"),
},
},
}
containerTwo := corev1.Container{
Name: "container-002",
Resources: corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceCPU: mustParseQuantity("1000m"),
},
Limits: corev1.ResourceList{
corev1.ResourceCPU: mustParseQuantity("4000m"),
},
},
}
podObj.Spec.Containers = append(podObj.Spec.Containers, containerOne)
podObj.Spec.Containers = append(podObj.Spec.Containers, containerTwo)
return nil
},
Update: func(ctx context.Context, client client.WithWatch, obj client.Object, opts ...client.UpdateOption) error {
_, ok := obj.(*corev1.Pod)
if !ok {
t.Fatalf("client update received non pod object")
var _ = Describe("Manager", func() {
var manager cpuboost.Manager
Describe("Registers startup-cpu-boost", func() {
var (
spec *autoscaling.StartupCPUBoost
boost cpuboost.StartupCPUBoost
err error
)
BeforeEach(func() {
spec = specTemplate.DeepCopy()
})
JustBeforeEach(func() {
manager = cpuboost.NewManager(nil)
boost, err = cpuboost.NewStartupCPUBoost(nil, spec)
Expect(err).ToNot(HaveOccurred())
})
When("startup-cpu-boost exists", func() {
JustBeforeEach(func() {
err = manager.AddStartupCPUBoost(context.TODO(), boost)
Expect(err).ToNot(HaveOccurred())
err = manager.AddStartupCPUBoost(context.TODO(), boost)
})
It("errors", func() {
Expect(err).To(HaveOccurred())
})
})
When("startup-cpu-boost does not exist", func() {
JustBeforeEach(func() {
err = manager.AddStartupCPUBoost(context.TODO(), boost)
})
It("does not error", func() {
Expect(err).NotTo(HaveOccurred())
})
It("stores the startup-cpu-boost", func() {
stored, ok := manager.StartupCPUBoost(spec.Namespace, spec.Name)
Expect(ok).To(BeTrue())
Expect(stored.Name()).To(Equal(spec.Name))
Expect(stored.Namespace()).To(Equal(spec.Namespace))
})
})

})
Describe("De-registers startup-cpu-boost", func() {
var (
spec *autoscaling.StartupCPUBoost
boost cpuboost.StartupCPUBoost
err error
)
BeforeEach(func() {
spec = specTemplate.DeepCopy()
})
JustBeforeEach(func() {
manager = cpuboost.NewManager(nil)
boost, err = cpuboost.NewStartupCPUBoost(nil, spec)
Expect(err).ToNot(HaveOccurred())
})
When("startup-cpu-boost exists", func() {
JustBeforeEach(func() {
err = manager.AddStartupCPUBoost(context.TODO(), boost)
Expect(err).ToNot(HaveOccurred())
manager.RemoveStartupCPUBoost(context.TODO(), boost.Namespace(), boost.Name())
})
It("removes the startup-cpu-boost", func() {
_, ok := manager.StartupCPUBoost(spec.Namespace, spec.Name)
Expect(ok).To(BeFalse())
})
})
})
Describe("retrieves startup-cpu-boost for a POD", func() {
var (
pod *corev1.Pod
podNameLabel string
podNameLabelValue string
boost cpuboost.StartupCPUBoost
found bool
)
BeforeEach(func() {
podNameLabel = "app.kubernetes.io/name"
podNameLabelValue = "app-001"
pod = podTemplate.DeepCopy()
pod.Labels[podNameLabel] = podNameLabelValue
})
JustBeforeEach(func() {
manager = cpuboost.NewManager(nil)
})
When("matching startup-cpu-boost does not exist", func() {
JustBeforeEach(func() {
boost, found = manager.StartupCPUBoostForPod(context.TODO(), pod)
})
It("returns false", func() {
Expect(found).To(BeFalse())
})
It("return nil", func() {
Expect(boost).To(BeNil())
})
})
When("matching startup-cpu-boost exists", func() {
var (
spec *autoscaling.StartupCPUBoost
err error
)
BeforeEach(func() {
spec = specTemplate.DeepCopy()
spec.Selector = *metav1.AddLabelToSelector(&metav1.LabelSelector{}, podNameLabel, podNameLabelValue)
})
JustBeforeEach(func() {
boost, err = cpuboost.NewStartupCPUBoost(nil, spec)
Expect(err).NotTo(HaveOccurred())
err = manager.AddStartupCPUBoost(context.TODO(), boost)
Expect(err).NotTo(HaveOccurred())
boost, found = manager.StartupCPUBoostForPod(context.TODO(), pod)
})
It("returns true", func() {
Expect(found).To(BeTrue())
})
It("returns valid boost", func() {
Expect(boost).NotTo(BeNil())
Expect(boost.Name()).To(Equal(spec.Name))
Expect(boost.Namespace()).To(Equal(spec.Namespace))
})
})
})
Describe("Runs on a time tick", func() {
var (
mockCtrl *gomock.Controller
mockTicker *mock.MockTimeTicker
ctx context.Context
cancel context.CancelFunc
err error
done chan int
)
BeforeEach(func() {
mockCtrl = gomock.NewController(GinkgoT())
mockTicker = mock.NewMockTimeTicker(mockCtrl)
ctx, cancel = context.WithCancel(context.TODO())
done = make(chan int)
})
JustBeforeEach(func() {
manager = cpuboost.NewManagerWithTicker(nil, mockTicker)
go func() {
defer GinkgoRecover()
err = manager.Start(ctx)
done <- 1
}()
})
When("There are no startup-cpu-boosts with fixed duration policy", func() {
var c chan time.Time
BeforeEach(func() {
c = make(chan time.Time, 1)
mockTicker.EXPECT().Tick().MinTimes(1).Return(c)
mockTicker.EXPECT().Stop().Return()
})
JustBeforeEach(func() {
c <- time.Now()
time.Sleep(500 * time.Millisecond)
cancel()
<-done
})
It("doesn't error", func() {
Expect(err).NotTo(HaveOccurred())
})
})
When("There are startup-cpu-boosts with fixed duration policy", func() {
var (
spec *autoscaling.StartupCPUBoost
boost cpuboost.StartupCPUBoost
pod *corev1.Pod
mockClient *mock.MockClient
c chan time.Time
)
BeforeEach(func() {
spec = specTemplate.DeepCopy()
var seconds int64 = 60
spec.Spec.DurationPolicy.Fixed = &autoscaling.FixedDurationPolicy{
Unit: autoscaling.FixedDurationPolicyUnitSec,
Value: seconds,
}
return nil
},
}).
Build()
manager := &managerImpl{
client: client,
}
pod := &startupCPUBoostPod{
name: "pod-001",
namespace: "demo",
boostName: "boost-001",
boostTimestamp: time.Now(),
initCPURequests: map[string]string{
"container-001": containerOneInitCPUReq,
"container-002": containerTwoInitCPUReq,
},
initCPULimits: map[string]string{
"container-001": containerOneInitCPULimit,
"container-002": containerTwoInitCPULimit,
},
}
err := manager.podCleanup(context.Background(), pod)
if err != nil {
t.Fatalf("err = %v; want nil", err)
}
}
pod = podTemplate.DeepCopy()
creationTimestamp := time.Now().Add(-1 * time.Duration(seconds) * time.Second).Add(-1 * time.Minute)
pod.CreationTimestamp = metav1.NewTime(creationTimestamp)
mockClient = mock.NewMockClient(mockCtrl)

c = make(chan time.Time, 1)
mockTicker.EXPECT().Tick().MinTimes(1).Return(c)
mockTicker.EXPECT().Stop().Return()
mockClient.EXPECT().Update(gomock.Any(), gomock.Eq(pod)).MinTimes(1).Return(nil)
})
JustBeforeEach(func() {
boost, err = cpuboost.NewStartupCPUBoost(mockClient, spec)
Expect(err).ShouldNot(HaveOccurred())
err = boost.UpsertPod(ctx, pod)
Expect(err).ShouldNot(HaveOccurred())
err = manager.AddStartupCPUBoost(context.TODO(), boost)
Expect(err).ShouldNot(HaveOccurred())

func mustParseQuantity(s string) apiResource.Quantity {
q, err := apiResource.ParseQuantity(s)
if err != nil {
panic("unparsable quantity: " + err.Error())
}
return q
}
c <- time.Now()
time.Sleep(500 * time.Millisecond)
cancel()
<-done
})
It("doesn't error", func() {
Expect(err).NotTo(HaveOccurred())
})
})
})
})
92 changes: 92 additions & 0 deletions internal/boost/pod/pod.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
// Copyright 2023 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package pod contains implementation of startup-cpu-boost POD manipulation
// functions
package pod

import (
"encoding/json"
"errors"
"fmt"
"time"

corev1 "k8s.io/api/core/v1"
apiResource "k8s.io/apimachinery/pkg/api/resource"
)

const (
BoostLabelKey = "autoscaling.x-k8s.io/startup-cpu-boost"
BoostAnnotationKey = "autoscaling.x-k8s.io/startup-cpu-boost"
)

type BoostPodAnnotation struct {
BoostTimestamp time.Time `json:"timestamp,omitempty"`
InitCPURequests map[string]string `json:"initCPURequests,omitempty"`
InitCPULimits map[string]string `json:"initCPULimits,omitempty"`
}

func NewBoostAnnotation() *BoostPodAnnotation {
return &BoostPodAnnotation{
BoostTimestamp: time.Now(),
InitCPURequests: make(map[string]string),
InitCPULimits: make(map[string]string),
}
}

func (a BoostPodAnnotation) ToJSON() string {
result, err := json.Marshal(a)
if err != nil {
panic("failed to marshall to JSON: " + err.Error())
}
return string(result)
}

func BoostAnnotationFromPod(pod *corev1.Pod) (*BoostPodAnnotation, error) {
annotation := &BoostPodAnnotation{}
data, ok := pod.Annotations[BoostAnnotationKey]
if !ok {
return nil, errors.New("boost annotation not found")
}
if err := json.Unmarshal([]byte(data), annotation); err != nil {
return nil, err
}
return annotation, nil
}

func RevertResourceBoost(pod *corev1.Pod) error {
annotation, err := BoostAnnotationFromPod(pod)
if err != nil {
return fmt.Errorf("failed to get boost annotation from pod: %s", err)
}
delete(pod.Labels, BoostLabelKey)
delete(pod.Annotations, BoostAnnotationKey)
for _, container := range pod.Spec.Containers {
if request, ok := annotation.InitCPURequests[container.Name]; ok {
if reqQuantity, err := apiResource.ParseQuantity(request); err == nil {
container.Resources.Requests[corev1.ResourceCPU] = reqQuantity
} else {
return fmt.Errorf("failed to parse CPU request: %s", err)
}
}
if limit, ok := annotation.InitCPULimits[container.Name]; ok {
if limitQuantity, err := apiResource.ParseQuantity(limit); err == nil {
container.Resources.Limits[corev1.ResourceCPU] = limitQuantity
} else {
return fmt.Errorf("failed to parse CPU limit: %s", err)
}
}
}
return nil
}
27 changes: 27 additions & 0 deletions internal/boost/pod/pod_suite_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
// Copyright 2023 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pod_test

import (
"testing"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)

func TestPod(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Pod Suite")
}
129 changes: 129 additions & 0 deletions internal/boost/pod/pod_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
// Copyright 2023 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pod_test

import (
"time"

bpod "github.com/google/kube-startup-cpu-boost/internal/boost/pod"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
corev1 "k8s.io/api/core/v1"
apiResource "k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

var _ = Describe("Pod", func() {
var pod *corev1.Pod
var annot *bpod.BoostPodAnnotation
var containerOne, containerTwo string
var reqQuantity, limitQuantity apiResource.Quantity
var err error

BeforeEach(func() {
containerOne = "container-one"
containerTwo = "container-one"
reqQuantity, err = apiResource.ParseQuantity("1")
Expect(err).ShouldNot(HaveOccurred())
limitQuantity, err = apiResource.ParseQuantity("2")
Expect(err).ShouldNot(HaveOccurred())
annot = &bpod.BoostPodAnnotation{
BoostTimestamp: time.Now(),
InitCPURequests: map[string]string{
containerOne: "500m",
containerTwo: "500m",
},
InitCPULimits: map[string]string{
containerOne: "1",
containerTwo: "1",
},
}
pod = &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "test-pod",
Labels: map[string]string{
bpod.BoostLabelKey: "boost-001",
},
Annotations: map[string]string{
bpod.BoostAnnotationKey: annot.ToJSON(),
},
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: containerOne,
Resources: corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceCPU: reqQuantity,
},
Limits: corev1.ResourceList{
corev1.ResourceCPU: limitQuantity,
},
},
},
{
Name: containerTwo,
Resources: corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceCPU: reqQuantity,
},
Limits: corev1.ResourceList{
corev1.ResourceCPU: limitQuantity,
},
},
},
},
},
}
})

Describe("Reverts the POD container resources to original values", func() {
When("POD is missing startup-cpu-boost annotation", func() {
BeforeEach(func() {
delete(pod.ObjectMeta.Annotations, bpod.BoostAnnotationKey)
err = bpod.RevertResourceBoost(pod)
})
It("errors", func() {
Expect(err).Should(HaveOccurred())
})
})
When("POD has valid startup-cpu-boost annotation", func() {
BeforeEach(func() {
err = bpod.RevertResourceBoost(pod)
})
It("does not error", func() {
Expect(err).ShouldNot(HaveOccurred())
})
It("removes startup-cpu-boost label", func() {
Expect(pod.Labels).NotTo(HaveKey(bpod.BoostLabelKey))
})
It("removes startup-cpu-boost annotation", func() {
Expect(pod.Annotations).NotTo(HaveKey(bpod.BoostAnnotationKey))
})
It("reverts CPU requests to initial values", func() {
cpuReqOne := pod.Spec.Containers[0].Resources.Requests[corev1.ResourceCPU]
cpuReqTwo := pod.Spec.Containers[1].Resources.Requests[corev1.ResourceCPU]
Expect(cpuReqOne.String()).Should(Equal(annot.InitCPURequests[containerOne]))
Expect(cpuReqTwo.String()).Should(Equal(annot.InitCPURequests[containerTwo]))
})
It("reverts CPU limits to initial values", func() {
cpuReqOne := pod.Spec.Containers[0].Resources.Limits[corev1.ResourceCPU]
cpuReqTwo := pod.Spec.Containers[1].Resources.Limits[corev1.ResourceCPU]
Expect(cpuReqOne.String()).Should(Equal(annot.InitCPULimits[containerOne]))
Expect(cpuReqTwo.String()).Should(Equal(annot.InitCPULimits[containerTwo]))
})
})
})
})
56 changes: 56 additions & 0 deletions internal/boost/policy/fixed.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
// Copyright 2023 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package policy

import (
"time"

v1 "k8s.io/api/core/v1"
)

const (
FixedDurationPolicyName = "FixedDuration"
)

type TimeFunc func() time.Time

type FixedDurationPolicy struct {
timeFunc TimeFunc
duration time.Duration
}

func NewFixedDurationPolicy(duration time.Duration) DurationPolicy {
return NewFixedDurationPolicyWithTimeFunc(time.Now, duration)
}

func NewFixedDurationPolicyWithTimeFunc(timeFunc TimeFunc, duration time.Duration) DurationPolicy {
return &FixedDurationPolicy{
timeFunc: timeFunc,
duration: duration,
}
}

func (*FixedDurationPolicy) Name() string {
return FixedDurationPolicyName
}

func (p *FixedDurationPolicy) Duration() time.Duration {
return p.duration
}

func (p *FixedDurationPolicy) Valid(pod *v1.Pod) bool {
now := p.timeFunc()
return pod.CreationTimestamp.Add(p.duration).After(now)
}
57 changes: 57 additions & 0 deletions internal/boost/policy/fixed_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
// Copyright 2023 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package policy_test

import (
"time"

bpolicy "github.com/google/kube-startup-cpu-boost/internal/boost/policy"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

var _ = Describe("FixedDurationPolicy", func() {
var policy bpolicy.DurationPolicy
var now time.Time
var duration time.Duration
var timeFunc bpolicy.TimeFunc

BeforeEach(func() {
now = time.Now()
duration = 5 * time.Second
timeFunc = func() time.Time {
return now
}
policy = bpolicy.NewFixedDurationPolicyWithTimeFunc(timeFunc, duration)
})

Describe("Validates POD", func() {
When("the life time of a POD exceeds the policy duration", func() {
It("returns policy is not valid", func() {
creationTimesamp := now.Add(-1 * duration).Add(-1 * time.Minute)
pod.CreationTimestamp = metav1.NewTime(creationTimesamp)
Expect(policy.Valid(pod)).To(BeFalse())
})
})
When("the life time of a POD is within policy duration", func() {
It("returns policy is valid", func() {
creationTimesamp := now.Add(-1 * duration).Add(1 * time.Minute)
pod.CreationTimestamp = metav1.NewTime(creationTimesamp)
Expect(policy.Valid(pod)).To(BeTrue())
})
})
})
})
59 changes: 59 additions & 0 deletions internal/boost/policy/podcondition.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
// Copyright 2023 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package policy

import (
corev1 "k8s.io/api/core/v1"
)

const (
PodConditionPolicyName = "PodCondition"
)

type PodConditionPolicy struct {
condition corev1.PodConditionType
status corev1.ConditionStatus
}

func NewPodConditionPolicy(condition corev1.PodConditionType, status corev1.ConditionStatus) DurationPolicy {
return &PodConditionPolicy{
condition: condition,
status: status,
}
}

func (*PodConditionPolicy) Name() string {
return PodConditionPolicyName
}

func (p *PodConditionPolicy) Condition() corev1.PodConditionType {
return p.condition
}

func (p *PodConditionPolicy) Status() corev1.ConditionStatus {
return p.status
}

func (p *PodConditionPolicy) Valid(pod *corev1.Pod) bool {
for _, condition := range pod.Status.Conditions {
if condition.Type != p.condition {
continue
}
if condition.Status == p.status {
return false
}
}
return true
}
66 changes: 66 additions & 0 deletions internal/boost/policy/podcondition_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
// Copyright 2023 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package policy_test

import (
bpolicy "github.com/google/kube-startup-cpu-boost/internal/boost/policy"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
corev1 "k8s.io/api/core/v1"
)

var _ = Describe("PodConditionPolicy", func() {
var policy bpolicy.DurationPolicy
var condition corev1.PodConditionType
var status corev1.ConditionStatus

BeforeEach(func() {
condition = corev1.PodReady
status = corev1.ConditionTrue
policy = bpolicy.NewPodConditionPolicy(condition, status)
})

Describe("Validates POD", func() {
Context("when the POD has no condition with matching type", func() {
BeforeEach(func() {
pod.Status.Conditions = make([]corev1.PodCondition, 0)
})
It("returns policy is valid", func() {
Expect(policy.Valid(pod)).To(BeTrue())
})
})
Context("when the POD has condition with matching type", func() {
BeforeEach(func() {
pod.Status.Conditions = []corev1.PodCondition{
{
Type: condition,
Status: corev1.ConditionUnknown,
},
}
})
When("condition status matches status in a policy", func() {
It("returns policy is invalid", func() {
pod.Status.Conditions[0].Status = status
Expect(policy.Valid(pod)).To(BeFalse())
})
})
When("condition status does not match status in a policy", func() {
It("returns policy is valid", func() {
Expect(policy.Valid(pod)).To(BeTrue())
})
})
})
})
})
28 changes: 28 additions & 0 deletions internal/boost/policy/policy.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
// Copyright 2023 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package policy contains implementation of resource boost duration policies
package policy

import corev1 "k8s.io/api/core/v1"

const (
PolicyTypeFixed = "Fixed"
PolicyTypePodCondition = "PodCondition"
)

type DurationPolicy interface {
Valid(pod *corev1.Pod) bool
Name() string
}
40 changes: 40 additions & 0 deletions internal/boost/policy/policy_suite_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
// Copyright 2023 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package policy_test

import (
"testing"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

var pod *corev1.Pod

func TestPolicy(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Policy Suite")
}

var _ = BeforeSuite(func() {
pod = &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{},
Status: corev1.PodStatus{
Conditions: []corev1.PodCondition{},
},
}
})
198 changes: 170 additions & 28 deletions internal/boost/startupcpuboost.go
Original file line number Diff line number Diff line change
@@ -15,63 +15,205 @@
package boost

import (
"errors"
"context"
"fmt"
"sync"
"time"

"github.com/go-logr/logr"
autoscaling "github.com/google/kube-startup-cpu-boost/api/v1alpha1"
bpod "github.com/google/kube-startup-cpu-boost/internal/boost/pod"
"github.com/google/kube-startup-cpu-boost/internal/boost/policy"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/labels"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

var (
errPodAlreadyExists = errors.New("pod already exists")
)
// StartupCPUBoost is an implementation of a StartupCPUBoost CRD
type StartupCPUBoost interface {
Name() string
Namespace() string
BoostPercent() int64
DurationPolicies() map[string]policy.DurationPolicy
Pod(name string) (*corev1.Pod, bool)
UpsertPod(ctx context.Context, pod *corev1.Pod) error
DeletePod(ctx context.Context, pod *corev1.Pod) error
ValidatePolicy(ctx context.Context, name string) []*corev1.Pod
RevertResources(ctx context.Context, pod *corev1.Pod) error
Matches(pod *corev1.Pod) bool
}

type startupCPUBoost struct {
// StartupCPUBoostImpl is an implementation of a StartupCPUBoost CRD
type StartupCPUBoostImpl struct {
sync.RWMutex
name string
namespace string
percent int64
time time.Duration
selector labels.Selector
pods map[string]*startupCPUBoostPod
policies map[string]policy.DurationPolicy
pods sync.Map
client client.Client
}

// NewStartupCPUBoost constructs startup-cpu-boost implementation from a given API spec
func NewStartupCPUBoost(client client.Client, boost *autoscaling.StartupCPUBoost) (StartupCPUBoost, error) {
selector, err := metav1.LabelSelectorAsSelector(&boost.Selector)
if err != nil {
return nil, err
}
return &StartupCPUBoostImpl{
name: boost.Name,
namespace: boost.Namespace,
selector: selector,
percent: boost.Spec.BoostPercent,
policies: policiesFromSpec(boost.Spec.DurationPolicy),
client: client,
}, nil
}

func (b *startupCPUBoost) Name() string {
// Name returns startup-cpu-boost name
func (b *StartupCPUBoostImpl) Name() string {
return b.name
}

func (b *startupCPUBoost) Namespace() string {
// Namespace returns startup-cpu-boost namespace
func (b *StartupCPUBoostImpl) Namespace() string {
return b.namespace
}

func (b *startupCPUBoost) BoostPercent() int64 {
// BoostPercent returns startup-cpu-boost boost percentage
func (b *StartupCPUBoostImpl) BoostPercent() int64 {
return b.percent
}

func (b *startupCPUBoost) AddPod(pod *startupCPUBoostPod) error {
b.Lock()
defer b.Unlock()
if _, ok := b.pods[pod.name]; ok {
return errPodAlreadyExists
// DurationPolicies returns configured duration policies
func (b *StartupCPUBoostImpl) DurationPolicies() map[string]policy.DurationPolicy {
return b.policies
}

// Pod returns a POD if tracked by startup-cpu-boost.
func (b *StartupCPUBoostImpl) Pod(name string) (*corev1.Pod, bool) {
if v, ok := b.pods.Load(name); ok {
return v.(*corev1.Pod), ok
}
return nil, false
}

// UpsertPod inserts new or updates existing POD to startup-cpu-boost tracking
// The update of existing POD triggers validation logic and may result in POD update
func (b *StartupCPUBoostImpl) UpsertPod(ctx context.Context, pod *corev1.Pod) error {
log := b.loggerFromContext(ctx).WithValues("pod", pod.Name)
log.V(5).Info("upserting a pod")
if _, loaded := b.pods.Swap(pod.Name, pod); !loaded {
log.V(5).Info("inserted non-existing pod")
return nil
}
log.V(5).Info("updating existing pod")
condPolicy, ok := b.policies[policy.PodConditionPolicyName]
if !ok {
log.V(5).Info("skipping pod update as podCondition policy is missing")
return nil
}
if valid := b.validatePolicyOnPod(ctx, condPolicy, pod); !valid {
log.V(5).Info("updating pod with initial resources")
if err := b.RevertResources(ctx, pod); err != nil {
return fmt.Errorf("failed to update pod: %s", err)
}
}
b.pods[pod.name] = pod
return nil
}

func newStartupCPUBoost(boost *autoscaling.StartupCPUBoost) (*startupCPUBoost, error) {
selector, err := metav1.LabelSelectorAsSelector(&boost.Selector)
if err != nil {
return nil, err
// DeletePod removes the POD from the startup-cpu-boost tracking
func (b *StartupCPUBoostImpl) DeletePod(ctx context.Context, pod *corev1.Pod) error {
log := b.loggerFromContext(ctx).WithValues("pod", pod.Name)
log.V(5).Info("handling pod delete")
if _, loaded := b.pods.LoadAndDelete(pod.Name); loaded {
log.Info("deletion of untracked pod")
}
return nil
}

// ValidatePolicy validates policy with a given name on all startup-cpu-boost PODs.
// The function returns slice of PODs that violated the policy.
func (b *StartupCPUBoostImpl) ValidatePolicy(ctx context.Context, name string) (violated []*corev1.Pod) {
violated = make([]*corev1.Pod, 0)
policy, ok := b.policies[name]
if !ok {
return
}
b.pods.Range(func(key, value any) bool {
pod := value.(*corev1.Pod)
if !b.validatePolicyOnPod(ctx, policy, pod) {
violated = append(violated, pod)
}
return true
})
return
}

// RevertResources updates POD's container resource requests and limits to their original
// values using the data from StartupCPUBoost annotation
func (b *StartupCPUBoostImpl) RevertResources(ctx context.Context, pod *corev1.Pod) error {
if err := bpod.RevertResourceBoost(pod); err != nil {
return fmt.Errorf("failed to update pod spec: %s", err)
}
if err := b.client.Update(ctx, pod); err != nil {
return err
}
b.pods.Delete(pod.Name)
return nil
}

// Matches verifies if a boost selector matches the given POD
func (b *StartupCPUBoostImpl) Matches(pod *corev1.Pod) bool {
return b.selector.Matches(labels.Set(pod.Labels))
}

// loggerFromContext provides Logger from a current context with configured
// values common for startup-cpu-boost like name or namespace
func (b *StartupCPUBoostImpl) loggerFromContext(ctx context.Context) logr.Logger {
return ctrl.LoggerFrom(ctx).
WithName("startup-cpu-boost").
WithValues(
"name", b.name,
"namespace", b.namespace,
)
}

// validatePolicyOnPod validates given policy on a given POD.
// The function returns true if policy is valid or false otherwise
func (b *StartupCPUBoostImpl) validatePolicyOnPod(ctx context.Context, p policy.DurationPolicy, pod *corev1.Pod) (valid bool) {
log := b.loggerFromContext(ctx).WithValues("pod", pod.Name)
if valid = p.Valid(pod); !valid {
log.WithValues("policy", p.Name()).V(5).Info("policy is not valid")
}
return
}

// policiesFromSpec maps the Duration Policies from the API spec to the map holding policy
// implementations under policy name keys
func policiesFromSpec(policiesSpec autoscaling.DurationPolicy) map[string]policy.DurationPolicy {
policies := make(map[string]policy.DurationPolicy)
if fixedPolicy := policiesSpec.Fixed; fixedPolicy != nil {
duration := fixedPolicyToDuration(*fixedPolicy)
policies[policy.FixedDurationPolicyName] = policy.NewFixedDurationPolicy(duration)
}
if condPolicy := policiesSpec.PodCondition; condPolicy != nil {
policies[policy.PodConditionPolicyName] = policy.NewPodConditionPolicy(condPolicy.Type, condPolicy.Status)
}
return policies
}

// fixedPolicyToDuration maps the attributes from FixedDurationPolicy API spec to the
// time duration
func fixedPolicyToDuration(policy autoscaling.FixedDurationPolicy) time.Duration {
switch policy.Unit {
case autoscaling.FixedDurationPolicyUnitMin:
return time.Duration(policy.Value) * time.Minute
default:
return time.Duration(policy.Value) * time.Second
}
return &startupCPUBoost{
name: boost.Name,
namespace: boost.Namespace,
selector: selector,
percent: boost.Spec.BoostPercent,
time: time.Duration(boost.Spec.TimePeriod) * time.Second,
pods: make(map[string]*startupCPUBoostPod),
}, nil
}
196 changes: 184 additions & 12 deletions internal/boost/startupcpuboost_test.go
Original file line number Diff line number Diff line change
@@ -12,17 +12,189 @@
// See the License for the specific language governing permissions and
// limitations under the License.

package boost
package boost_test

import "testing"
import (
"context"
"time"

func TestGetBoostPercent(t *testing.T) {
var perc int64 = 50
boost := startupCPUBoost{
percent: perc,
}
result := boost.BoostPercent()
if boost.BoostPercent() != perc {
t.Errorf("perc = %v; want %v", result, perc)
}
}
autoscaling "github.com/google/kube-startup-cpu-boost/api/v1alpha1"
cpuboost "github.com/google/kube-startup-cpu-boost/internal/boost"
"github.com/google/kube-startup-cpu-boost/internal/boost/policy"
"github.com/google/kube-startup-cpu-boost/internal/mock"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"go.uber.org/mock/gomock"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

var _ = Describe("StartupCPUBoost", func() {
var (
spec *autoscaling.StartupCPUBoost
boost cpuboost.StartupCPUBoost
err error
pod *corev1.Pod
)
BeforeEach(func() {
pod = podTemplate.DeepCopy()
spec = specTemplate.DeepCopy()
})
Describe("Instantiates from the API specification", func() {
JustBeforeEach(func() {
boost, err = cpuboost.NewStartupCPUBoost(nil, spec)
Expect(err).ShouldNot(HaveOccurred())
})
It("returns valid name", func() {
Expect(boost.Name()).To(Equal(spec.Name))
})
It("returns valid namespace", func() {
Expect(boost.Namespace()).To(Equal(spec.Namespace))
})
It("returns valid boost percent", func() {
Expect(boost.BoostPercent()).To(Equal(spec.Spec.BoostPercent))
})
When("the spec has fixed duration policy", func() {
BeforeEach(func() {
spec.Spec.DurationPolicy.Fixed = &autoscaling.FixedDurationPolicy{
Unit: autoscaling.FixedDurationPolicyUnitSec,
Value: 123,
}
})
It("returns fixed duration policy implementation", func() {
Expect(boost.DurationPolicies()).To(HaveKey(policy.FixedDurationPolicyName))
})
It("returned fixed duration policy implementation is valid", func() {
p := boost.DurationPolicies()[policy.FixedDurationPolicyName]
fixedP, ok := p.(*policy.FixedDurationPolicy)
Expect(ok).To(BeTrue())
expDuration := time.Duration(spec.Spec.DurationPolicy.Fixed.Value) * time.Second
Expect(fixedP.Duration()).To(Equal(expDuration))
})
})
When("the spec has pod condition duration policy", func() {
BeforeEach(func() {
spec.Spec.DurationPolicy.Fixed = &autoscaling.FixedDurationPolicy{
Unit: autoscaling.FixedDurationPolicyUnitSec,
Value: 123,
}
spec.Spec.DurationPolicy.PodCondition = &autoscaling.PodConditionDurationPolicy{
Type: corev1.PodReady,
Status: corev1.ConditionTrue,
}
})
It("returns pod condition duration policy implementation", func() {
Expect(boost.DurationPolicies()).To(HaveKey(policy.PodConditionPolicyName))
})
It("returned pod condition duration policy implementation is valid", func() {
p := boost.DurationPolicies()[policy.PodConditionPolicyName]
podCondP, ok := p.(*policy.PodConditionPolicy)
Expect(ok).To(BeTrue())
Expect(podCondP.Condition()).To(Equal(spec.Spec.DurationPolicy.PodCondition.Type))
Expect(podCondP.Status()).To(Equal(spec.Spec.DurationPolicy.PodCondition.Status))
})
})
})

Describe("Upserts a POD", func() {
var (
mockCtrl *gomock.Controller
mockClient *mock.MockClient
)
BeforeEach(func() {
mockCtrl = gomock.NewController(GinkgoT())
mockClient = mock.NewMockClient(mockCtrl)
})
JustBeforeEach(func() {
boost, err = cpuboost.NewStartupCPUBoost(mockClient, spec)
Expect(err).ShouldNot(HaveOccurred())
})
When("POD does not exist", func() {
JustBeforeEach(func() {
err = boost.UpsertPod(context.TODO(), pod)
})
It("doesn't error", func() {
Expect(err).ShouldNot(HaveOccurred())
})
It("stores a POD", func() {
p, ok := boost.Pod(pod.Name)
Expect(ok).To(BeTrue())
Expect(p.Name).To(Equal(pod.Name))
})
})
When("POD exists", func() {
var existingPod *corev1.Pod
var createTimestamp metav1.Time
BeforeEach(func() {
existingPod = podTemplate.DeepCopy()
createTimestamp = metav1.NewTime(time.Now())
pod.CreationTimestamp = createTimestamp
})
JustBeforeEach(func() {
err = boost.UpsertPod(context.TODO(), existingPod)
Expect(err).ShouldNot(HaveOccurred())
err = boost.UpsertPod(context.TODO(), pod)
})
It("doesn't error", func() {
Expect(err).ShouldNot(HaveOccurred())
})
It("stores an updated POD", func() {
p, found := boost.Pod(pod.Name)
Expect(found).To(BeTrue())
Expect(p.Name).To(Equal(pod.Name))
Expect(p.CreationTimestamp).To(Equal(createTimestamp))
})
When("boost spec has pod condition policy", func() {
BeforeEach(func() {
spec.Spec.DurationPolicy.PodCondition = &autoscaling.PodConditionDurationPolicy{
Type: corev1.PodReady,
Status: corev1.ConditionTrue,
}
})
When("POD condition matches spec policy", func() {
BeforeEach(func() {
pod.Status.Conditions = []corev1.PodCondition{{
Type: corev1.PodReady,
Status: corev1.ConditionTrue,
}}
mockClient.EXPECT().
Update(gomock.Any(), gomock.Eq(pod)).
Return(nil)
})
It("doesn't error", func() {
Expect(err).NotTo(HaveOccurred())
})
})
When("POD condition does not match spec policy", func() {
BeforeEach(func() {
pod.Status.Conditions = []corev1.PodCondition{{
Type: corev1.PodReady,
Status: corev1.ConditionFalse,
}}
})
It("doesn't error", func() {
Expect(err).NotTo(HaveOccurred())
})
})
})
})
})

Describe("Deletes a pod", func() {
JustBeforeEach(func() {
boost, err = cpuboost.NewStartupCPUBoost(nil, spec)
Expect(err).ShouldNot(HaveOccurred())
})
When("Pod exists", func() {
JustBeforeEach(func() {
err = boost.UpsertPod(context.TODO(), pod)
Expect(err).ShouldNot(HaveOccurred())
err = boost.DeletePod(context.TODO(), pod)
})
It("removes stored pod", func() {
_, found := boost.Pod(pod.Name)
Expect(found).To(BeFalse())
})
})
})
})
110 changes: 0 additions & 110 deletions internal/boost/startupcpuboostpod.go

This file was deleted.

127 changes: 0 additions & 127 deletions internal/boost/startupcpuboostpod_test.go

This file was deleted.

88 changes: 69 additions & 19 deletions internal/controller/boost_pod_handler.go
Original file line number Diff line number Diff line change
@@ -19,51 +19,93 @@ import (

"github.com/go-logr/logr"
"github.com/google/kube-startup-cpu-boost/internal/boost"
bpod "github.com/google/kube-startup-cpu-boost/internal/boost/pod"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/util/workqueue"
"sigs.k8s.io/controller-runtime/pkg/event"
)

type BoostPodHandler interface {
Create(context.Context, event.CreateEvent, workqueue.RateLimitingInterface)
Delete(context.Context, event.DeleteEvent, workqueue.RateLimitingInterface)
Update(context.Context, event.UpdateEvent, workqueue.RateLimitingInterface)
Generic(context.Context, event.GenericEvent, workqueue.RateLimitingInterface)
GetPodLabelSelector() *metav1.LabelSelector
}

type boostPodHandler struct {
manager boost.Manager
log logr.Logger
}

func (h *boostPodHandler) Create(ctx context.Context, e event.CreateEvent, q workqueue.RateLimitingInterface) {
func NewBoostPodHandler(manager boost.Manager, log logr.Logger) BoostPodHandler {
return &boostPodHandler{
manager: manager,
log: log,
}
}

func (h *boostPodHandler) Create(ctx context.Context, e event.CreateEvent, wq workqueue.RateLimitingInterface) {
pod, ok := e.Object.(*corev1.Pod)
if !ok {
h.log.V(2).Info("Pod create event contains non-pod object")
return
}
log := h.log.WithValues("pod.Name", pod.Name, "pod.Namespace", pod.Namespace)
log.V(5).Info("Pod create event")
boostPod, err := boost.NewStartupCPUBoostPod(pod)
if err != nil {
log = log.WithValues("error", err.Error())
log.V(2).Info("failed to parse startup cpu boost pod")
log := h.log.WithValues("pod", pod.Name, "namespace", pod.Namespace)
log.V(5).Info("handling pod create")
boost, ok := h.boostForPod(pod)
if !ok {
log.V(5).Info("failed to get boost for pod")
return
}
boost, ok := h.manager.GetStartupCPUBoost(boostPod.GetNamespace(), boostPod.GetBoostName())
log.WithValues("boost", boost.Name())
if err := boost.UpsertPod(ctx, pod); err != nil {
log.Error(err, "failed to handle pod create")
}
}

func (h *boostPodHandler) Delete(ctx context.Context, e event.DeleteEvent, wq workqueue.RateLimitingInterface) {
pod, ok := e.Object.(*corev1.Pod)
if !ok {
log.V(2).Info("failed to get startup cpu boost for a pod")
return
}
log = log.WithValues("boost.Name", boost.Name())
log.V(5).Info("Found boost in manager")
if err := boost.AddPod(boostPod); err != nil {
log.Error(err, "failed to add pod to startup cpu boost")
log := h.log.WithValues("pod", pod.Name, "namespace", pod.Namespace)
log.V(5).Info("handling pod delete")
boost, ok := h.boostForPod(pod)
if !ok {
log.V(5).Info("failed to get boost for pod")
return
}
if err := boost.DeletePod(ctx, pod); err != nil {
log.Error(err, "failed to handle pod delete")
}
}

func (h *boostPodHandler) Delete(context.Context, event.DeleteEvent, workqueue.RateLimitingInterface) {
}

func (h *boostPodHandler) Update(context.Context, event.UpdateEvent, workqueue.RateLimitingInterface) {
func (h *boostPodHandler) Update(ctx context.Context, e event.UpdateEvent, wq workqueue.RateLimitingInterface) {
pod, ok := e.ObjectNew.(*corev1.Pod)
if !ok {
return
}
log := h.log.WithValues("pod", pod.Name, "namespace", pod.Namespace)
log.V(5).Info("handling pod update")
//TODO react only on POD or container condition updates
boost, ok := h.boostForPod(pod)
if !ok {
log.V(5).Info("failed to get boost for pod")
return
}
if err := boost.UpsertPod(ctx, pod); err != nil {
log.Error(err, "failed to handle pod update")
}
}

func (h *boostPodHandler) Generic(context.Context, event.GenericEvent, workqueue.RateLimitingInterface) {
func (h *boostPodHandler) Generic(ctx context.Context, e event.GenericEvent, wq workqueue.RateLimitingInterface) {
pod, ok := e.Object.(*corev1.Pod)
if !ok {
return
}
log := h.log.WithValues("pod", pod.Name, "namespace", pod.Namespace)
log.V(5).Info("got pod generic event")
}

func (h *boostPodHandler) GetPodLabelSelector() *metav1.LabelSelector {
@@ -77,3 +119,11 @@ func (h *boostPodHandler) GetPodLabelSelector() *metav1.LabelSelector {
},
}
}

func (h *boostPodHandler) boostForPod(pod *corev1.Pod) (boost.StartupCPUBoost, bool) {
boostName, ok := pod.Labels[bpod.BoostLabelKey]
if !ok {
return nil, false
}
return h.manager.StartupCPUBoost(pod.Namespace, boostName)
}
213 changes: 185 additions & 28 deletions internal/controller/boost_pod_handler_test.go
Original file line number Diff line number Diff line change
@@ -12,38 +12,195 @@
// See the License for the specific language governing permissions and
// limitations under the License.

package controller
package controller_test

import (
"testing"
"context"

"github.com/go-logr/logr"
"github.com/google/kube-startup-cpu-boost/internal/boost"
"github.com/google/kube-startup-cpu-boost/internal/controller"
"github.com/google/kube-startup-cpu-boost/internal/mock"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"go.uber.org/mock/gomock"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"sigs.k8s.io/controller-runtime/pkg/event"
)

func TestPodLabelSelector(t *testing.T) {
h := &boostPodHandler{}
labelSelector := h.GetPodLabelSelector()
selector, err := metav1.LabelSelectorAsSelector(labelSelector)
if err != nil {
t.Fatalf("could not create selector from label selector: %s", err)
}

inputs := []labels.Set{
{"test": "value"},
{"test": "value", boost.StartupCPUBoostPodLabelKey: "boost-001"},
{"test": "value", boost.StartupCPUBoostPodLabelKey: ""},
}
expected := []bool{
false,
true,
true,
}
for i := range inputs {
result := selector.Matches(inputs[i])
if result != expected[i] {
t.Errorf("input %d result = %v; want %v", i, result, expected[i])
}
}
}
var _ = Describe("BoostPodHandler", func() {
var (
mockCtrl *gomock.Controller
mgrMock *mock.MockManager
mgrMockCall *gomock.Call
podHandler controller.BoostPodHandler
)
BeforeEach(func() {
mockCtrl = gomock.NewController(GinkgoT())
mgrMock = mock.NewMockManager(mockCtrl)
})
JustBeforeEach(func() {
podHandler = controller.NewBoostPodHandler(mgrMock, logr.Discard())
})
Describe("Receives create event", func() {
var (
pod *corev1.Pod
createEvent event.CreateEvent
)
BeforeEach(func() {
pod = podTemplate.DeepCopy()
createEvent = event.CreateEvent{
Object: pod,
}
mgrMockCall = mgrMock.EXPECT().StartupCPUBoost(
gomock.Eq(pod.Namespace),
gomock.Eq(specTemplate.Name),
)
})
JustBeforeEach(func() {
podHandler.Create(context.TODO(), createEvent, nil)
})
When("There is no boost matching the POD", func() {
BeforeEach(func() {
mgrMockCall.Return(nil, false)
})
It("sends a valid call to the boost manager", func() {
mgrMockCall.Times(1)
})
})
When("There is a boost matching the POD", func() {
var (
boostMockNameCall *gomock.Call
boostMockUpsertCall *gomock.Call
)
BeforeEach(func() {
boostMock := mock.NewMockStartupCPUBoost(mockCtrl)
boostMockNameCall = boostMock.EXPECT().Name().
Return(specTemplate.Name)
boostMockUpsertCall = boostMock.EXPECT().UpsertPod(
gomock.Any(),
gomock.Eq(pod),
).Return(nil)
mgrMockCall.Return(boostMock, true)
})
It("sends a valid call to the boost manager and a boost", func() {
mgrMockCall.Times(1)
boostMockNameCall.Times(1)
boostMockUpsertCall.Times(1)
})
})
})
Describe("Receives delete event", func() {
var (
pod *corev1.Pod
deleteEvent event.DeleteEvent
)
BeforeEach(func() {
pod = podTemplate.DeepCopy()
deleteEvent = event.DeleteEvent{
Object: pod,
}
mgrMockCall = mgrMock.EXPECT().StartupCPUBoost(
gomock.Eq(pod.Namespace),
gomock.Eq(specTemplate.Name),
)
})
JustBeforeEach(func() {
podHandler.Delete(context.TODO(), deleteEvent, nil)
})
When("There is no boost matching the POD", func() {
BeforeEach(func() {
mgrMockCall.Return(nil, false)
})
It("sends a valid call to the boost manager", func() {
mgrMockCall.Times(1)
})
})
When("There is a boost matching the POD", func() {
var (
boostMockDeleteCall *gomock.Call
)
BeforeEach(func() {
boostMock := mock.NewMockStartupCPUBoost(mockCtrl)
boostMockDeleteCall = boostMock.EXPECT().DeletePod(
gomock.Any(),
gomock.Eq(pod),
).Return(nil)
mgrMockCall.Return(boostMock, true)
})
It("sends a valid call to the boost manager and a boost", func() {
mgrMockCall.Times(1)
boostMockDeleteCall.Times(1)
})
})
})
Describe("Receives an update event", func() {
var (
pod *corev1.Pod
updateEvent event.UpdateEvent
)
BeforeEach(func() {
pod = podTemplate.DeepCopy()
updateEvent = event.UpdateEvent{
ObjectNew: pod,
}
mgrMockCall = mgrMock.EXPECT().StartupCPUBoost(
gomock.Eq(pod.Namespace),
gomock.Eq(specTemplate.Name),
)
})
JustBeforeEach(func() {
podHandler.Update(context.TODO(), updateEvent, nil)
})
When("There is no boost matching the POD", func() {
BeforeEach(func() {
mgrMockCall.Return(nil, false)
})
It("sends a valid call to the boost manager", func() {
mgrMockCall.Times(1)
})
})
When("There is a boost matching the POD", func() {
var (
boostMockUpsertCall *gomock.Call
)
BeforeEach(func() {
boostMock := mock.NewMockStartupCPUBoost(mockCtrl)
boostMockUpsertCall = boostMock.EXPECT().UpsertPod(
gomock.Any(),
gomock.Eq(pod),
).Return(nil)
mgrMockCall.Return(boostMock, true)
})
It("sends a valid call to the boost manager and a boost", func() {
mgrMockCall.Times(1)
boostMockUpsertCall.Times(1)
})
})
})
Describe("Provides the POD label selector", func() {
var selector *metav1.LabelSelector
JustBeforeEach(func() {
selector = podHandler.GetPodLabelSelector()
})
It("returns selector with a single match expression", func() {
Expect(selector.MatchExpressions).To(HaveLen(1))
})
When("The selector has a single match expression", func() {
var m *metav1.LabelSelectorRequirement
JustBeforeEach(func() {
m = &selector.MatchExpressions[0]
})
It("has a valid key", func() {
Expect(m.Key).To(Equal(boost.StartupCPUBoostPodLabelKey))
})
It("has a valid operator", func() {
Expect(m.Key).To(Equal(boost.StartupCPUBoostPodLabelKey))
})
It("has empty values list", func() {
Expect(m.Values).To(HaveLen(0))
})
})
})
})
104 changes: 104 additions & 0 deletions internal/controller/controller_suite_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
// Copyright 2023 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package controller_test

import (
"testing"
"time"

autoscaling "github.com/google/kube-startup-cpu-boost/api/v1alpha1"
bpod "github.com/google/kube-startup-cpu-boost/internal/boost/pod"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
corev1 "k8s.io/api/core/v1"
apiResource "k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

var (
podTemplate *corev1.Pod
annotTemplate *bpod.BoostPodAnnotation
specTemplate *autoscaling.StartupCPUBoost
)

func TestController(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Controller Suite")
}

var _ = BeforeSuite(func() {
specTemplate = &autoscaling.StartupCPUBoost{
ObjectMeta: metav1.ObjectMeta{
Name: "boost-001",
Namespace: "demo",
},
Spec: autoscaling.StartupCPUBoostSpec{
BoostPercent: 55,
},
}
annotTemplate = &bpod.BoostPodAnnotation{
BoostTimestamp: time.Now(),
InitCPURequests: map[string]string{
"container-one": "500m",
"continer-two": "500m",
},
InitCPULimits: map[string]string{
"container-one": "1",
"continer-two": "1",
},
}
reqQuantity, err := apiResource.ParseQuantity("1")
Expect(err).ShouldNot(HaveOccurred())
limitQuantity, err := apiResource.ParseQuantity("2")
Expect(err).ShouldNot(HaveOccurred())
podTemplate = &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "test-pod",
Namespace: specTemplate.Namespace,
Labels: map[string]string{
bpod.BoostLabelKey: specTemplate.Name,
},
Annotations: map[string]string{
bpod.BoostAnnotationKey: annotTemplate.ToJSON(),
},
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "container-one",
Resources: corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceCPU: reqQuantity,
},
Limits: corev1.ResourceList{
corev1.ResourceCPU: limitQuantity,
},
},
},
{
Name: "container-two",
Resources: corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceCPU: reqQuantity,
},
Limits: corev1.ResourceList{
corev1.ResourceCPU: limitQuantity,
},
},
},
},
},
}
})
17 changes: 17 additions & 0 deletions internal/controller/doc.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
// Copyright 2023 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package controller contains implementation of a controllers managed
// by the controller-manager
package controller
28 changes: 15 additions & 13 deletions internal/controller/startupcpuboost_controller.go
Original file line number Diff line number Diff line change
@@ -65,10 +65,7 @@ func (r *StartupCPUBoostReconciler) Reconcile(ctx context.Context, req ctrl.Requ

// SetupWithManager sets up the controller with the Manager.
func (r *StartupCPUBoostReconciler) SetupWithManager(mgr ctrl.Manager) error {
boostPodHandler := &boostPodHandler{
manager: r.Manager,
log: r.Log,
}
boostPodHandler := NewBoostPodHandler(r.Manager, r.Log.WithName("pod-handler"))
lsPredicate, err := predicate.LabelSelectorPredicate(*boostPodHandler.GetPodLabelSelector())
if err != nil {
return err
@@ -83,15 +80,19 @@ func (r *StartupCPUBoostReconciler) SetupWithManager(mgr ctrl.Manager) error {
}

func (r *StartupCPUBoostReconciler) Create(e event.CreateEvent) bool {
boostObj, ok := e.Object.(*autoscaling.StartupCPUBoost)
spec, ok := e.Object.(*autoscaling.StartupCPUBoost)
if !ok {
return true
}
log := r.Log.WithValues("StartupCPUBoost", klog.KObj(boostObj))
log.V(2).Info("StartupCPUBoost create event")
log := r.Log.WithValues("StartupCPUBoost", klog.KObj(spec))
log.V(2).Info("handling startup-cpu-boost create")
ctx := ctrl.LoggerInto(context.Background(), log)
if err := r.Manager.AddStartupCPUBoost(ctx, boostObj); err != nil {
log.Error(err, "Failed to add startupCPUBoost to boost manager")
boost, err := boost.NewStartupCPUBoost(r.Client, spec)
if err != nil {
log.Error(err, "failed to create startup-cpu-boost from spec")
}
if err := r.Manager.AddStartupCPUBoost(ctx, boost); err != nil {
log.Error(err, "failed to register startup-cpu-boost in manager")
}
return true
}
@@ -102,19 +103,20 @@ func (r *StartupCPUBoostReconciler) Delete(e event.DeleteEvent) bool {
return true
}
log := r.Log.WithValues("StartupCPUBoost", klog.KObj(e.Object))
log.V(2).Info("StartupCPUBoost delete event")
r.Manager.DeleteStartupCPUBoost(boostObj)
log.V(2).Info("handling startup-cpu-boost delete")
ctx := ctrl.LoggerInto(context.Background(), log)
r.Manager.RemoveStartupCPUBoost(ctx, boostObj.Namespace, boostObj.Name)
return true
}

func (r *StartupCPUBoostReconciler) Update(e event.UpdateEvent) bool {
log := r.Log.WithValues("StartupCPUBoost", klog.KObj(e.ObjectNew))
log.V(2).Info("StartupCPUBoost update event")
log.V(2).Info("handling startup-cpu-boost update")
return true
}

func (r *StartupCPUBoostReconciler) Generic(e event.GenericEvent) bool {
log := r.Log.WithValues("StartupCPUBoost", klog.KObj(e.Object))
log.V(2).Info("StartupCPUBoost generic event")
log.V(2).Info("handling startup-cpu-boost generic event")
return true
}
78 changes: 0 additions & 78 deletions internal/controller/suite_test.go

This file was deleted.

125 changes: 125 additions & 0 deletions internal/mock/boost_manager.go
16 changes: 16 additions & 0 deletions internal/mock/doc.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
// Copyright 2023 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package mock is a generated GoMock package.
package mock
277 changes: 277 additions & 0 deletions internal/mock/k8s_client.go
Loading

0 comments on commit 42508d4

Please sign in to comment.