Skip to content

Commit

Permalink
Integration test basic implementation (#111)
Browse files Browse the repository at this point in the history
Co-authored-by: Himanshu Sharma <[email protected]>
  • Loading branch information
AxiomSamarth and himanshu-kun authored Mar 21, 2022
1 parent d02086e commit 418db31
Show file tree
Hide file tree
Showing 226 changed files with 178,379 additions and 86 deletions.
5 changes: 5 additions & 0 deletions .ci/local_integration_test
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
cd ./integration/

echo "Starting integration tests..."

ginkgo -v
2 changes: 1 addition & 1 deletion .ci/test
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ fi

##############################################################################

go test $(go list ./cluster-autoscaler/... | grep -v cloudprovider | grep -v vendor)
go test $(go list ./cluster-autoscaler/... | grep -v cloudprovider | grep -v vendor | grep -v integration)
go test $(go list ./cluster-autoscaler/cloudprovider/mcm/... | grep -v vendor)

#TODO: Return success failure properly
Expand Down
26 changes: 26 additions & 0 deletions cluster-autoscaler/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -109,3 +109,29 @@ test-in-docker: clean docker-builder
bash -c 'cd /gopath/src/k8s.io/autoscaler/cluster-autoscaler && GO111MODULE=off go test -race ./... ${TAGS_FLAG}'

.PHONY: all build test-unit clean format execute-release dev-release docker-builder build-in-docker release generate push-image push-manifest

.PHONY: start
start:
@GO111MODULE=on go run main.go \
--kubeconfig=$(TARGET_KUBECONFIG) \
--cloud-provider=mcm \
--nodes=1:2:$(MACHINE_DEPLOYMENT_ZONE_1) \
--nodes=0:1:$(MACHINE_DEPLOYMENT_ZONE_2) \
--nodes=0:1:$(MACHINE_DEPLOYMENT_ZONE_3) \
--skip-nodes-with-system-pods=false \
--skip-nodes-with-local-storage=false \
--scale-down-delay-after-add=10s \
--scale-down-delay-after-failure=2s \
--v=4 \
--expander=least-waste \
--scale-down-unneeded-time=5s \
--balance-similar-node-groups=true \
--max-node-provision-time=4m \
--leader-elect=$(LEADER_ELECT) \
--leader-elect-retry-period="20s" \
--leader-elect-renew-deadline="30s" \
--leader-elect-lease-duration="40s"

.PHONY: test-integration
test-integration:
../.ci/local_integration_test
62 changes: 32 additions & 30 deletions cluster-autoscaler/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ require (
github.com/golang/mock v1.4.4
github.com/jmespath/go-jmespath v0.4.0
github.com/json-iterator/go v1.1.10
github.com/onsi/ginkgo v1.16.2
github.com/onsi/gomega v1.11.0
github.com/pkg/errors v0.9.1
github.com/satori/go.uuid v1.2.0
github.com/spf13/pflag v1.0.5
Expand All @@ -32,7 +34,7 @@ require (
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
google.golang.org/api v0.20.0
gopkg.in/gcfg.v1 v1.2.0
gopkg.in/yaml.v2 v2.2.8
gopkg.in/yaml.v2 v2.4.0
k8s.io/api v0.20.6
k8s.io/apimachinery v0.20.6
k8s.io/apiserver v0.20.6
Expand Down Expand Up @@ -287,8 +289,8 @@ replace (
github.com/naoina/go-stringutil => github.com/naoina/go-stringutil v0.1.0
github.com/naoina/toml => github.com/naoina/toml v0.1.1
github.com/olekukonko/tablewriter => github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5
github.com/onsi/ginkgo => github.com/onsi/ginkgo v1.11.0
github.com/onsi/gomega => github.com/onsi/gomega v1.7.0
github.com/onsi/ginkgo => github.com/onsi/ginkgo v1.16.2
github.com/onsi/gomega => github.com/onsi/gomega v1.11.0
github.com/opencontainers/go-digest => github.com/opencontainers/go-digest v1.0.0
github.com/opencontainers/image-spec => github.com/opencontainers/image-spec v1.0.1
github.com/opencontainers/runc => github.com/opencontainers/runc v1.0.0-rc92
Expand Down Expand Up @@ -394,36 +396,36 @@ replace (
gotest.tools => gotest.tools v2.2.0+incompatible
gotest.tools/v3 => gotest.tools/v3 v3.0.2
honnef.co/go/tools => honnef.co/go/tools v0.0.1-2020.1.3
k8s.io/api => /tmp/ca-update-vendor.4sZn/kubernetes/staging/src/k8s.io/api
k8s.io/apiextensions-apiserver => /tmp/ca-update-vendor.4sZn/kubernetes/staging/src/k8s.io/apiextensions-apiserver
k8s.io/apimachinery => /tmp/ca-update-vendor.4sZn/kubernetes/staging/src/k8s.io/apimachinery
k8s.io/apiserver => /tmp/ca-update-vendor.4sZn/kubernetes/staging/src/k8s.io/apiserver
k8s.io/cli-runtime => /tmp/ca-update-vendor.4sZn/kubernetes/staging/src/k8s.io/cli-runtime
k8s.io/client-go => /tmp/ca-update-vendor.4sZn/kubernetes/staging/src/k8s.io/client-go
k8s.io/cloud-provider => /tmp/ca-update-vendor.4sZn/kubernetes/staging/src/k8s.io/cloud-provider
k8s.io/cluster-bootstrap => /tmp/ca-update-vendor.4sZn/kubernetes/staging/src/k8s.io/cluster-bootstrap
k8s.io/code-generator => /tmp/ca-update-vendor.4sZn/kubernetes/staging/src/k8s.io/code-generator
k8s.io/component-base => /tmp/ca-update-vendor.4sZn/kubernetes/staging/src/k8s.io/component-base
k8s.io/component-helpers => /tmp/ca-update-vendor.4sZn/kubernetes/staging/src/k8s.io/component-helpers
k8s.io/controller-manager => /tmp/ca-update-vendor.4sZn/kubernetes/staging/src/k8s.io/controller-manager
k8s.io/cri-api => /tmp/ca-update-vendor.4sZn/kubernetes/staging/src/k8s.io/cri-api
k8s.io/csi-translation-lib => /tmp/ca-update-vendor.4sZn/kubernetes/staging/src/k8s.io/csi-translation-lib
k8s.io/api => /tmp/ca-update-vendor.zK4S/kubernetes/staging/src/k8s.io/api
k8s.io/apiextensions-apiserver => /tmp/ca-update-vendor.zK4S/kubernetes/staging/src/k8s.io/apiextensions-apiserver
k8s.io/apimachinery => /tmp/ca-update-vendor.zK4S/kubernetes/staging/src/k8s.io/apimachinery
k8s.io/apiserver => /tmp/ca-update-vendor.zK4S/kubernetes/staging/src/k8s.io/apiserver
k8s.io/cli-runtime => /tmp/ca-update-vendor.zK4S/kubernetes/staging/src/k8s.io/cli-runtime
k8s.io/client-go => /tmp/ca-update-vendor.zK4S/kubernetes/staging/src/k8s.io/client-go
k8s.io/cloud-provider => /tmp/ca-update-vendor.zK4S/kubernetes/staging/src/k8s.io/cloud-provider
k8s.io/cluster-bootstrap => /tmp/ca-update-vendor.zK4S/kubernetes/staging/src/k8s.io/cluster-bootstrap
k8s.io/code-generator => /tmp/ca-update-vendor.zK4S/kubernetes/staging/src/k8s.io/code-generator
k8s.io/component-base => /tmp/ca-update-vendor.zK4S/kubernetes/staging/src/k8s.io/component-base
k8s.io/component-helpers => /tmp/ca-update-vendor.zK4S/kubernetes/staging/src/k8s.io/component-helpers
k8s.io/controller-manager => /tmp/ca-update-vendor.zK4S/kubernetes/staging/src/k8s.io/controller-manager
k8s.io/cri-api => /tmp/ca-update-vendor.zK4S/kubernetes/staging/src/k8s.io/cri-api
k8s.io/csi-translation-lib => /tmp/ca-update-vendor.zK4S/kubernetes/staging/src/k8s.io/csi-translation-lib
k8s.io/gengo => k8s.io/gengo v0.0.0-20201113003025-83324d819ded
k8s.io/heapster => k8s.io/heapster v1.2.0-beta.1
k8s.io/klog/v2 => k8s.io/klog/v2 v2.4.0
k8s.io/kube-aggregator => /tmp/ca-update-vendor.4sZn/kubernetes/staging/src/k8s.io/kube-aggregator
k8s.io/kube-controller-manager => /tmp/ca-update-vendor.4sZn/kubernetes/staging/src/k8s.io/kube-controller-manager
k8s.io/kube-aggregator => /tmp/ca-update-vendor.zK4S/kubernetes/staging/src/k8s.io/kube-aggregator
k8s.io/kube-controller-manager => /tmp/ca-update-vendor.zK4S/kubernetes/staging/src/k8s.io/kube-controller-manager
k8s.io/kube-openapi => k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd
k8s.io/kube-proxy => /tmp/ca-update-vendor.4sZn/kubernetes/staging/src/k8s.io/kube-proxy
k8s.io/kube-scheduler => /tmp/ca-update-vendor.4sZn/kubernetes/staging/src/k8s.io/kube-scheduler
k8s.io/kubectl => /tmp/ca-update-vendor.4sZn/kubernetes/staging/src/k8s.io/kubectl
k8s.io/kubelet => /tmp/ca-update-vendor.4sZn/kubernetes/staging/src/k8s.io/kubelet
k8s.io/legacy-cloud-providers => /tmp/ca-update-vendor.4sZn/kubernetes/staging/src/k8s.io/legacy-cloud-providers
k8s.io/metrics => /tmp/ca-update-vendor.4sZn/kubernetes/staging/src/k8s.io/metrics
k8s.io/mount-utils => /tmp/ca-update-vendor.4sZn/kubernetes/staging/src/k8s.io/mount-utils
k8s.io/sample-apiserver => /tmp/ca-update-vendor.4sZn/kubernetes/staging/src/k8s.io/sample-apiserver
k8s.io/sample-cli-plugin => /tmp/ca-update-vendor.4sZn/kubernetes/staging/src/k8s.io/sample-cli-plugin
k8s.io/sample-controller => /tmp/ca-update-vendor.4sZn/kubernetes/staging/src/k8s.io/sample-controller
k8s.io/kube-proxy => /tmp/ca-update-vendor.zK4S/kubernetes/staging/src/k8s.io/kube-proxy
k8s.io/kube-scheduler => /tmp/ca-update-vendor.zK4S/kubernetes/staging/src/k8s.io/kube-scheduler
k8s.io/kubectl => /tmp/ca-update-vendor.zK4S/kubernetes/staging/src/k8s.io/kubectl
k8s.io/kubelet => /tmp/ca-update-vendor.zK4S/kubernetes/staging/src/k8s.io/kubelet
k8s.io/legacy-cloud-providers => /tmp/ca-update-vendor.zK4S/kubernetes/staging/src/k8s.io/legacy-cloud-providers
k8s.io/metrics => /tmp/ca-update-vendor.zK4S/kubernetes/staging/src/k8s.io/metrics
k8s.io/mount-utils => /tmp/ca-update-vendor.zK4S/kubernetes/staging/src/k8s.io/mount-utils
k8s.io/sample-apiserver => /tmp/ca-update-vendor.zK4S/kubernetes/staging/src/k8s.io/sample-apiserver
k8s.io/sample-cli-plugin => /tmp/ca-update-vendor.zK4S/kubernetes/staging/src/k8s.io/sample-cli-plugin
k8s.io/sample-controller => /tmp/ca-update-vendor.zK4S/kubernetes/staging/src/k8s.io/sample-controller
k8s.io/system-validators => k8s.io/system-validators v1.2.0
k8s.io/utils => k8s.io/utils v0.0.0-20201110183641-67b214c5f920
modernc.org/cc => modernc.org/cc v1.0.0
Expand All @@ -450,4 +452,4 @@ replace github.com/gardener/machine-controller-manager-provider-aws => github.co

replace github.com/gardener/machine-controller-manager-provider-azure => github.com/gardener/machine-controller-manager-provider-azure v0.5.0

replace k8s.io/kubernetes => /tmp/ca-update-vendor.4sZn/kubernetes
replace k8s.io/kubernetes => /tmp/ca-update-vendor.zK4S/kubernetes
2 changes: 2 additions & 0 deletions cluster-autoscaler/go.mod-extra
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ module k8s.io/autoscaler/cluster-autoscaler
go 1.15

require (
github.com/onsi/ginkgo v1.16.2
github.com/onsi/gomega v1.11.0
github.com/digitalocean/godo v1.27.0
github.com/rancher/go-rancher v0.1.0
github.com/gardener/machine-controller-manager v0.42.0
Expand Down
15 changes: 7 additions & 8 deletions cluster-autoscaler/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -211,6 +211,7 @@ github.com/go-openapi/validate v0.19.5/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85n
github.com/go-ozzo/ozzo-validation v3.5.0+incompatible h1:sUy/in/P6askYr16XJgTKq/0SZhiWsdg4WZGaLsGQkM=
github.com/go-ozzo/ozzo-validation v3.5.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
github.com/godbus/dbus/v5 v5.0.3 h1:ZqHaoEF7TBzh4jzPmqVhE/5A1z9of6orkAe5uHoAeME=
github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
Expand Down Expand Up @@ -287,8 +288,6 @@ github.com/heketi/heketi v9.0.1-0.20190917153846-c2e2a4ab7ab9+incompatible h1:ys
github.com/heketi/heketi v9.0.1-0.20190917153846-c2e2a4ab7ab9+incompatible/go.mod h1:bB9ly3RchcQqsQ9CpyaQwvva7RS5ytVoSoholZQON6o=
github.com/heketi/tests v0.0.0-20151005000721-f3775cbcefd6 h1:oJ/NLadJn5HoxvonA6VxG31lg0d6XOURNA09BTtM4fY=
github.com/heketi/tests v0.0.0-20151005000721-f3775cbcefd6/go.mod h1:xGMAM8JLi7UkZt1i4FQeQy0R2T8GLUwQhOP5M1gBhy4=
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q=
github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
Expand Down Expand Up @@ -386,11 +385,13 @@ github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+
github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0=
github.com/naoina/toml v0.1.1/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
github.com/onsi/ginkgo v1.11.0 h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw=
github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=
github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/ginkgo v1.16.2 h1:HFB2fbVIlhIfCfOW81bZFbiC/RvnpXSdhbF2/DJr134=
github.com/onsi/ginkgo v1.16.2/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E=
github.com/onsi/gomega v1.11.0 h1:+CqWgvj0OZycCaqclBD1pxKHAU+tOkHmQIWvDHq2aug=
github.com/onsi/gomega v1.11.0/go.mod h1:azGKhqFUon9Vuj0YmTfLSmx0FUwqXYSTl5re8lQLTUg=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI=
Expand Down Expand Up @@ -537,8 +538,6 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogR
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/gcfg.v1 v1.2.0 h1:0HIbH907iBTAntm+88IJV2qmJALDAh8sPekI9Vc1fm0=
gopkg.in/gcfg.v1 v1.2.0/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o=
gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
Expand Down
122 changes: 122 additions & 0 deletions cluster-autoscaler/integration/factory.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
package integration

import (
"fmt"
"os"
"path/filepath"

MCMClientset "github.com/gardener/machine-controller-manager/pkg/client/clientset/versioned"
"github.com/onsi/gomega/gexec"

"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
)

var (
// path for storing log files of autoscaler process
targetDir = filepath.Join("logs")

// autoscaler log file
CALogFile = filepath.Join(targetDir, "autoscaler_processs.log")

// make processes/sessions started by gexec. available only if the controllers are running in local setup. updated during runtime
autoscalerSession *gexec.Session

// controlClusterNamespace is the Shoot namespace in the Seed
controlClusterNamespace = os.Getenv("CONTROL_NAMESPACE")
)

// Cluster holds the clients of a Cluster (like Control Cluster and Target Cluster)
type Cluster struct {
restConfig *rest.Config
Clientset *kubernetes.Clientset
MCMClient *MCMClientset.Clientset
KubeConfigFilePath string
}

// ClusterName retrieves cluster name from the kubeconfig
func (c *Cluster) ClusterName() (string, error) {
var clusterName string
config, err := clientcmd.LoadFromFile(c.KubeConfigFilePath)
if err != nil {
return clusterName, err
}
for contextName, context := range config.Contexts {
if contextName == config.CurrentContext {
clusterName = context.Cluster
}
}
return clusterName, err
}

// Driver is the driver used for executing various integration tests and utils
// interacting with both control and target clusters
type Driver struct {
// Control cluster resource containing ClientSets for accessing kubernetes resources
// And kubeconfig file path for the cluster
controlCluster *Cluster

// Target cluster resource containing ClientSets for accessing kubernetes resources
// And kubeconfig file path for the cluster
targetCluster *Cluster
}

// NewDriver is the construtor for the driver type
func NewDriver(controlKubeconfig, targetKubeconfig string) *Driver {
var (
driver = &Driver{}
err error
)

driver.controlCluster, err = NewCluster(controlKubeconfig)
if err != nil {
fmt.Printf("%s", err.Error())
return nil
}

driver.targetCluster, err = NewCluster(targetKubeconfig)
if err != nil {
return nil
}

if controlClusterNamespace == "" {
controlClusterNamespace, err := driver.targetCluster.ClusterName()
if err != nil {
return nil
}

err = os.Setenv("CONTROL_NAMESPACE", controlClusterNamespace)
if err != nil {
return nil
}
}

return driver
}

// NewCluster returns a Cluster struct
func NewCluster(kubeConfigPath string) (c *Cluster, e error) {
config, err := clientcmd.BuildConfigFromFlags("", kubeConfigPath)
if err == nil {
c = &Cluster{
KubeConfigFilePath: kubeConfigPath,
restConfig: config,
}
} else {
fmt.Printf("%s", err.Error())
c = &Cluster{}
}

clientset, err := kubernetes.NewForConfig(c.restConfig)
if err == nil {
c.Clientset = clientset
}

MCMClient, err := MCMClientset.NewForConfig(c.restConfig)
if err == nil {
c.MCMClient = MCMClient
}

return c, err
}
Loading

0 comments on commit 418db31

Please sign in to comment.