diff --git a/Makefile b/Makefile index 91e39ef8096d..2ec410800088 100644 --- a/Makefile +++ b/Makefile @@ -24,20 +24,23 @@ dev: verify test ## Run all steps in the developer loop ci: toolchain verify licenses battletest ## Run all steps used by continuous integration test: ## Run tests - ginkgo -r + go test -v ./pkg/... strongertests: # Run randomized, racing, code coveraged, tests ginkgo -r \ -cover -coverprofile=coverage.out -outputdir=. -coverpkg=./pkg/... \ - --randomizeAllSpecs --randomizeSuites -race + --randomizeAllSpecs --randomizeSuites -race ./pkg/... + +e2etests: ## Run the e2e suite against your local cluster + go test -v ./test/... -environment-name=${CLUSTER_NAME} benchmark: go test -tags=test_performance -run=NoTests -bench=. ./... deflake: for i in $(shell seq 1 5); do make strongertests || exit 1; done - ginkgo -r -race -tags random_test_delay + ginkgo -r pkg -race -tags random_test_delay battletest: strongertests go tool cover -html coverage.out -o coverage.html diff --git a/pkg/apis/provisioning/v1alpha5/labels.go b/pkg/apis/provisioning/v1alpha5/labels.go index 881753032b23..df087023ad36 100644 --- a/pkg/apis/provisioning/v1alpha5/labels.go +++ b/pkg/apis/provisioning/v1alpha5/labels.go @@ -29,6 +29,12 @@ var ( // Karpenter specific domains and labels KarpenterLabelDomain = "karpenter.sh" + + ProvisionerNameLabelKey = Group + "/provisioner-name" + DoNotEvictPodAnnotationKey = Group + "/do-not-evict" + EmptinessTimestampAnnotationKey = Group + "/emptiness-timestamp" + TerminationFinalizer = Group + "/termination" + LabelCapacityType = KarpenterLabelDomain + "/capacity-type" LabelNodeInitialized = KarpenterLabelDomain + "/initialized" diff --git a/pkg/apis/provisioning/v1alpha5/register.go b/pkg/apis/provisioning/v1alpha5/register.go index 061562b5321f..501838311d93 100644 --- a/pkg/apis/provisioning/v1alpha5/register.go +++ b/pkg/apis/provisioning/v1alpha5/register.go @@ -40,10 +40,6 @@ var ( metav1.AddToGroupVersion(scheme, SchemeGroupVersion) return nil }) - ProvisionerNameLabelKey = Group + "/provisioner-name" - DoNotEvictPodAnnotationKey = Group + "/do-not-evict" - EmptinessTimestampAnnotationKey = Group + "/emptiness-timestamp" - TerminationFinalizer = Group + "/termination" ) const ( diff --git a/pkg/cloudprovider/aws/apis/v1alpha1/provider.go b/pkg/cloudprovider/aws/apis/v1alpha1/provider.go index 89549d9f8fad..ce24c67a1c62 100644 --- a/pkg/cloudprovider/aws/apis/v1alpha1/provider.go +++ b/pkg/cloudprovider/aws/apis/v1alpha1/provider.go @@ -192,7 +192,7 @@ type BlockDevice struct { func Deserialize(provider *v1alpha5.Provider) (*AWS, error) { if provider == nil { - return nil, fmt.Errorf("invariant violated: spec.provider is not defined. Is the defaulting webhook installed?") + return nil, fmt.Errorf("invariant violated: spec.provider is not defined. Is the validating webhook installed?") } a := &AWS{} _, gvk, err := Codec.UniversalDeserializer().Decode(provider.Raw, nil, a) @@ -207,7 +207,7 @@ func Deserialize(provider *v1alpha5.Provider) (*AWS, error) { func (a *AWS) Serialize(provider *v1alpha5.Provider) error { if provider == nil { - return fmt.Errorf("invariant violated: spec.provider is not defined. Is the defaulting webhook installed?") + return fmt.Errorf("invariant violated: spec.provider is not defined. Is the validating webhook installed?") } bytes, err := json.Marshal(a) if err != nil { diff --git a/pkg/cloudprovider/aws/suite_test.go b/pkg/cloudprovider/aws/suite_test.go index 075f2dcca6b0..58edcd23e086 100644 --- a/pkg/cloudprovider/aws/suite_test.go +++ b/pkg/cloudprovider/aws/suite_test.go @@ -982,7 +982,7 @@ var _ = Describe("Allocation", func() { } nodeTemplate := test.AWSNodeTemplate(test.AWSNodeTemplateOptions{ UserData: aws.String(string(content)), - AWS: provider, + AWS: *provider, ObjectMeta: metav1.ObjectMeta{Name: providerRefName}}) ExpectApplied(ctx, env.Client, nodeTemplate) controller := provisioning.NewController(injection.WithOptions(ctx, opts), cfg, env.Client, clientSet.CoreV1(), recorder, cloudProvider, cluster) @@ -1010,7 +1010,7 @@ var _ = Describe("Allocation", func() { } nodeTemplate := test.AWSNodeTemplate(test.AWSNodeTemplateOptions{ UserData: nil, - AWS: provider, + AWS: *provider, ObjectMeta: metav1.ObjectMeta{Name: providerRefName}}) ExpectApplied(ctx, env.Client, nodeTemplate) controller := provisioning.NewController(injection.WithOptions(ctx, opts), cfg, env.Client, clientSet.CoreV1(), recorder, cloudProvider, cluster) @@ -1049,7 +1049,7 @@ var _ = Describe("Allocation", func() { } nodeTemplate := test.AWSNodeTemplate(test.AWSNodeTemplateOptions{ UserData: aws.String("#/bin/bash\n ./not-toml.sh"), - AWS: provider, + AWS: *provider, ObjectMeta: metav1.ObjectMeta{Name: providerRefName}}) ExpectApplied(ctx, env.Client, nodeTemplate) controller := provisioning.NewController(ctx, cfg, env.Client, clientSet.CoreV1(), recorder, cloudProvider, cluster) @@ -1071,7 +1071,7 @@ var _ = Describe("Allocation", func() { } nodeTemplate := test.AWSNodeTemplate(test.AWSNodeTemplateOptions{ UserData: aws.String(string(content)), - AWS: provider, + AWS: *provider, ObjectMeta: metav1.ObjectMeta{Name: providerRefName}}) ExpectApplied(ctx, env.Client, nodeTemplate) controller := provisioning.NewController(injection.WithOptions(ctx, opts), cfg, env.Client, clientSet.CoreV1(), recorder, cloudProvider, cluster) @@ -1095,7 +1095,7 @@ var _ = Describe("Allocation", func() { } nodeTemplate := test.AWSNodeTemplate(test.AWSNodeTemplateOptions{ UserData: nil, - AWS: provider, + AWS: *provider, ObjectMeta: metav1.ObjectMeta{Name: providerRefName}}) ExpectApplied(ctx, env.Client, nodeTemplate) controller := provisioning.NewController(injection.WithOptions(ctx, opts), cfg, env.Client, clientSet.CoreV1(), recorder, cloudProvider, cluster) @@ -1119,7 +1119,7 @@ var _ = Describe("Allocation", func() { } nodeTemplate := test.AWSNodeTemplate(test.AWSNodeTemplateOptions{ UserData: aws.String("#/bin/bash\n ./not-mime.sh"), - AWS: provider, + AWS: *provider, ObjectMeta: metav1.ObjectMeta{Name: providerRefName}}) ExpectApplied(ctx, env.Client, nodeTemplate) controller := provisioning.NewController(injection.WithOptions(ctx, opts), cfg, env.Client, clientSet.CoreV1(), recorder, cloudProvider, cluster) diff --git a/pkg/test/awsnodetemplate.go b/pkg/test/awsnodetemplate.go index 8251de8dc499..1a0c7b69b4dd 100644 --- a/pkg/test/awsnodetemplate.go +++ b/pkg/test/awsnodetemplate.go @@ -27,7 +27,7 @@ import ( type AWSNodeTemplateOptions struct { metav1.ObjectMeta UserData *string - AWS *aws.AWS + AWS aws.AWS } func AWSNodeTemplate(overrides ...AWSNodeTemplateOptions) *v1alpha1.AWSNodeTemplate { @@ -41,7 +41,7 @@ func AWSNodeTemplate(overrides ...AWSNodeTemplateOptions) *v1alpha1.AWSNodeTempl ObjectMeta: ObjectMeta(options.ObjectMeta), Spec: v1alpha1.AWSNodeTemplateSpec{ UserData: options.UserData, - AWS: *options.AWS, + AWS: options.AWS, }, } } diff --git a/pkg/test/expectations/expectations.go b/pkg/test/expectations/expectations.go index 73f5e1ddf540..c7730ea425a6 100644 --- a/pkg/test/expectations/expectations.go +++ b/pkg/test/expectations/expectations.go @@ -24,9 +24,7 @@ import ( "github.com/aws/karpenter/pkg/test" "github.com/onsi/ginkgo" - - //nolint:revive,stylecheck - . "github.com/onsi/gomega" + . "github.com/onsi/gomega" //nolint:revive,stylecheck prometheus "github.com/prometheus/client_model/go" "sigs.k8s.io/controller-runtime/pkg/metrics" diff --git a/pkg/test/pods.go b/pkg/test/pods.go index 41ef203c0593..af33bce8ba9c 100644 --- a/pkg/test/pods.go +++ b/pkg/test/pods.go @@ -75,7 +75,7 @@ func Pod(overrides ...PodOptions) *v1.Pod { } } if options.Image == "" { - options.Image = "k8s.gcr.io/pause" + options.Image = "alpine" } volumes := []v1.Volume{} for _, pvc := range options.PersistentVolumeClaims { diff --git a/pkg/test/provisioner.go b/pkg/test/provisioner.go index ba69f012aa68..ffbff728c8cf 100644 --- a/pkg/test/provisioner.go +++ b/pkg/test/provisioner.go @@ -26,6 +26,8 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "knative.dev/pkg/logging" + "knative.dev/pkg/ptr" "github.com/aws/karpenter/pkg/apis/provisioning/v1alpha5" ) @@ -44,7 +46,7 @@ type ProvisionerOptions struct { Status v1alpha5.ProvisionerStatus } -// Provisioner creates a test pod with defaults that can be overridden by ProvisionerOptions. +// Provisioner creates a test provisioner with defaults that can be overridden by ProvisionerOptions. // Overrides are applied in order, with a last write wins semantic. func Provisioner(overrides ...ProvisionerOptions) *v1alpha5.Provisioner { options := ProvisionerOptions{} @@ -59,25 +61,35 @@ func Provisioner(overrides ...ProvisionerOptions) *v1alpha5.Provisioner { if options.Limits == nil { options.Limits = v1.ResourceList{v1.ResourceCPU: resource.MustParse("1000")} } - if options.Provider == nil { - options.Provider = struct{}{} - } - provider, _ := json.Marshal(options.Provider) + provisioner := &v1alpha5.Provisioner{ ObjectMeta: ObjectMeta(options.ObjectMeta), Spec: v1alpha5.ProvisionerSpec{ Requirements: options.Requirements, KubeletConfiguration: options.Kubelet, - Provider: &runtime.RawExtension{Raw: provider}, ProviderRef: options.ProviderRef, Taints: options.Taints, StartupTaints: options.StartupTaints, Labels: options.Labels, Limits: &v1alpha5.Limits{Resources: options.Limits}, + TTLSecondsAfterEmpty: ptr.Int64(10), }, Status: options.Status, } + + if options.ProviderRef == nil { + if options.Provider == nil { + options.Provider = struct{}{} + } + provider, err := json.Marshal(options.Provider) + if err != nil { + panic(err) + } + provisioner.Spec.Provider = &runtime.RawExtension{Raw: provider} + } provisioner.SetDefaults(context.Background()) - _ = provisioner.Validate(context.Background()) + if err := provisioner.Validate(context.Background()); err != nil { + logging.FromContext(context.TODO()).Info("TODO: Fix the tests that cause this") + } return provisioner } diff --git a/test/README.md b/test/README.md index 4df48047e69f..310e4a38ac1c 100644 --- a/test/README.md +++ b/test/README.md @@ -13,3 +13,10 @@ Testing infrastructure will be divided up into three layers: Management Cluster, - `Clusters Under Test`: Rapid iteration [KIT](https://github.com/awslabs/kubernetes-iteration-toolkit) Guest Clusters and EKS Clusters where test suites will run. *Note: A more formal design discussing testing infrastructure will come soon.* + +## Developing +Use the Tekton UI to manage and monitor resources and test-runs: +``` +kubectl port-forward service/tekton-dashboard -n tekton-pipelines 9097& +open http://localhost:9097 +``` diff --git a/test/infrastructure/management-cluster.cloudformation.yaml b/test/infrastructure/management-cluster.cloudformation.yaml index 7f05d85f29c6..e673ec77bc4b 100644 --- a/test/infrastructure/management-cluster.cloudformation.yaml +++ b/test/infrastructure/management-cluster.cloudformation.yaml @@ -5,17 +5,17 @@ Parameters: Type: String Description: "Host cluster name" Resources: - KarpenterGuestClusterNodeInstanceProfile: + KarpenterNodeInstanceProfile: Type: "AWS::IAM::InstanceProfile" Properties: - InstanceProfileName: !Sub "KarpenterGuestClusterNodeInstanceProfile-${ClusterName}" + InstanceProfileName: !Sub "KarpenterNodeInstanceProfile-${ClusterName}" Path: "/" Roles: - - Ref: "KarpenterGuestClusterNodeRole" - KarpenterGuestClusterNodeRole: + - Ref: "KarpenterNodeRole" + KarpenterNodeRole: Type: "AWS::IAM::Role" Properties: - RoleName: !Sub "KarpenterGuestClusterNodeRole-${ClusterName}" + RoleName: !Sub "KarpenterNodeRole-${ClusterName}" Path: / AssumeRolePolicyDocument: Version: "2012-10-17" diff --git a/test/infrastructure/scripts/step-01-config.sh b/test/infrastructure/scripts/step-01-config.sh index 544d352ab832..d408146be59a 100755 --- a/test/infrastructure/scripts/step-01-config.sh +++ b/test/infrastructure/scripts/step-01-config.sh @@ -1,6 +1,5 @@ -export CLUSTER_NAME="${CLUSTER_NAME:-karpenter-test-cluster}" +export CLUSTER_NAME="${CLUSTER_NAME:-karpenter-test-infrastructure}" export AWS_PROFILE="${AWS_PROFILE:-karpenter-ci}" export AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) export AWS_REGION="${AWS_REGION:-us-west-2}" -export KARPENTER_VERSION="${KARPENTER_VERSION:-v0.9.0}" export AWS_PAGER="" diff --git a/test/infrastructure/scripts/step-02-eksctl-cluster.sh b/test/infrastructure/scripts/step-02-eksctl-cluster.sh index 0ad508ad9d9d..f4a1f765bf6b 100755 --- a/test/infrastructure/scripts/step-02-eksctl-cluster.sh +++ b/test/infrastructure/scripts/step-02-eksctl-cluster.sh @@ -1,7 +1,7 @@ cmd="create" K8S_VERSION="1.22" eksctl get cluster --name "${CLUSTER_NAME}" && cmd="upgrade" -eksctl ${cmd} cluster -f - << EOF +eksctl ${cmd} cluster -f - < /tmp/aws-auth-patch.yml +ROLE=" - rolearn: arn:aws:iam::${AWS_ACCOUNT_ID}:role/KarpenterNodeRole-${CLUSTER_NAME}\n username: system:node:{{EC2PrivateDNSName}}\n groups:\n - system:nodes\n - system:bootstrappers" +kubectl get -n kube-system configmap/aws-auth -o yaml | awk "/mapRoles: \|/{print;print \"${ROLE}\";next}1" >/tmp/aws-auth-patch.yml kubectl patch configmap/aws-auth -n kube-system --patch "$(cat /tmp/aws-auth-patch.yml)" eksctl create iamserviceaccount \ @@ -21,15 +21,16 @@ export KARPENTER_IAM_ROLE_ARN="arn:aws:iam::${AWS_ACCOUNT_ID}:role/${CLUSTER_NAM aws iam create-service-linked-role --aws-service-name spot.amazonaws.com || true -if [ -z "$(helm repo list | grep karpenter)" ] ; then +if [ -z "$(helm repo list | grep karpenter)" ]; then helm repo add karpenter https://charts.karpenter.sh fi helm repo update -helm upgrade --install --namespace karpenter --create-namespace \ - karpenter karpenter/karpenter \ - --version ${KARPENTER_VERSION} \ +helm upgrade --install --namespace karpenter --create-namespace karpenter --version v0.9.0 \ --set serviceAccount.annotations."eks\.amazonaws\.com/role-arn"="arn:aws:iam::${AWS_ACCOUNT_ID}:role/${CLUSTER_NAME}-karpenter" \ --set clusterName=${CLUSTER_NAME} \ - --set clusterEndpoint=$(aws eks describe-cluster --name ${CLUSTER_NAME} --query "cluster.endpoint" --output json) \ - --set aws.defaultInstanceProfile=KarpenterGuestClusterNodeInstanceProfile-${CLUSTER_NAME} \ + --set clusterEndpoint=$(aws eks describe-cluster --name ${CLUSTER_NAME} --query "cluster.endpoint" --output json) \ + --set aws.defaultInstanceProfile=KarpenterNodeInstanceProfile-${CLUSTER_NAME} \ + --set tolerations[0].key="CriticalAddonsOnly" \ + --set tolerations[0].operator="Exists" \ + karpenter/karpenter \ --wait diff --git a/test/infrastructure/scripts/step-07-provisioners.sh b/test/infrastructure/scripts/step-07-provisioners.sh index 512f1bc562fd..23b626922dd2 100755 --- a/test/infrastructure/scripts/step-07-provisioners.sh +++ b/test/infrastructure/scripts/step-07-provisioners.sh @@ -1,4 +1,4 @@ -# Provisioner for KIT Guest Clusters +# General purpose provisioner for test execution cat < /tmp/aws-auth-patch.yml +kubectl get -n kube-system configmap/aws-auth -o yaml | awk "/mapRoles: \|/{print;print \"${ROLE}\";next}1" >/tmp/aws-auth-patch.yml kubectl patch configmap/aws-auth -n kube-system --patch "$(cat /tmp/aws-auth-patch.yml)" kubectl annotate --overwrite serviceaccount -n tekton-tests tekton "eks.amazonaws.com/role-arn=arn:aws:iam::${AWS_ACCOUNT_ID}:role/${CLUSTER_NAME}-tekton" diff --git a/test/infrastructure/setup-management-cluster.sh b/test/infrastructure/setup-management-cluster.sh index e098292b11c4..d294d55ecbb1 100755 --- a/test/infrastructure/setup-management-cluster.sh +++ b/test/infrastructure/setup-management-cluster.sh @@ -1,7 +1,10 @@ #!/usr/bin/env bash set -euo pipefail -SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" +SCRIPTPATH="$( + cd "$(dirname "$0")" + pwd -P +)" declare -a steps=( scripts/step-01-config.sh @@ -22,4 +25,3 @@ for step in "${steps[@]}"; do done echo "✅ Successfully setup test infrastructure" - diff --git a/test/pkg/environment/environment.go b/test/pkg/environment/environment.go new file mode 100644 index 000000000000..d15d098cf2ed --- /dev/null +++ b/test/pkg/environment/environment.go @@ -0,0 +1,84 @@ +package environment + +import ( + "context" + "flag" + "fmt" + "os" + "path" + "testing" + "time" + + "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "knative.dev/pkg/environment" + loggingtesting "knative.dev/pkg/logging/testing" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/aws/karpenter/pkg/apis" + "github.com/aws/karpenter/pkg/utils/env" +) + +var EnvironmentName = flag.String("environment-name", env.WithDefaultString("ENVIRONMENT_NAME", ""), "Environment name enables discovery of the testing environment") + +type Environment struct { + context.Context + Options *Options + Client client.Client +} + +func NewEnvironment(t *testing.T) (*Environment, error) { + ctx := loggingtesting.TestContextWithLogger(t) + client, err := NewLocalClient() + if err != nil { + return nil, err + } + options, err := NewOptions() + if err != nil { + return nil, err + } + gomega.SetDefaultEventuallyTimeout(10 * time.Minute) + gomega.SetDefaultEventuallyPollingInterval(1 * time.Second) + return &Environment{Context: ctx, Options: options, Client: client}, nil +} + +func NewLocalClient() (client.Client, error) { + scheme := runtime.NewScheme() + if err := clientgoscheme.AddToScheme(scheme); err != nil { + return nil, err + } + if err := apis.AddToScheme(scheme); err != nil { + return nil, err + } + home, err := os.UserHomeDir() + if err != nil { + return nil, err + } + config, err := (&environment.ClientConfig{Kubeconfig: path.Join(home, ".kube/config")}).GetRESTConfig() + if err != nil { + return nil, err + } + return client.New(config, client.Options{Scheme: scheme}) +} + +type Options struct { + EnvironmentName string +} + +func NewOptions() (*Options, error) { + options := &Options{ + EnvironmentName: *EnvironmentName, + } + if err := options.Validate(); err != nil { + return nil, err + } + return options, nil +} + +func (o Options) Validate() error { + if o.EnvironmentName == "" { + return fmt.Errorf("--environment-name must be defined") + } + return nil +} diff --git a/test/pkg/environment/expectations.go b/test/pkg/environment/expectations.go new file mode 100644 index 000000000000..efada4d33f0e --- /dev/null +++ b/test/pkg/environment/expectations.go @@ -0,0 +1,69 @@ +package environment + +import ( + "sync" + + . "github.com/onsi/gomega" //nolint:revive,stylecheck + "github.com/samber/lo" + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + "k8s.io/api/policy/v1beta1" + storagev1 "k8s.io/api/storage/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/aws/karpenter/pkg/apis/awsnodetemplate/v1alpha1" + "github.com/aws/karpenter/pkg/apis/provisioning/v1alpha5" +) + +var ( + EnvironmentLabelName = "testing.karpenter.sh/environment-name" + CleanableObjects = []client.Object{ + &v1alpha5.Provisioner{}, + &v1.Pod{}, + &appsv1.Deployment{}, + &appsv1.DaemonSet{}, + &v1beta1.PodDisruptionBudget{}, + &v1.PersistentVolumeClaim{}, + &v1.PersistentVolume{}, + &storagev1.StorageClass{}, + &v1alpha1.AWSNodeTemplate{}, + } +) + +func (env *Environment) ExpectCreated(objects ...client.Object) { + for _, object := range objects { + object.SetLabels(lo.Assign(object.GetLabels(), map[string]string{EnvironmentLabelName: env.Options.EnvironmentName})) + Expect(env.Client.Create(env, object)).To(Succeed()) + } +} + +func (env *Environment) ExpectCleaned() { + namespaces := &v1.NamespaceList{} + Expect(env.Client.List(env, namespaces)).To(Succeed()) + wg := sync.WaitGroup{} + for _, object := range CleanableObjects { + for _, namespace := range namespaces.Items { + wg.Add(1) + go func(object client.Object, namespace string) { + Expect(env.Client.DeleteAllOf(env, object, + client.InNamespace(namespace), + client.MatchingLabels(map[string]string{EnvironmentLabelName: env.Options.EnvironmentName}), + )).ToNot(HaveOccurred()) + wg.Done() + }(object, namespace.Name) + } + } + wg.Wait() +} + +func (env *Environment) EventuallyExpectHealthy(pods ...*v1.Pod) { + for _, pod := range pods { + Eventually(func(g Gomega) { + g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(pod), pod)).To(Succeed()) + g.Expect(pod.Status.Conditions).To(ContainElement(And( + HaveField("Type", Equal(v1.PodReady)), + HaveField("Status", Equal(v1.ConditionTrue)), + ))) + }).Should(Succeed()) + } +} diff --git a/test/suites/integration/pipeline.yaml b/test/suites/integration/pipeline.yaml new file mode 100644 index 000000000000..2ec5b2136e01 --- /dev/null +++ b/test/suites/integration/pipeline.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: tekton.dev/v1beta1 +kind: Pipeline +metadata: + name: karpenter-integration-suite +spec: + params: + - name: commit + tasks: + - name: github-action + taskRef: + name: github-action + kind: ClusterTask + params: + - name: repository + value: github.com/aws/karpenter + - name: commit + value: $(params.commit) + - name: command + value: go test ./test/suites/integration -environment-name=$(context.pipelineRun.name) diff --git a/test/suites/integration/suite_test.go b/test/suites/integration/suite_test.go new file mode 100644 index 000000000000..b75a2cf3fb86 --- /dev/null +++ b/test/suites/integration/suite_test.go @@ -0,0 +1,41 @@ +package integration + +import ( + "testing" + + "github.com/aws/karpenter/pkg/apis/provisioning/v1alpha5" + "github.com/aws/karpenter/pkg/cloudprovider/aws/apis/v1alpha1" + "github.com/aws/karpenter/pkg/test" + "github.com/aws/karpenter/test/pkg/environment" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var env *environment.Environment + +func TestAPIs(t *testing.T) { + RegisterFailHandler(Fail) + BeforeSuite(func() { + var err error + env, err = environment.NewEnvironment(t) + Expect(err).ToNot(HaveOccurred()) + }) + RunSpecs(t, "Integration Suite") +} + +var _ = AfterEach(func() { + env.ExpectCleaned() +}) + +var _ = Describe("Sanity Checks", func() { + It("should provision nodes", func() { + provider := test.AWSNodeTemplate(test.AWSNodeTemplateOptions{AWS: v1alpha1.AWS{ + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.Options.EnvironmentName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.Options.EnvironmentName}, + }}) + provisioner := test.Provisioner(test.ProvisionerOptions{ProviderRef: &v1alpha5.ProviderRef{Name: provider.Name}}) + pod := test.Pod() + env.ExpectCreated(provisioner, provider, pod) + env.EventuallyExpectHealthy(pod) + }) +})