feat: Introduce InstanceType CRD for overriding instance type resources

.golangci.yaml

@@ -36,7 +36,8 @@ linters-settings:
     check-shadowing: true
   misspell:
     locale: US
-    ignore-words: []
+    ignore-words:
+      - Karpenter
   goimports:
     local-prefixes: github.com/aws/karpenter
   goheader:

Makefile

@@ -49,7 +49,7 @@ battletest: ## Run randomized, racing, code coveraged, tests
 
 e2etests: ## Run the e2e suite against your local cluster
 	go clean -testcache
-	go test -p 1 -timeout 180m -v ./test/suites/... -run=${TEST_FILTER}
+	CLUSTER_NAME=${CLUSTER_NAME} go test -p 1 -timeout 180m -v ./test/suites/... -run=${TEST_FILTER}
 
 benchmark:
 	go test -tags=test_performance -run=NoTests -bench=. ./...

charts/karpenter/crds/karpenter.sh_instancetypes.yaml

@@ -0,0 +1,68 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.8.0
+  creationTimestamp: null
+  name: instancetypes.karpenter.sh
+spec:
+  group: karpenter.sh
+  names:
+    categories:
+    - karpenter
+    kind: InstanceType
+    listKind: InstanceTypeList
+    plural: instancetypes
+    shortNames:
+    - it
+    - its
+    singular: instancetype
+  scope: Cluster
+  versions:
+  - name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        description: InstanceType is the Schema for the InstanceType API
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: InstanceTypeSpec is the instance type setting override specification
+              for specifying custom values on a per-instance type basis for scheduling
+              and launching of nodes
+            properties:
+              capacity:
+                additionalProperties:
+                  anyOf:
+                  - type: integer
+                  - type: string
+                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                  x-kubernetes-int-or-string: true
+                description: Capacity contains a map of allocatable resources for
+                  the instance type used by the scheduler. This resource list can
+                  contain known resources (cpu, memory, etc.) or it may also contain
+                  unknown custom device resources for custom device plugins.
+                type: object
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []

charts/karpenter/templates/clusterrole.yaml

@@ -11,7 +11,7 @@ metadata:
 rules:
   # Read
   - apiGroups: ["karpenter.sh"]
-    resources: ["provisioners", "provisioners/status"]
+    resources: ["provisioners", "provisioners/status", "instancetypes", "instancetypes/status"]
     verbs: ["get", "list", "watch"]
   - apiGroups: ["karpenter.k8s.aws"]
     resources: ["awsnodetemplates"]

examples/instancetypes/custom-requests.yaml

@@ -0,0 +1,12 @@
+# This example instance type override will
+# override allocatable capacity for c5.large instance types
+# provisioned by Karpenter
+
+apiVersion: karpenter.sh/v1alpha1
+kind: InstanceType
+metadata:
+  name: c5.large
+spec:
+  allocatable:
+    hardware.vendor.com/resource: 2
+    hardware.vendor.com/other-resource: 10

examples/instancetypes/memory-overhead.yaml

@@ -0,0 +1,11 @@
+# This example instance type override will
+# override allocatable capacity for c5.large instance types
+# provisioned by Karpenter
+
+apiVersion: karpenter.sh/v1alpha1
+kind: InstanceType
+metadata:
+  name: c5.large
+spec:
+  allocatable:
+    memory: 3876Mi

hack/docs/instancetypes_gen_docs.go

@@ -25,17 +25,18 @@ import (
 	"sort"
 	"strings"
 
+	"github.com/samber/lo"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	"k8s.io/apimachinery/pkg/util/sets"
+
 	"github.com/aws/karpenter/pkg/apis/provisioning/v1alpha5"
 	"github.com/aws/karpenter/pkg/cloudprovider"
 	"github.com/aws/karpenter/pkg/cloudprovider/aws"
 	"github.com/aws/karpenter/pkg/cloudprovider/aws/apis/v1alpha1"
 	"github.com/aws/karpenter/pkg/utils/injection"
 	"github.com/aws/karpenter/pkg/utils/options"
 	"github.com/aws/karpenter/pkg/utils/resources"
-	"github.com/samber/lo"
-	v1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/api/resource"
-	"k8s.io/apimachinery/pkg/util/sets"
 )
 
 func main() {
@@ -98,7 +99,7 @@ linkTitle: "Instance Types"
 weight: 100
 
 description: >
-  Evaluate Instance Type Resources
+  Evaluate Instance Type Capacity
 ---
 `)
 	fmt.Fprintln(f, "<!-- this document is generated from hack/docs/instancetypes_gen_docs.go -->")
@@ -180,7 +181,7 @@ below are the resources available with some assumptions and after the instance o
 					fmt.Fprintf(f, " |%s|%s|\n", label, req.Values()[0])
 				}
 			}
-			fmt.Fprintln(f, "#### Resources")
+			fmt.Fprintln(f, "#### Capacity")
 			fmt.Fprintln(f, " | Resource | Quantity |")
 			fmt.Fprintln(f, " |--|--|")
 			for _, resourceName := range resourceNames {

pkg/apis/apis.go

@@ -20,21 +20,24 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"knative.dev/pkg/webhook/resourcesemantics"
 
-	"github.com/aws/karpenter/pkg/apis/awsnodetemplate/v1alpha1"
+	awsv1alpha1 "github.com/aws/karpenter/pkg/apis/awsnodetemplate/v1alpha1"
+	instancetypev1alpha1 "github.com/aws/karpenter/pkg/apis/instancetype/v1alpha1"
 	"github.com/aws/karpenter/pkg/apis/provisioning/v1alpha5"
 )
 
 var (
 	// Builder includes all types within the apis package
 	Builder = runtime.NewSchemeBuilder(
 		v1alpha5.SchemeBuilder.AddToScheme,
-		v1alpha1.SchemeBuilder.AddToScheme,
+		awsv1alpha1.SchemeBuilder.AddToScheme,
+		instancetypev1alpha1.SchemeBuilder.AddToScheme,
 	)
 	// AddToScheme may be used to add all resources defined in the project to a Scheme
 	AddToScheme = Builder.AddToScheme
 	// Resources defined in the project
 	Resources = map[schema.GroupVersionKind]resourcesemantics.GenericCRD{
-		v1alpha5.SchemeGroupVersion.WithKind("Provisioner"):     &v1alpha5.Provisioner{},
-		v1alpha1.SchemeGroupVersion.WithKind("AWSNodeTemplate"): &v1alpha1.AWSNodeTemplate{},
+		v1alpha5.SchemeGroupVersion.WithKind("Provisioner"):              &v1alpha5.Provisioner{},
+		awsv1alpha1.SchemeGroupVersion.WithKind("AWSNodeTemplate"):       &awsv1alpha1.AWSNodeTemplate{},
+		instancetypev1alpha1.SchemeGroupVersion.WithKind("InstanceType"): &instancetypev1alpha1.InstanceType{},
 	}
 )

pkg/apis/instancetype/v1alpha1/doc.go

@@ -0,0 +1,19 @@
+/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// +k8s:openapi-gen=true
+// +k8s:deepcopy-gen=package,register
+// +k8s:defaulter-gen=TypeMeta
+// +groupName=karpenter.sh
+package v1alpha1 // doc.go is discovered by codegen

pkg/apis/instancetype/v1alpha1/instancetype.go

@@ -0,0 +1,50 @@
+/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// InstanceTypeSpec is the instance type setting override specification
+// for specifying custom values on a per-instance type basis for scheduling and
+// launching of nodes
+type InstanceTypeSpec struct {
+	// Capacity contains a map of allocatable resources for the instance type
+	// used by the scheduler. This resource list can contain known resources (cpu, memory, etc.)
+	// or it may also contain unknown custom device resources for custom device plugins.
+	// +optional
+	Capacity v1.ResourceList `json:"capacity,omitempty"`
+}
+
+// InstanceType is the Schema for the InstanceType API
+// +kubebuilder:object:root=true
+// +kubebuilder:resource:path=instancetypes,scope=Cluster,categories=karpenter,shortName={it,its}
+// +kubebuilder:subresource:status
+type InstanceType struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec InstanceTypeSpec `json:"spec,omitempty"`
+}
+
+// InstanceTypeList contains a list of InstanceType
+// +kubebuilder:object:root=true
+type InstanceTypeList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []InstanceType `json:"items"`
+}

pkg/apis/instancetype/v1alpha1/instancetype_defaults.go

@@ -0,0 +1,23 @@
+/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	"context"
+)
+
+// SetDefaults for the InstanceType
+func (in *InstanceType) SetDefaults(_ context.Context) {
+}

pkg/apis/instancetype/v1alpha1/instancetype_validation.go

@@ -0,0 +1,61 @@
+/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	"context"
+	"fmt"
+
+	"k8s.io/apimachinery/pkg/api/validation"
+	"knative.dev/pkg/apis"
+
+	"github.com/aws/karpenter/pkg/apis/provisioning/v1alpha5"
+)
+
+func (in *InstanceType) Validate(_ context.Context) (errs *apis.FieldError) {
+	return errs.Also(
+		in.validateMetadata().ViaField("metadata"),
+		in.Spec.validate().ViaField("spec"),
+	)
+}
+
+func (in *InstanceType) validateMetadata() *apis.FieldError {
+	msgs := validation.NameIsDNSSubdomain(in.Name, false)
+	if len(msgs) > 0 {
+		return &apis.FieldError{
+			Message: fmt.Sprintf("not a DNS subdomain name: %v", msgs),
+			Paths:   []string{"name"},
+		}
+	}
+	return nil
+}
+
+func (in *InstanceTypeSpec) validate() (errs *apis.FieldError) {
+	return errs.Also(
+		in.validateResources(),
+	)
+}
+
+// validateResources should reject the InstanceType if any of the key values
+// are part of the well-known requirements set
+func (in *InstanceTypeSpec) validateResources() (errs *apis.FieldError) {
+	fmt.Println(v1alpha5.WellKnownLabels)
+	for k := range in.Capacity {
+		if v1alpha5.WellKnownLabels.Has(k.String()) {
+			errs = errs.Also(apis.ErrInvalidValue("cannot be from the set of well-known requirements", fmt.Sprintf("resources[%s]", k)))
+		}
+	}
+	return errs
+}

pkg/apis/instancetype/v1alpha1/register.go

@@ -0,0 +1,42 @@
+/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"knative.dev/pkg/apis"
+)
+
+var (
+	Group              = "karpenter.sh"
+	SchemeGroupVersion = schema.GroupVersion{Group: Group, Version: "v1alpha1"}
+	SchemeBuilder      = runtime.NewSchemeBuilder(func(scheme *runtime.Scheme) error {
+		scheme.AddKnownTypes(SchemeGroupVersion,
+			&InstanceType{},
+			&InstanceTypeList{},
+		)
+		metav1.AddToGroupVersion(scheme, SchemeGroupVersion)
+		return nil
+	})
+)
+
+const (
+	// Active is a condition implemented by all resources. It indicates that the
+	// controller is able to take actions: it's correctly configured, can make
+	// necessary API calls, and isn't disabled.
+	Active apis.ConditionType = "Active"
+)