From 835a11c01b71be45e501bc40d32cd2ff12ba1df7 Mon Sep 17 00:00:00 2001 From: Jonathan Innis Date: Wed, 31 Aug 2022 09:23:36 -0700 Subject: [PATCH] Update resources to allocatable --- charts/karpenter/crds/karpenter.sh_instancetypes.yaml | 6 +++--- examples/instancetypes/custom-requests.yaml | 2 +- examples/instancetypes/memory-overhead.yaml | 11 +++++++++++ pkg/apis/instancetype/v1alpha1/instancetype.go | 6 +++--- .../instancetype/v1alpha1/instancetype_validation.go | 9 +-------- .../instancetype/v1alpha1/zz_generated.deepcopy.go | 4 ++-- pkg/cloudprovider/aws/instancetypes.go | 7 +++++-- pkg/cloudprovider/fake/instancetype.go | 5 ++--- pkg/scheduling/requirements.go | 7 ------- pkg/test/instancetype.go | 2 +- 10 files changed, 29 insertions(+), 30 deletions(-) create mode 100644 examples/instancetypes/memory-overhead.yaml diff --git a/charts/karpenter/crds/karpenter.sh_instancetypes.yaml b/charts/karpenter/crds/karpenter.sh_instancetypes.yaml index 8d1dc045de37..4f96b5f75a50 100644 --- a/charts/karpenter/crds/karpenter.sh_instancetypes.yaml +++ b/charts/karpenter/crds/karpenter.sh_instancetypes.yaml @@ -42,17 +42,17 @@ spec: for specifying custom values on a per-instance type basis for scheduling and launching of nodes properties: - resources: + allocatable: additionalProperties: anyOf: - type: integer - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: Resources contains a map of allocatable resources for + description: Allocatable contains a map of allocatable resources for the instance type used by the scheduler. This resource list can contain known resources (cpu, memory, etc.) or it may also contain - unknown custom device resources for custom device plugins + unknown custom device resources for custom device plugins. type: object type: object type: object diff --git a/examples/instancetypes/custom-requests.yaml b/examples/instancetypes/custom-requests.yaml index ba26b85d0f0a..620c497173bb 100644 --- a/examples/instancetypes/custom-requests.yaml +++ b/examples/instancetypes/custom-requests.yaml @@ -7,6 +7,6 @@ kind: InstanceType metadata: name: c5.large spec: - resources: + allocatable: hardware.vendor.com/resource: 2 hardware.vendor.com/other-resource: 10 \ No newline at end of file diff --git a/examples/instancetypes/memory-overhead.yaml b/examples/instancetypes/memory-overhead.yaml new file mode 100644 index 000000000000..98e85e79f2ac --- /dev/null +++ b/examples/instancetypes/memory-overhead.yaml @@ -0,0 +1,11 @@ +# This example instance type override will +# override allocatable capacity for c5.large instance types +# provisioned by Karpenter + +apiVersion: karpenter.sh/v1alpha1 +kind: InstanceType +metadata: + name: c5.large +spec: + allocatable: + memory: 3876Mi \ No newline at end of file diff --git a/pkg/apis/instancetype/v1alpha1/instancetype.go b/pkg/apis/instancetype/v1alpha1/instancetype.go index 08c436ec2859..98092f228ac7 100644 --- a/pkg/apis/instancetype/v1alpha1/instancetype.go +++ b/pkg/apis/instancetype/v1alpha1/instancetype.go @@ -23,11 +23,11 @@ import ( // for specifying custom values on a per-instance type basis for scheduling and // launching of nodes type InstanceTypeSpec struct { - // Resources contains a map of allocatable resources for the instance type + // Allocatable contains a map of allocatable resources for the instance type // used by the scheduler. This resource list can contain known resources (cpu, memory, etc.) - // or it may also contain unknown custom device resources for custom device plugins + // or it may also contain unknown custom device resources for custom device plugins. // +optional - Resources v1.ResourceList `json:"resources,omitempty"` + Allocatable v1.ResourceList `json:"allocatable,omitempty"` } // InstanceType is the Schema for the InstanceType API diff --git a/pkg/apis/instancetype/v1alpha1/instancetype_validation.go b/pkg/apis/instancetype/v1alpha1/instancetype_validation.go index 20e1eca1b773..e03a7478d628 100644 --- a/pkg/apis/instancetype/v1alpha1/instancetype_validation.go +++ b/pkg/apis/instancetype/v1alpha1/instancetype_validation.go @@ -17,14 +17,11 @@ package v1alpha1 import ( "context" "fmt" - "strings" - v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/validation" "knative.dev/pkg/apis" "github.com/aws/karpenter/pkg/apis/provisioning/v1alpha5" - "github.com/aws/karpenter/pkg/utils/resources" ) func (in *InstanceType) Validate(_ context.Context) (errs *apis.FieldError) { @@ -55,13 +52,9 @@ func (in *InstanceTypeSpec) validate() (errs *apis.FieldError) { // are part of the well-known requirements set func (in *InstanceTypeSpec) validateResources() (errs *apis.FieldError) { fmt.Println(v1alpha5.WellKnownLabels) - for k := range in.Resources { + for k := range in.Allocatable { if v1alpha5.WellKnownLabels.Has(k.String()) { errs = errs.Also(apis.ErrInvalidValue("cannot be from the set of well-known requirements", fmt.Sprintf("resources[%s]", k))) - } else if resources.WellKnownResourceNames.Has(k.String()) { - errs = errs.Also(apis.ErrInvalidValue("cannot be from the set of well-known resource names", fmt.Sprintf("resources[%s]", k))) - } else if strings.HasPrefix(k.String(), v1.ResourceHugePagesPrefix) || strings.HasPrefix(k.String(), v1.ResourceAttachableVolumesPrefix) { - errs = errs.Also(apis.ErrInvalidValue("cannot use a resource name from a set of well-known prefixes", fmt.Sprintf("resources[%s]", k))) } } return errs diff --git a/pkg/apis/instancetype/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/instancetype/v1alpha1/zz_generated.deepcopy.go index c8444a5940bd..458960ca8a98 100644 --- a/pkg/apis/instancetype/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/instancetype/v1alpha1/zz_generated.deepcopy.go @@ -85,8 +85,8 @@ func (in *InstanceTypeList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *InstanceTypeSpec) DeepCopyInto(out *InstanceTypeSpec) { *out = *in - if in.Resources != nil { - in, out := &in.Resources, &out.Resources + if in.Allocatable != nil { + in, out := &in.Allocatable, &out.Allocatable *out = make(v1.ResourceList, len(*in)) for key, val := range *in { (*out)[key] = val.DeepCopy() diff --git a/pkg/cloudprovider/aws/instancetypes.go b/pkg/cloudprovider/aws/instancetypes.go index 5df79e625aa3..3e00f235cbcc 100644 --- a/pkg/cloudprovider/aws/instancetypes.go +++ b/pkg/cloudprovider/aws/instancetypes.go @@ -27,6 +27,7 @@ import ( instancetypev1alpha1 "github.com/aws/karpenter/pkg/apis/instancetype/v1alpha1" "github.com/aws/karpenter/pkg/apis/provisioning/v1alpha5" "github.com/aws/karpenter/pkg/scheduling" + "github.com/aws/karpenter/pkg/utils/resources" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" @@ -290,9 +291,11 @@ func UnavailableOfferingsCacheKey(instanceType string, zone string, capacityType // pulled from the EC2 APIs. Resources should already be validated when they get here, so we will automatically // add them to the instance type resources and its requirements func mergeInstanceTypeOverrides(it *InstanceType, instanceType instancetypev1alpha1.InstanceType) *InstanceType { - for name, quantity := range instanceType.Spec.Resources { + for name, quantity := range instanceType.Spec.Allocatable { it.resources[name] = quantity - it.requirements.Upsert(scheduling.NewRequirement(name.String(), v1.NodeSelectorOpIn, fmt.Sprint(quantity.Value()))) + if !resources.WellKnownResourceNames.Has(name.String()) && !v1alpha5.WellKnownLabels.Has(name.String()) { + it.requirements.Add(scheduling.NewRequirement(name.String(), v1.NodeSelectorOpIn, fmt.Sprint(quantity.Value()))) + } } return it } diff --git a/pkg/cloudprovider/fake/instancetype.go b/pkg/cloudprovider/fake/instancetype.go index 63787377a2b2..4febd41dc31d 100644 --- a/pkg/cloudprovider/fake/instancetype.go +++ b/pkg/cloudprovider/fake/instancetype.go @@ -21,11 +21,10 @@ import ( "github.com/samber/lo" "github.com/aws/karpenter/pkg/apis/provisioning/v1alpha5" + "github.com/aws/karpenter/pkg/cloudprovider/aws/apis/v1alpha1" "github.com/aws/karpenter/pkg/scheduling" "github.com/aws/karpenter/pkg/utils/resources" - "github.com/aws/karpenter/pkg/cloudprovider/aws/apis/v1alpha1" - v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" utilsets "k8s.io/apimachinery/pkg/util/sets" @@ -218,7 +217,7 @@ func (i *InstanceType) Requirements() scheduling.Requirements { // Add custom resources as requirements values on the node for k, v := range i.options.Resources { if !resources.WellKnownResourceNames.Has(k.String()) && !v1alpha5.WellKnownLabels.Has(k.String()) { - requirements.Upsert(scheduling.NewRequirement(k.String(), v1.NodeSelectorOpIn, fmt.Sprint(v.Value()))) + requirements.Add(scheduling.NewRequirement(k.String(), v1.NodeSelectorOpIn, fmt.Sprint(v.Value()))) } } return requirements diff --git a/pkg/scheduling/requirements.go b/pkg/scheduling/requirements.go index 45ad48ea7477..55f60d0c6eb5 100644 --- a/pkg/scheduling/requirements.go +++ b/pkg/scheduling/requirements.go @@ -87,13 +87,6 @@ func (r Requirements) Add(requirements ...*Requirement) { } } -// Upsert replaces the requirements set with all existing requirements -func (r Requirements) Upsert(requirements ...*Requirement) { - for _, requirement := range requirements { - r[requirement.Key] = requirement - } -} - // Keys returns unique set of the label keys from the requirements func (r Requirements) Keys() sets.String { keys := sets.NewString() diff --git a/pkg/test/instancetype.go b/pkg/test/instancetype.go index 9fca3e31acf0..517de4430787 100644 --- a/pkg/test/instancetype.go +++ b/pkg/test/instancetype.go @@ -51,7 +51,7 @@ func InstanceType(name string, overrides ...InstanceTypeOptions) *v1alpha1.Insta instanceType := &v1alpha1.InstanceType{ ObjectMeta: ObjectMeta(options.ObjectMeta), Spec: v1alpha1.InstanceTypeSpec{ - Resources: options.Resources, + Allocatable: options.Resources, }, } instanceType.SetDefaults(context.Background())