From 9f750dc9d42cf0ddee3763f9bd2e6c111bc0bef9 Mon Sep 17 00:00:00 2001 From: Alex Leites <18728999+tallaxes@users.noreply.github.com> Date: Tue, 3 Dec 2024 02:42:40 +0000 Subject: [PATCH 01/19] chore: update sigs.k8s.io/karpenter to v1.1.0 --- .../templates/karpenter.sh_nodeclaims.yaml | 443 +------------- .../templates/karpenter.sh_nodepools.yaml | 576 +----------------- go.mod | 72 +-- go.sum | 138 +++-- pkg/apis/crds/karpenter.sh_nodeclaims.yaml | 443 +------------- pkg/apis/crds/karpenter.sh_nodepools.yaml | 576 +----------------- 6 files changed, 154 insertions(+), 2094 deletions(-) diff --git a/charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml b/charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml index e12564c25..b12e4f7aa 100644 --- a/charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml +++ b/charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml @@ -88,12 +88,21 @@ spec: description: API version of the referent pattern: ^[^/]*$ type: string + x-kubernetes-validations: + - message: group may not be empty + rule: self != '' kind: description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' type: string + x-kubernetes-validations: + - message: kind may not be empty + rule: self != '' name: description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' type: string + x-kubernetes-validations: + - message: name may not be empty + rule: self != '' required: - group - kind @@ -380,440 +389,6 @@ spec: storage: true subresources: status: {} - - additionalPrinterColumns: - - jsonPath: .metadata.labels.node\.kubernetes\.io/instance-type - name: Type - type: string - - jsonPath: .metadata.labels.topology\.kubernetes\.io/zone - name: Zone - type: string - - jsonPath: .status.nodeName - name: Node - type: string - - jsonPath: .status.conditions[?(@.type=="Ready")].status - name: Ready - type: string - - jsonPath: .metadata.creationTimestamp - name: Age - type: date - - jsonPath: .metadata.labels.karpenter\.sh/capacity-type - name: Capacity - priority: 1 - type: string - - jsonPath: .metadata.labels.karpenter\.sh/nodepool - name: NodePool - priority: 1 - type: string - - jsonPath: .spec.nodeClassRef.name - name: NodeClass - priority: 1 - type: string - name: v1beta1 - schema: - openAPIV3Schema: - description: NodeClaim is the Schema for the NodeClaims API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: NodeClaimSpec describes the desired state of the NodeClaim - properties: - kubelet: - description: |- - Kubelet defines args to be used when configuring kubelet on provisioned nodes. - They are a subset of the upstream types, recognizing not all options may be supported. - Wherever possible, the types and names should reflect the upstream kubelet types. - properties: - clusterDNS: - description: |- - clusterDNS is a list of IP addresses for the cluster DNS server. - Note that not all providers may use all addresses. - items: - type: string - type: array - cpuCFSQuota: - description: CPUCFSQuota enables CPU CFS quota enforcement for containers that specify CPU limits. - type: boolean - evictionHard: - additionalProperties: - type: string - pattern: ^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$ - description: EvictionHard is the map of signal names to quantities that define hard eviction thresholds - type: object - x-kubernetes-validations: - - message: valid keys for evictionHard are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] - rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) - evictionMaxPodGracePeriod: - description: |- - EvictionMaxPodGracePeriod is the maximum allowed grace period (in seconds) to use when terminating pods in - response to soft eviction thresholds being met. - format: int32 - type: integer - evictionSoft: - additionalProperties: - type: string - pattern: ^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$ - description: EvictionSoft is the map of signal names to quantities that define soft eviction thresholds - type: object - x-kubernetes-validations: - - message: valid keys for evictionSoft are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] - rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) - evictionSoftGracePeriod: - additionalProperties: - type: string - description: EvictionSoftGracePeriod is the map of signal names to quantities that define grace periods for each eviction signal - type: object - x-kubernetes-validations: - - message: valid keys for evictionSoftGracePeriod are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] - rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) - imageGCHighThresholdPercent: - description: |- - ImageGCHighThresholdPercent is the percent of disk usage after which image - garbage collection is always run. The percent is calculated by dividing this - field value by 100, so this field must be between 0 and 100, inclusive. - When specified, the value must be greater than ImageGCLowThresholdPercent. - format: int32 - maximum: 100 - minimum: 0 - type: integer - imageGCLowThresholdPercent: - description: |- - ImageGCLowThresholdPercent is the percent of disk usage before which image - garbage collection is never run. Lowest disk usage to garbage collect to. - The percent is calculated by dividing this field value by 100, - so the field value must be between 0 and 100, inclusive. - When specified, the value must be less than imageGCHighThresholdPercent - format: int32 - maximum: 100 - minimum: 0 - type: integer - kubeReserved: - additionalProperties: - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - description: KubeReserved contains resources reserved for Kubernetes system components. - type: object - x-kubernetes-validations: - - message: valid keys for kubeReserved are ['cpu','memory','ephemeral-storage','pid'] - rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid') - - message: kubeReserved value cannot be a negative resource quantity - rule: self.all(x, !self[x].startsWith('-')) - maxPods: - description: |- - MaxPods is an override for the maximum number of pods that can run on - a worker node instance. - format: int32 - minimum: 0 - type: integer - podsPerCore: - description: |- - PodsPerCore is an override for the number of pods that can run on a worker node - instance based on the number of cpu cores. This value cannot exceed MaxPods, so, if - MaxPods is a lower value, that value will be used. - format: int32 - minimum: 0 - type: integer - systemReserved: - additionalProperties: - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - description: SystemReserved contains resources reserved for OS system daemons and kernel memory. - type: object - x-kubernetes-validations: - - message: valid keys for systemReserved are ['cpu','memory','ephemeral-storage','pid'] - rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid') - - message: systemReserved value cannot be a negative resource quantity - rule: self.all(x, !self[x].startsWith('-')) - type: object - x-kubernetes-validations: - - message: imageGCHighThresholdPercent must be greater than imageGCLowThresholdPercent - rule: 'has(self.imageGCHighThresholdPercent) && has(self.imageGCLowThresholdPercent) ? self.imageGCHighThresholdPercent > self.imageGCLowThresholdPercent : true' - - message: evictionSoft OwnerKey does not have a matching evictionSoftGracePeriod - rule: has(self.evictionSoft) ? self.evictionSoft.all(e, (e in self.evictionSoftGracePeriod)):true - - message: evictionSoftGracePeriod OwnerKey does not have a matching evictionSoft - rule: has(self.evictionSoftGracePeriod) ? self.evictionSoftGracePeriod.all(e, (e in self.evictionSoft)):true - nodeClassRef: - description: NodeClassRef is a reference to an object that defines provider specific configuration - properties: - apiVersion: - description: API version of the referent - type: string - kind: - description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' - type: string - name: - description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' - type: string - required: - - name - type: object - requirements: - description: Requirements are layered with GetLabels and applied to every node. - items: - description: |- - A node selector requirement with min values is a selector that contains values, a key, an operator that relates the key and values - and minValues that represent the requirement to have at least that many values. - properties: - key: - description: The label key that the selector applies to. - type: string - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ - x-kubernetes-validations: - - message: label domain "kubernetes.io" is restricted - rule: self in ["beta.kubernetes.io/instance-type", "failure-domain.beta.kubernetes.io/region", "beta.kubernetes.io/os", "beta.kubernetes.io/arch", "failure-domain.beta.kubernetes.io/zone", "topology.kubernetes.io/zone", "topology.kubernetes.io/region", "node.kubernetes.io/instance-type", "kubernetes.io/arch", "kubernetes.io/os", "node.kubernetes.io/windows-build"] || self.find("^([^/]+)").endsWith("node.kubernetes.io") || self.find("^([^/]+)").endsWith("node-restriction.kubernetes.io") || !self.find("^([^/]+)").endsWith("kubernetes.io") - - message: label domain "k8s.io" is restricted - rule: self.find("^([^/]+)").endsWith("kops.k8s.io") || !self.find("^([^/]+)").endsWith("k8s.io") - - message: label domain "karpenter.sh" is restricted - rule: self in ["karpenter.sh/capacity-type", "karpenter.sh/nodepool"] || !self.find("^([^/]+)").endsWith("karpenter.sh") - - message: label "kubernetes.io/hostname" is restricted - rule: self != "kubernetes.io/hostname" - minValues: - description: |- - This field is ALPHA and can be dropped or replaced at any time - MinValues is the minimum number of unique values required to define the flexibility of the specific requirement. - maximum: 50 - minimum: 1 - type: integer - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - enum: - - In - - NotIn - - Exists - - DoesNotExist - - Gt - - Lt - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - maxLength: 63 - pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ - required: - - key - - operator - type: object - maxItems: 100 - type: array - x-kubernetes-validations: - - message: requirements with operator 'In' must have a value defined - rule: 'self.all(x, x.operator == ''In'' ? x.values.size() != 0 : true)' - - message: requirements operator 'Gt' or 'Lt' must have a single positive integer value - rule: 'self.all(x, (x.operator == ''Gt'' || x.operator == ''Lt'') ? (x.values.size() == 1 && int(x.values[0]) >= 0) : true)' - - message: requirements with 'minValues' must have at least that many values specified in the 'values' field - rule: 'self.all(x, (x.operator == ''In'' && has(x.minValues)) ? x.values.size() >= x.minValues : true)' - resources: - description: Resources models the resource requirements for the NodeClaim to launch - properties: - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: Requests describes the minimum required resources for the NodeClaim to launch - type: object - type: object - startupTaints: - description: |- - StartupTaints are taints that are applied to nodes upon startup which are expected to be removed automatically - within a short period of time, typically by a DaemonSet that tolerates the taint. These are commonly used by - daemonsets to allow initialization and enforce startup ordering. StartupTaints are ignored for provisioning - purposes in that pods are not required to tolerate a StartupTaint in order to have nodes provisioned for them. - items: - description: |- - The node this Taint is attached to has the "effect" on - any pod that does not tolerate the Taint. - properties: - effect: - description: |- - Required. The effect of the taint on pods - that do not tolerate the taint. - Valid effects are NoSchedule, PreferNoSchedule and NoExecute. - type: string - enum: - - NoSchedule - - PreferNoSchedule - - NoExecute - key: - description: Required. The taint key to be applied to a node. - type: string - minLength: 1 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ - timeAdded: - description: |- - TimeAdded represents the time at which the taint was added. - It is only written for NoExecute taints. - format: date-time - type: string - value: - description: The taint value corresponding to the taint key. - type: string - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ - required: - - effect - - key - type: object - type: array - taints: - description: Taints will be applied to the NodeClaim's node. - items: - description: |- - The node this Taint is attached to has the "effect" on - any pod that does not tolerate the Taint. - properties: - effect: - description: |- - Required. The effect of the taint on pods - that do not tolerate the taint. - Valid effects are NoSchedule, PreferNoSchedule and NoExecute. - type: string - enum: - - NoSchedule - - PreferNoSchedule - - NoExecute - key: - description: Required. The taint key to be applied to a node. - type: string - minLength: 1 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ - timeAdded: - description: |- - TimeAdded represents the time at which the taint was added. - It is only written for NoExecute taints. - format: date-time - type: string - value: - description: The taint value corresponding to the taint key. - type: string - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ - required: - - effect - - key - type: object - type: array - required: - - nodeClassRef - - requirements - type: object - status: - description: NodeClaimStatus defines the observed state of NodeClaim - properties: - allocatable: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: Allocatable is the estimated allocatable capacity of the node - type: object - capacity: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: Capacity is the estimated full capacity of the node - type: object - conditions: - description: Conditions contains signals for health and readiness - items: - description: Condition aliases the upstream type and adds additional helper methods - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - pattern: ^([A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?|)$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - status - - type - type: object - type: array - imageID: - description: ImageID is an identifier for the image that runs on the node - type: string - nodeName: - description: NodeName is the name of the corresponding node object - type: string - providerID: - description: ProviderID of the corresponding node object - type: string - type: object - required: - - spec - type: object - served: true - storage: false - subresources: - status: {} {{- if .Values.webhook.enabled }} conversion: strategy: Webhook diff --git a/charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml b/charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml index d6f851bb7..d8a0c9a31 100644 --- a/charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml +++ b/charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml @@ -231,17 +231,31 @@ spec: description: API version of the referent pattern: ^[^/]*$ type: string + x-kubernetes-validations: + - message: group may not be empty + rule: self != '' kind: description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' type: string + x-kubernetes-validations: + - message: kind may not be empty + rule: self != '' name: description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' type: string + x-kubernetes-validations: + - message: name may not be empty + rule: self != '' required: - group - kind - name type: object + x-kubernetes-validations: + - message: nodeClassRef.group is immutable + rule: self.group == oldSelf.group + - message: nodeClassRef.kind is immutable + rule: self.kind == oldSelf.kind requirements: description: Requirements are layered with GetLabels and applied to every node. items: @@ -503,568 +517,6 @@ spec: storage: true subresources: status: {} - - additionalPrinterColumns: - - jsonPath: .spec.template.spec.nodeClassRef.name - name: NodeClass - type: string - - jsonPath: .spec.weight - name: Weight - priority: 1 - type: string - name: v1beta1 - schema: - openAPIV3Schema: - description: NodePool is the Schema for the NodePools API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: |- - NodePoolSpec is the top level nodepool specification. Nodepools - launch nodes in response to pods that are unschedulable. A single nodepool - is capable of managing a diverse set of nodes. Node properties are determined - from a combination of nodepool and pod scheduling constraints. - properties: - disruption: - default: - consolidationPolicy: WhenUnderutilized - expireAfter: 720h - description: Disruption contains the parameters that relate to Karpenter's disruption logic - properties: - budgets: - default: - - nodes: 10% - description: |- - Budgets is a list of Budgets. - If there are multiple active budgets, Karpenter uses - the most restrictive value. If left undefined, - this will default to one budget with a value to 10%. - items: - description: |- - Budget defines when Karpenter will restrict the - number of Node Claims that can be terminating simultaneously. - properties: - duration: - description: |- - Duration determines how long a Budget is active since each Schedule hit. - Only minutes and hours are accepted, as cron does not work in seconds. - If omitted, the budget is always active. - This is required if Schedule is set. - This regex has an optional 0s at the end since the duration.String() always adds - a 0s at the end. - pattern: ^((([0-9]+(h|m))|([0-9]+h[0-9]+m))(0s)?)$ - type: string - nodes: - default: 10% - description: |- - Nodes dictates the maximum number of NodeClaims owned by this NodePool - that can be terminating at once. This is calculated by counting nodes that - have a deletion timestamp set, or are actively being deleted by Karpenter. - This field is required when specifying a budget. - This cannot be of type intstr.IntOrString since kubebuilder doesn't support pattern - checking for int nodes for IntOrString nodes. - Ref: https://github.com/kubernetes-sigs/controller-tools/blob/55efe4be40394a288216dab63156b0a64fb82929/pkg/crd/markers/validation.go#L379-L388 - pattern: ^((100|[0-9]{1,2})%|[0-9]+)$ - type: string - schedule: - description: |- - Schedule specifies when a budget begins being active, following - the upstream cronjob syntax. If omitted, the budget is always active. - Timezones are not supported. - This field is required if Duration is set. - pattern: ^(@(annually|yearly|monthly|weekly|daily|midnight|hourly))|((.+)\s(.+)\s(.+)\s(.+)\s(.+))$ - type: string - required: - - nodes - type: object - maxItems: 50 - type: array - x-kubernetes-validations: - - message: '''schedule'' must be set with ''duration''' - rule: self.all(x, has(x.schedule) == has(x.duration)) - consolidateAfter: - description: |- - ConsolidateAfter is the duration the controller will wait - before attempting to terminate nodes that are underutilized. - Refer to ConsolidationPolicy for how underutilization is considered. - pattern: ^(([0-9]+(s|m|h))+)|(Never)$ - type: string - consolidationPolicy: - default: WhenUnderutilized - description: |- - ConsolidationPolicy describes which nodes Karpenter can disrupt through its consolidation - algorithm. This policy defaults to "WhenUnderutilized" if not specified - enum: - - WhenEmpty - - WhenUnderutilized - type: string - expireAfter: - default: 720h - description: |- - ExpireAfter is the duration the controller will wait - before terminating a node, measured from when the node is created. This - is useful to implement features like eventually consistent node upgrade, - memory leak protection, and disruption testing. - pattern: ^(([0-9]+(s|m|h))+)|(Never)$ - type: string - type: object - x-kubernetes-validations: - - message: consolidateAfter cannot be combined with consolidationPolicy=WhenUnderutilized - rule: 'has(self.consolidateAfter) ? self.consolidationPolicy != ''WhenUnderutilized'' || self.consolidateAfter == ''Never'' : true' - - message: consolidateAfter must be specified with consolidationPolicy=WhenEmpty - rule: 'self.consolidationPolicy == ''WhenEmpty'' ? has(self.consolidateAfter) : true' - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: Limits define a set of bounds for provisioning capacity. - type: object - template: - description: |- - Template contains the template of possibilities for the provisioning logic to launch a NodeClaim with. - NodeClaims launched from this NodePool will often be further constrained than the template specifies. - properties: - metadata: - properties: - annotations: - additionalProperties: - type: string - description: |- - Annotations is an unstructured key value map stored with a resource that may be - set by external tools to store and retrieve arbitrary metadata. They are not - queryable and should be preserved when modifying objects. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations - type: object - labels: - additionalProperties: - type: string - maxLength: 63 - pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ - description: |- - Map of string keys and values that can be used to organize and categorize - (scope and select) objects. May match selectors of replication controllers - and services. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels - type: object - maxProperties: 100 - x-kubernetes-validations: - - message: label domain "kubernetes.io" is restricted - rule: self.all(x, x in ["beta.kubernetes.io/instance-type", "failure-domain.beta.kubernetes.io/region", "beta.kubernetes.io/os", "beta.kubernetes.io/arch", "failure-domain.beta.kubernetes.io/zone", "topology.kubernetes.io/region", "kubernetes.io/arch", "kubernetes.io/os", "node.kubernetes.io/windows-build"] || x.find("^([^/]+)").endsWith("node.kubernetes.io") || x.find("^([^/]+)").endsWith("node-restriction.kubernetes.io") || !x.find("^([^/]+)").endsWith("kubernetes.io")) - - message: label domain "k8s.io" is restricted - rule: self.all(x, x.find("^([^/]+)").endsWith("kops.k8s.io") || !x.find("^([^/]+)").endsWith("k8s.io")) - - message: label domain "karpenter.sh" is restricted - rule: self.all(x, x in ["karpenter.sh/capacity-type", "karpenter.sh/nodepool"] || !x.find("^([^/]+)").endsWith("karpenter.sh")) - - message: label "karpenter.sh/nodepool" is restricted - rule: self.all(x, x != "karpenter.sh/nodepool") - - message: label "kubernetes.io/hostname" is restricted - rule: self.all(x, x != "kubernetes.io/hostname") - type: object - spec: - description: NodeClaimSpec describes the desired state of the NodeClaim - properties: - kubelet: - description: |- - Kubelet defines args to be used when configuring kubelet on provisioned nodes. - They are a subset of the upstream types, recognizing not all options may be supported. - Wherever possible, the types and names should reflect the upstream kubelet types. - properties: - clusterDNS: - description: |- - clusterDNS is a list of IP addresses for the cluster DNS server. - Note that not all providers may use all addresses. - items: - type: string - type: array - cpuCFSQuota: - description: CPUCFSQuota enables CPU CFS quota enforcement for containers that specify CPU limits. - type: boolean - evictionHard: - additionalProperties: - type: string - pattern: ^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$ - description: EvictionHard is the map of signal names to quantities that define hard eviction thresholds - type: object - x-kubernetes-validations: - - message: valid keys for evictionHard are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] - rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) - evictionMaxPodGracePeriod: - description: |- - EvictionMaxPodGracePeriod is the maximum allowed grace period (in seconds) to use when terminating pods in - response to soft eviction thresholds being met. - format: int32 - type: integer - evictionSoft: - additionalProperties: - type: string - pattern: ^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$ - description: EvictionSoft is the map of signal names to quantities that define soft eviction thresholds - type: object - x-kubernetes-validations: - - message: valid keys for evictionSoft are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] - rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) - evictionSoftGracePeriod: - additionalProperties: - type: string - description: EvictionSoftGracePeriod is the map of signal names to quantities that define grace periods for each eviction signal - type: object - x-kubernetes-validations: - - message: valid keys for evictionSoftGracePeriod are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] - rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) - imageGCHighThresholdPercent: - description: |- - ImageGCHighThresholdPercent is the percent of disk usage after which image - garbage collection is always run. The percent is calculated by dividing this - field value by 100, so this field must be between 0 and 100, inclusive. - When specified, the value must be greater than ImageGCLowThresholdPercent. - format: int32 - maximum: 100 - minimum: 0 - type: integer - imageGCLowThresholdPercent: - description: |- - ImageGCLowThresholdPercent is the percent of disk usage before which image - garbage collection is never run. Lowest disk usage to garbage collect to. - The percent is calculated by dividing this field value by 100, - so the field value must be between 0 and 100, inclusive. - When specified, the value must be less than imageGCHighThresholdPercent - format: int32 - maximum: 100 - minimum: 0 - type: integer - kubeReserved: - additionalProperties: - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - description: KubeReserved contains resources reserved for Kubernetes system components. - type: object - x-kubernetes-validations: - - message: valid keys for kubeReserved are ['cpu','memory','ephemeral-storage','pid'] - rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid') - - message: kubeReserved value cannot be a negative resource quantity - rule: self.all(x, !self[x].startsWith('-')) - maxPods: - description: |- - MaxPods is an override for the maximum number of pods that can run on - a worker node instance. - format: int32 - minimum: 0 - type: integer - podsPerCore: - description: |- - PodsPerCore is an override for the number of pods that can run on a worker node - instance based on the number of cpu cores. This value cannot exceed MaxPods, so, if - MaxPods is a lower value, that value will be used. - format: int32 - minimum: 0 - type: integer - systemReserved: - additionalProperties: - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - description: SystemReserved contains resources reserved for OS system daemons and kernel memory. - type: object - x-kubernetes-validations: - - message: valid keys for systemReserved are ['cpu','memory','ephemeral-storage','pid'] - rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid') - - message: systemReserved value cannot be a negative resource quantity - rule: self.all(x, !self[x].startsWith('-')) - type: object - x-kubernetes-validations: - - message: imageGCHighThresholdPercent must be greater than imageGCLowThresholdPercent - rule: 'has(self.imageGCHighThresholdPercent) && has(self.imageGCLowThresholdPercent) ? self.imageGCHighThresholdPercent > self.imageGCLowThresholdPercent : true' - - message: evictionSoft OwnerKey does not have a matching evictionSoftGracePeriod - rule: has(self.evictionSoft) ? self.evictionSoft.all(e, (e in self.evictionSoftGracePeriod)):true - - message: evictionSoftGracePeriod OwnerKey does not have a matching evictionSoft - rule: has(self.evictionSoftGracePeriod) ? self.evictionSoftGracePeriod.all(e, (e in self.evictionSoft)):true - nodeClassRef: - description: NodeClassRef is a reference to an object that defines provider specific configuration - properties: - apiVersion: - description: API version of the referent - type: string - kind: - description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' - type: string - name: - description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' - type: string - required: - - name - type: object - requirements: - description: Requirements are layered with GetLabels and applied to every node. - items: - description: |- - A node selector requirement with min values is a selector that contains values, a key, an operator that relates the key and values - and minValues that represent the requirement to have at least that many values. - properties: - key: - description: The label key that the selector applies to. - type: string - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ - x-kubernetes-validations: - - message: label domain "kubernetes.io" is restricted - rule: self in ["beta.kubernetes.io/instance-type", "failure-domain.beta.kubernetes.io/region", "beta.kubernetes.io/os", "beta.kubernetes.io/arch", "failure-domain.beta.kubernetes.io/zone", "topology.kubernetes.io/region", "node.kubernetes.io/instance-type", "kubernetes.io/arch", "kubernetes.io/os", "node.kubernetes.io/windows-build"] || self.find("^([^/]+)").endsWith("node.kubernetes.io") || self.find("^([^/]+)").endsWith("node-restriction.kubernetes.io") || !self.find("^([^/]+)").endsWith("kubernetes.io") - - message: label domain "k8s.io" is restricted - rule: self.find("^([^/]+)").endsWith("kops.k8s.io") || !self.find("^([^/]+)").endsWith("k8s.io") - - message: label domain "karpenter.sh" is restricted - rule: self in ["karpenter.sh/capacity-type", "karpenter.sh/nodepool"] || !self.find("^([^/]+)").endsWith("karpenter.sh") - - message: label "karpenter.sh/nodepool" is restricted - rule: self != "karpenter.sh/nodepool" - - message: label "kubernetes.io/hostname" is restricted - rule: self != "kubernetes.io/hostname" - minValues: - description: |- - This field is ALPHA and can be dropped or replaced at any time - MinValues is the minimum number of unique values required to define the flexibility of the specific requirement. - maximum: 50 - minimum: 1 - type: integer - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - enum: - - In - - NotIn - - Exists - - DoesNotExist - - Gt - - Lt - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - maxLength: 63 - pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ - required: - - key - - operator - type: object - maxItems: 100 - type: array - x-kubernetes-validations: - - message: requirements with operator 'In' must have a value defined - rule: 'self.all(x, x.operator == ''In'' ? x.values.size() != 0 : true)' - - message: requirements operator 'Gt' or 'Lt' must have a single positive integer value - rule: 'self.all(x, (x.operator == ''Gt'' || x.operator == ''Lt'') ? (x.values.size() == 1 && int(x.values[0]) >= 0) : true)' - - message: requirements with 'minValues' must have at least that many values specified in the 'values' field - rule: 'self.all(x, (x.operator == ''In'' && has(x.minValues)) ? x.values.size() >= x.minValues : true)' - resources: - description: Resources models the resource requirements for the NodeClaim to launch - properties: - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: Requests describes the minimum required resources for the NodeClaim to launch - type: object - type: object - maxProperties: 0 - startupTaints: - description: |- - StartupTaints are taints that are applied to nodes upon startup which are expected to be removed automatically - within a short period of time, typically by a DaemonSet that tolerates the taint. These are commonly used by - daemonsets to allow initialization and enforce startup ordering. StartupTaints are ignored for provisioning - purposes in that pods are not required to tolerate a StartupTaint in order to have nodes provisioned for them. - items: - description: |- - The node this Taint is attached to has the "effect" on - any pod that does not tolerate the Taint. - properties: - effect: - description: |- - Required. The effect of the taint on pods - that do not tolerate the taint. - Valid effects are NoSchedule, PreferNoSchedule and NoExecute. - type: string - enum: - - NoSchedule - - PreferNoSchedule - - NoExecute - key: - description: Required. The taint key to be applied to a node. - type: string - minLength: 1 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ - timeAdded: - description: |- - TimeAdded represents the time at which the taint was added. - It is only written for NoExecute taints. - format: date-time - type: string - value: - description: The taint value corresponding to the taint key. - type: string - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ - required: - - effect - - key - type: object - type: array - taints: - description: Taints will be applied to the NodeClaim's node. - items: - description: |- - The node this Taint is attached to has the "effect" on - any pod that does not tolerate the Taint. - properties: - effect: - description: |- - Required. The effect of the taint on pods - that do not tolerate the taint. - Valid effects are NoSchedule, PreferNoSchedule and NoExecute. - type: string - enum: - - NoSchedule - - PreferNoSchedule - - NoExecute - key: - description: Required. The taint key to be applied to a node. - type: string - minLength: 1 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ - timeAdded: - description: |- - TimeAdded represents the time at which the taint was added. - It is only written for NoExecute taints. - format: date-time - type: string - value: - description: The taint value corresponding to the taint key. - type: string - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ - required: - - effect - - key - type: object - type: array - required: - - nodeClassRef - - requirements - type: object - required: - - spec - type: object - weight: - description: |- - Weight is the priority given to the nodepool during scheduling. A higher - numerical weight indicates that this nodepool will be ordered - ahead of other nodepools with lower weights. A nodepool with no weight - will be treated as if it is a nodepool with a weight of 0. - format: int32 - maximum: 100 - minimum: 1 - type: integer - required: - - template - type: object - status: - description: NodePoolStatus defines the observed state of NodePool - properties: - conditions: - description: Conditions contains signals for health and readiness - items: - description: Condition aliases the upstream type and adds additional helper methods - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - resources: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: Resources is the list of resources that have been provisioned. - type: object - type: object - required: - - spec - type: object - served: true - storage: false - subresources: - status: {} {{- if .Values.webhook.enabled }} conversion: strategy: Webhook diff --git a/go.mod b/go.mod index c8fc14502..ca025ef6e 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ module github.com/Azure/karpenter-provider-azure -go 1.23.0 +go 1.23.2 + +toolchain go1.23.3 require ( github.com/Azure/azure-kusto-go v0.16.1 @@ -17,9 +19,8 @@ require ( github.com/Azure/skewer v0.0.19 github.com/Pallinder/go-randomdata v1.2.0 github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 - github.com/awslabs/operatorpkg v0.0.0-20240805231134-67d0acfb6306 + github.com/awslabs/operatorpkg v0.0.0-20241125173122-bef8fba1bdf6 github.com/blang/semver/v4 v4.0.0 - github.com/go-logr/logr v1.4.2 github.com/go-logr/zapr v1.3.0 github.com/go-openapi/errors v0.22.0 github.com/go-openapi/runtime v0.28.0 @@ -27,30 +28,28 @@ require ( github.com/go-openapi/swag v0.23.0 github.com/go-openapi/validate v0.24.0 github.com/go-playground/validator/v10 v10.22.1 - github.com/google/go-cmp v0.6.0 github.com/imdario/mergo v0.3.16 github.com/jongio/azidext/go/azidext v0.5.0 github.com/mitchellh/hashstructure/v2 v2.0.2 - github.com/onsi/ginkgo/v2 v2.20.2 - github.com/onsi/gomega v1.34.2 + github.com/onsi/ginkgo/v2 v2.22.0 + github.com/onsi/gomega v1.36.0 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/prometheus/client_golang v1.20.5 github.com/samber/lo v1.47.0 github.com/stretchr/testify v1.9.0 go.uber.org/multierr v1.11.0 go.uber.org/zap v1.27.0 - golang.org/x/sync v0.8.0 gopkg.in/yaml.v2 v2.4.0 - k8s.io/api v0.30.3 - k8s.io/apiextensions-apiserver v0.30.3 - k8s.io/apimachinery v0.30.3 - k8s.io/client-go v0.30.3 + k8s.io/api v0.31.3 + k8s.io/apiextensions-apiserver v0.31.3 + k8s.io/apimachinery v0.31.3 + k8s.io/client-go v0.31.3 k8s.io/klog/v2 v2.130.1 k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 knative.dev/pkg v0.0.0-20240910170930-fdbc0b5adde7 sigs.k8s.io/cloud-provider-azure v1.29.3 - sigs.k8s.io/controller-runtime v0.18.4 - sigs.k8s.io/karpenter v1.0.5 + sigs.k8s.io/controller-runtime v0.19.2 + sigs.k8s.io/karpenter v1.1.0 ) require ( @@ -80,15 +79,16 @@ require ( github.com/blendle/zapdriver v1.3.1 // indirect github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/go-restful/v3 v3.12.1 // indirect - github.com/evanphx/json-patch v5.9.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.9.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/gabriel-vasile/mimetype v1.4.3 // indirect github.com/go-kit/log v0.2.1 // indirect github.com/go-logfmt/logfmt v0.6.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/analysis v0.23.0 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect @@ -104,8 +104,9 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/gnostic-models v0.6.8 // indirect + github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5 // indirect + github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db // indirect github.com/google/uuid v1.6.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.21.0 // indirect github.com/hashicorp/golang-lru v1.0.2 // indirect @@ -125,45 +126,46 @@ require ( github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.55.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/prometheus/statsd_exporter v0.24.0 // indirect github.com/robfig/cron/v3 v3.0.1 // indirect github.com/shopspring/decimal v1.3.1 // indirect - github.com/spf13/cobra v1.8.0 // indirect + github.com/spf13/cobra v1.8.1 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/x448/float16 v0.8.4 // indirect go.mongodb.org/mongo-driver v1.14.0 // indirect go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect - go.opentelemetry.io/otel v1.24.0 // indirect - go.opentelemetry.io/otel/metric v1.24.0 // indirect - go.opentelemetry.io/otel/trace v1.24.0 // indirect - go.uber.org/automaxprocs v1.5.3 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect + go.opentelemetry.io/otel v1.28.0 // indirect + go.opentelemetry.io/otel/metric v1.28.0 // indirect + go.opentelemetry.io/otel/trace v1.28.0 // indirect go.uber.org/mock v0.4.0 // indirect - golang.org/x/crypto v0.27.0 // indirect + golang.org/x/crypto v0.28.0 // indirect golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa // indirect - golang.org/x/net v0.29.0 // indirect + golang.org/x/net v0.30.0 // indirect golang.org/x/oauth2 v0.22.0 // indirect - golang.org/x/sys v0.25.0 // indirect - golang.org/x/term v0.24.0 // indirect - golang.org/x/text v0.18.0 // indirect - golang.org/x/time v0.6.0 // indirect - golang.org/x/tools v0.24.0 // indirect + golang.org/x/sync v0.9.0 // indirect + golang.org/x/sys v0.26.0 // indirect + golang.org/x/term v0.25.0 // indirect + golang.org/x/text v0.20.0 // indirect + golang.org/x/time v0.8.0 // indirect + golang.org/x/tools v0.26.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/api v0.183.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240808171019-573a1156607a // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240808171019-573a1156607a // indirect google.golang.org/grpc v1.65.0 // indirect - google.golang.org/protobuf v1.34.2 // indirect + google.golang.org/protobuf v1.35.1 // indirect gopkg.in/dnaeon/go-vcr.v3 v3.2.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/cloud-provider v0.30.3 // indirect - k8s.io/component-base v0.30.3 // indirect - k8s.io/component-helpers v0.30.3 // indirect - k8s.io/csi-translation-lib v0.30.3 // indirect + k8s.io/cloud-provider v0.31.3 // indirect + k8s.io/component-base v0.31.3 // indirect + k8s.io/component-helpers v0.31.3 // indirect + k8s.io/csi-translation-lib v0.31.3 // indirect k8s.io/kube-openapi v0.0.0-20240808142205-8e686545bdb8 // indirect sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.0.26 // indirect sigs.k8s.io/cloud-provider-azure/pkg/azclient/configloader v0.0.1 // indirect diff --git a/go.sum b/go.sum index 92808e710..676642a85 100644 --- a/go.sum +++ b/go.sum @@ -125,8 +125,8 @@ github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3d github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0= github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY= -github.com/awslabs/operatorpkg v0.0.0-20240805231134-67d0acfb6306 h1:0dzaVod1XLEc38H4IB+KOgStoCt8RkCVI4t+XsSPrWE= -github.com/awslabs/operatorpkg v0.0.0-20240805231134-67d0acfb6306/go.mod h1:7u2ugtOiWSvqqwNlnQ8W+2TjwnSTbHoMHnR1AKpKVMA= +github.com/awslabs/operatorpkg v0.0.0-20241125173122-bef8fba1bdf6 h1:O32mqYd4TFNsiZP93RSPfkAKaskV0vP8pBcwTkR9p2M= +github.com/awslabs/operatorpkg v0.0.0-20241125173122-bef8fba1bdf6/go.mod h1:jina2fQk+b3oa9r5bRuMDbpy6mfhTGuruh05oVVWwnk= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -147,10 +147,11 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtzpL63nKAU= @@ -167,6 +168,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= +github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -293,8 +296,8 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5 h1:5iH8iuqE5apketRbSFBy+X1V0o+l+8NF1avt4HWl7cA= -github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -371,10 +374,10 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/onsi/ginkgo/v2 v2.20.2 h1:7NVCeyIWROIAheY21RLS+3j2bb52W0W82tkberYytp4= -github.com/onsi/ginkgo/v2 v2.20.2/go.mod h1:K9gyxPIlb+aIvnZ8bd9Ak+YP18w3APlR+5coaZoE2ag= -github.com/onsi/gomega v1.34.2 h1:pNCwDkzrsv7MS9kpaQvVb1aVLahQXyJ/Tv5oAZMI3i8= -github.com/onsi/gomega v1.34.2/go.mod h1:v1xfxRgk0KIsG+QOdm7p8UosrOzPYRo60fd3B/1Dukc= +github.com/onsi/ginkgo/v2 v2.22.0 h1:Yed107/8DjTr0lKCNt7Dn8yQ6ybuDRQoMGrNFKzMfHg= +github.com/onsi/ginkgo/v2 v2.22.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= +github.com/onsi/gomega v1.36.0 h1:Pb12RlruUtj4XUuPUqeEWc6j5DkVVVA49Uf6YLfC95Y= +github.com/onsi/gomega v1.36.0/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= @@ -385,10 +388,9 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= -github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= @@ -439,8 +441,8 @@ github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFR github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= -github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -458,6 +460,8 @@ github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stvp/go-udp-testing v0.0.0-20201019212854-469649b16807/go.mod h1:7jxmlfBCDBXRzr0eAQJ48XC1hBu1np4CS5+cHEYfwpc= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -473,19 +477,17 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= -go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= -go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= -go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= -go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= -go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw= -go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg= -go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= -go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 h1:4K4tsIXefpVJtvA/8srF4V4y0akAoPHkIslgAkjixJA= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0/go.mod h1:jjdQuTGVsXV4vSs+CJ2qYDeDPf9yIJV23qlIzBm73Vg= +go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= +go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= +go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q= +go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s= +go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE= +go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg= +go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g= +go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/automaxprocs v1.5.3 h1:kWazyxZUrS3Gs4qUpbwo5kEIMGe/DAvi5Z4tl2NW4j8= -go.uber.org/automaxprocs v1.5.3/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= @@ -506,8 +508,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= -golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= -golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= +golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= +golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -542,8 +544,8 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0= -golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= +golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -583,8 +585,8 @@ golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= -golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= +golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= +golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -607,8 +609,8 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= -golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.9.0 h1:fEo0HyrW1GIgZdpbhCRO0PkJajUS5H9IFUztCgEo2jQ= +golang.org/x/sync v0.9.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -655,15 +657,15 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= -golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= -golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM= -golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8= +golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24= +golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -674,13 +676,13 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= -golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug= +golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= -golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.8.0 h1:9i3RxcPv3PZnitoVGMPDKZSq1xW1gK1Xy3ArNOGZfEg= +golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -725,8 +727,8 @@ golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24= -golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ= +golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= +golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -822,8 +824,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= +google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -833,6 +835,8 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV gopkg.in/dnaeon/go-vcr.v3 v3.2.0 h1:Rltp0Vf+Aq0u4rQXgmXgtgoRDStTnFN83cWgSGSoRzM= gopkg.in/dnaeon/go-vcr.v3 v3.2.0/go.mod h1:2IMOnnlx9I6u9x+YBsM3tAMx6AlOxnJ0pWxQAzZ79Ag= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= +gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -854,22 +858,22 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.30.3 h1:ImHwK9DCsPA9uoU3rVh4QHAHHK5dTSv1nxJUapx8hoQ= -k8s.io/api v0.30.3/go.mod h1:GPc8jlzoe5JG3pb0KJCSLX5oAFIW3/qNJITlDj8BH04= -k8s.io/apiextensions-apiserver v0.30.3 h1:oChu5li2vsZHx2IvnGP3ah8Nj3KyqG3kRSaKmijhB9U= -k8s.io/apiextensions-apiserver v0.30.3/go.mod h1:uhXxYDkMAvl6CJw4lrDN4CPbONkF3+XL9cacCT44kV4= -k8s.io/apimachinery v0.30.3 h1:q1laaWCmrszyQuSQCfNB8cFgCuDAoPszKY4ucAjDwHc= -k8s.io/apimachinery v0.30.3/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= -k8s.io/client-go v0.30.3 h1:bHrJu3xQZNXIi8/MoxYtZBBWQQXwy16zqJwloXXfD3k= -k8s.io/client-go v0.30.3/go.mod h1:8d4pf8vYu665/kUbsxWAQ/JDBNWqfFeZnvFiVdmx89U= -k8s.io/cloud-provider v0.30.3 h1:SNWZmllTymOTzIPJuhtZH6il/qVi75dQARRQAm9k6VY= -k8s.io/cloud-provider v0.30.3/go.mod h1:Ax0AVdHnM7tMYnJH1Ycy4SMBD98+4zA+tboUR9eYsY8= -k8s.io/component-base v0.30.3 h1:Ci0UqKWf4oiwy8hr1+E3dsnliKnkMLZMVbWzeorlk7s= -k8s.io/component-base v0.30.3/go.mod h1:C1SshT3rGPCuNtBs14RmVD2xW0EhRSeLvBh7AGk1quA= -k8s.io/component-helpers v0.30.3 h1:KPc8l0eGx9Wg2OcKc58k9ozNcVcOInAi3NGiuS2xJ/c= -k8s.io/component-helpers v0.30.3/go.mod h1:VOQ7g3q+YbKWwKeACG2BwPv4ftaN8jXYJ5U3xpzuYAE= -k8s.io/csi-translation-lib v0.30.3 h1:wBaPWnOi14/vANRIrp8pmbdx/Pgz2QRcroH7wkodezc= -k8s.io/csi-translation-lib v0.30.3/go.mod h1:3AizNZbDttVDH1RO0x1yGEQP74e9Xbfb60IBP1oWO1o= +k8s.io/api v0.31.3 h1:umzm5o8lFbdN/hIXbrK9oRpOproJO62CV1zqxXrLgk8= +k8s.io/api v0.31.3/go.mod h1:UJrkIp9pnMOI9K2nlL6vwpxRzzEX5sWgn8kGQe92kCE= +k8s.io/apiextensions-apiserver v0.31.3 h1:+GFGj2qFiU7rGCsA5o+p/rul1OQIq6oYpQw4+u+nciE= +k8s.io/apiextensions-apiserver v0.31.3/go.mod h1:2DSpFhUZZJmn/cr/RweH1cEVVbzFw9YBu4T+U3mf1e4= +k8s.io/apimachinery v0.31.3 h1:6l0WhcYgasZ/wk9ktLq5vLaoXJJr5ts6lkaQzgeYPq4= +k8s.io/apimachinery v0.31.3/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= +k8s.io/client-go v0.31.3 h1:CAlZuM+PH2cm+86LOBemaJI/lQ5linJ6UFxKX/SoG+4= +k8s.io/client-go v0.31.3/go.mod h1:2CgjPUTpv3fE5dNygAr2NcM8nhHzXvxB8KL5gYc3kJs= +k8s.io/cloud-provider v0.31.3 h1:7C3CHQUUwnv/HWWVIaibZH06iPg663RYQ6C6Zy4FnO8= +k8s.io/cloud-provider v0.31.3/go.mod h1:c7csKppoVb9Ej6upJ28AvHy4B3BtlRMzXfgezsDdPKw= +k8s.io/component-base v0.31.3 h1:DMCXXVx546Rfvhj+3cOm2EUxhS+EyztH423j+8sOwhQ= +k8s.io/component-base v0.31.3/go.mod h1:xME6BHfUOafRgT0rGVBGl7TuSg8Z9/deT7qq6w7qjIU= +k8s.io/component-helpers v0.31.3 h1:0zGPD2PrekhFWgmz85XxlMEl7dfhlKC1tERZDe3onQc= +k8s.io/component-helpers v0.31.3/go.mod h1:HZ1HZx2TKXM7xSUV2cR9L5yDoyZPhhHQNaE3BPBLPUQ= +k8s.io/csi-translation-lib v0.31.3 h1:hxcPRNdtEsk766jCXSKjgH1V8jUNx5tVqdooQ1Ars/M= +k8s.io/csi-translation-lib v0.31.3/go.mod h1:0B1gQwd868XUIDwJYy5gB2jDXWEwlcWvSsfcQEgzbRk= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20240808142205-8e686545bdb8 h1:1Wof1cGQgA5pqgo8MxKPtf+qN6Sh/0JzznmeGPm1HnE= @@ -887,12 +891,12 @@ sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.0.26 h1:BHauRhfjzs4UWu/yiLw82WK sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.0.26/go.mod h1:02JRJ7ioAoT9PZzIxlR4Kw7WbejsMIy1eeDyYX8sgvk= sigs.k8s.io/cloud-provider-azure/pkg/azclient/configloader v0.0.1 h1:Lp0nALZmvMJoiVeVV6XjnZv1uClfArnThhuDAjaqE5A= sigs.k8s.io/cloud-provider-azure/pkg/azclient/configloader v0.0.1/go.mod h1:pPkJPx/eMVWP3R+LhPoOYGoY7lywcMJev5L2uSfH+Jo= -sigs.k8s.io/controller-runtime v0.18.4 h1:87+guW1zhvuPLh1PHybKdYFLU0YJp4FhJRmiHvm5BZw= -sigs.k8s.io/controller-runtime v0.18.4/go.mod h1:TVoGrfdpbA9VRFaRnKgk9P5/atA0pMwq+f+msb9M8Sg= +sigs.k8s.io/controller-runtime v0.19.2 h1:3sPrF58XQEPzbE8T81TN6selQIMGbtYwuaJ6eDssDF8= +sigs.k8s.io/controller-runtime v0.19.2/go.mod h1:iRmWllt8IlaLjvTTDLhRBXIEtkCK6hwVBJJsYS9Ajf4= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/karpenter v1.0.5 h1:QePds7w1dGCzTXI59fKpYyV5GL/LQgodOCMC1pYjqhM= -sigs.k8s.io/karpenter v1.0.5/go.mod h1:3NLmsnHHw8p4VutpjTOPUZyhE3qH6yGTs8O94Lsu8uw= +sigs.k8s.io/karpenter v1.1.0 h1:4SdCXihNKH4Y5xLwtKh7sqvCPYTQ3vCvCTPFRaNPQ5Y= +sigs.k8s.io/karpenter v1.1.0/go.mod h1:LPVsUkC/MdmIjCBcmNVzy1wLp86xRKTRKFGPSNAW+YE= sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= diff --git a/pkg/apis/crds/karpenter.sh_nodeclaims.yaml b/pkg/apis/crds/karpenter.sh_nodeclaims.yaml index 1a6f12088..0e17daa43 100644 --- a/pkg/apis/crds/karpenter.sh_nodeclaims.yaml +++ b/pkg/apis/crds/karpenter.sh_nodeclaims.yaml @@ -88,12 +88,21 @@ spec: description: API version of the referent pattern: ^[^/]*$ type: string + x-kubernetes-validations: + - message: group may not be empty + rule: self != '' kind: description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' type: string + x-kubernetes-validations: + - message: kind may not be empty + rule: self != '' name: description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' type: string + x-kubernetes-validations: + - message: name may not be empty + rule: self != '' required: - group - kind @@ -380,440 +389,6 @@ spec: storage: true subresources: status: {} - - additionalPrinterColumns: - - jsonPath: .metadata.labels.node\.kubernetes\.io/instance-type - name: Type - type: string - - jsonPath: .metadata.labels.topology\.kubernetes\.io/zone - name: Zone - type: string - - jsonPath: .status.nodeName - name: Node - type: string - - jsonPath: .status.conditions[?(@.type=="Ready")].status - name: Ready - type: string - - jsonPath: .metadata.creationTimestamp - name: Age - type: date - - jsonPath: .metadata.labels.karpenter\.sh/capacity-type - name: Capacity - priority: 1 - type: string - - jsonPath: .metadata.labels.karpenter\.sh/nodepool - name: NodePool - priority: 1 - type: string - - jsonPath: .spec.nodeClassRef.name - name: NodeClass - priority: 1 - type: string - name: v1beta1 - schema: - openAPIV3Schema: - description: NodeClaim is the Schema for the NodeClaims API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: NodeClaimSpec describes the desired state of the NodeClaim - properties: - kubelet: - description: |- - Kubelet defines args to be used when configuring kubelet on provisioned nodes. - They are a subset of the upstream types, recognizing not all options may be supported. - Wherever possible, the types and names should reflect the upstream kubelet types. - properties: - clusterDNS: - description: |- - clusterDNS is a list of IP addresses for the cluster DNS server. - Note that not all providers may use all addresses. - items: - type: string - type: array - cpuCFSQuota: - description: CPUCFSQuota enables CPU CFS quota enforcement for containers that specify CPU limits. - type: boolean - evictionHard: - additionalProperties: - type: string - pattern: ^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$ - description: EvictionHard is the map of signal names to quantities that define hard eviction thresholds - type: object - x-kubernetes-validations: - - message: valid keys for evictionHard are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] - rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) - evictionMaxPodGracePeriod: - description: |- - EvictionMaxPodGracePeriod is the maximum allowed grace period (in seconds) to use when terminating pods in - response to soft eviction thresholds being met. - format: int32 - type: integer - evictionSoft: - additionalProperties: - type: string - pattern: ^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$ - description: EvictionSoft is the map of signal names to quantities that define soft eviction thresholds - type: object - x-kubernetes-validations: - - message: valid keys for evictionSoft are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] - rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) - evictionSoftGracePeriod: - additionalProperties: - type: string - description: EvictionSoftGracePeriod is the map of signal names to quantities that define grace periods for each eviction signal - type: object - x-kubernetes-validations: - - message: valid keys for evictionSoftGracePeriod are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] - rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) - imageGCHighThresholdPercent: - description: |- - ImageGCHighThresholdPercent is the percent of disk usage after which image - garbage collection is always run. The percent is calculated by dividing this - field value by 100, so this field must be between 0 and 100, inclusive. - When specified, the value must be greater than ImageGCLowThresholdPercent. - format: int32 - maximum: 100 - minimum: 0 - type: integer - imageGCLowThresholdPercent: - description: |- - ImageGCLowThresholdPercent is the percent of disk usage before which image - garbage collection is never run. Lowest disk usage to garbage collect to. - The percent is calculated by dividing this field value by 100, - so the field value must be between 0 and 100, inclusive. - When specified, the value must be less than imageGCHighThresholdPercent - format: int32 - maximum: 100 - minimum: 0 - type: integer - kubeReserved: - additionalProperties: - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - description: KubeReserved contains resources reserved for Kubernetes system components. - type: object - x-kubernetes-validations: - - message: valid keys for kubeReserved are ['cpu','memory','ephemeral-storage','pid'] - rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid') - - message: kubeReserved value cannot be a negative resource quantity - rule: self.all(x, !self[x].startsWith('-')) - maxPods: - description: |- - MaxPods is an override for the maximum number of pods that can run on - a worker node instance. - format: int32 - minimum: 0 - type: integer - podsPerCore: - description: |- - PodsPerCore is an override for the number of pods that can run on a worker node - instance based on the number of cpu cores. This value cannot exceed MaxPods, so, if - MaxPods is a lower value, that value will be used. - format: int32 - minimum: 0 - type: integer - systemReserved: - additionalProperties: - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - description: SystemReserved contains resources reserved for OS system daemons and kernel memory. - type: object - x-kubernetes-validations: - - message: valid keys for systemReserved are ['cpu','memory','ephemeral-storage','pid'] - rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid') - - message: systemReserved value cannot be a negative resource quantity - rule: self.all(x, !self[x].startsWith('-')) - type: object - x-kubernetes-validations: - - message: imageGCHighThresholdPercent must be greater than imageGCLowThresholdPercent - rule: 'has(self.imageGCHighThresholdPercent) && has(self.imageGCLowThresholdPercent) ? self.imageGCHighThresholdPercent > self.imageGCLowThresholdPercent : true' - - message: evictionSoft OwnerKey does not have a matching evictionSoftGracePeriod - rule: has(self.evictionSoft) ? self.evictionSoft.all(e, (e in self.evictionSoftGracePeriod)):true - - message: evictionSoftGracePeriod OwnerKey does not have a matching evictionSoft - rule: has(self.evictionSoftGracePeriod) ? self.evictionSoftGracePeriod.all(e, (e in self.evictionSoft)):true - nodeClassRef: - description: NodeClassRef is a reference to an object that defines provider specific configuration - properties: - apiVersion: - description: API version of the referent - type: string - kind: - description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' - type: string - name: - description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' - type: string - required: - - name - type: object - requirements: - description: Requirements are layered with GetLabels and applied to every node. - items: - description: |- - A node selector requirement with min values is a selector that contains values, a key, an operator that relates the key and values - and minValues that represent the requirement to have at least that many values. - properties: - key: - description: The label key that the selector applies to. - type: string - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ - x-kubernetes-validations: - - message: label domain "kubernetes.io" is restricted - rule: self in ["beta.kubernetes.io/instance-type", "failure-domain.beta.kubernetes.io/region", "beta.kubernetes.io/os", "beta.kubernetes.io/arch", "failure-domain.beta.kubernetes.io/zone", "topology.kubernetes.io/zone", "topology.kubernetes.io/region", "node.kubernetes.io/instance-type", "kubernetes.io/arch", "kubernetes.io/os", "node.kubernetes.io/windows-build"] || self.find("^([^/]+)").endsWith("node.kubernetes.io") || self.find("^([^/]+)").endsWith("node-restriction.kubernetes.io") || !self.find("^([^/]+)").endsWith("kubernetes.io") - - message: label domain "k8s.io" is restricted - rule: self.find("^([^/]+)").endsWith("kops.k8s.io") || !self.find("^([^/]+)").endsWith("k8s.io") - - message: label domain "karpenter.sh" is restricted - rule: self in ["karpenter.sh/capacity-type", "karpenter.sh/nodepool"] || !self.find("^([^/]+)").endsWith("karpenter.sh") - - message: label "kubernetes.io/hostname" is restricted - rule: self != "kubernetes.io/hostname" - minValues: - description: |- - This field is ALPHA and can be dropped or replaced at any time - MinValues is the minimum number of unique values required to define the flexibility of the specific requirement. - maximum: 50 - minimum: 1 - type: integer - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - enum: - - In - - NotIn - - Exists - - DoesNotExist - - Gt - - Lt - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - maxLength: 63 - pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ - required: - - key - - operator - type: object - maxItems: 100 - type: array - x-kubernetes-validations: - - message: requirements with operator 'In' must have a value defined - rule: 'self.all(x, x.operator == ''In'' ? x.values.size() != 0 : true)' - - message: requirements operator 'Gt' or 'Lt' must have a single positive integer value - rule: 'self.all(x, (x.operator == ''Gt'' || x.operator == ''Lt'') ? (x.values.size() == 1 && int(x.values[0]) >= 0) : true)' - - message: requirements with 'minValues' must have at least that many values specified in the 'values' field - rule: 'self.all(x, (x.operator == ''In'' && has(x.minValues)) ? x.values.size() >= x.minValues : true)' - resources: - description: Resources models the resource requirements for the NodeClaim to launch - properties: - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: Requests describes the minimum required resources for the NodeClaim to launch - type: object - type: object - startupTaints: - description: |- - StartupTaints are taints that are applied to nodes upon startup which are expected to be removed automatically - within a short period of time, typically by a DaemonSet that tolerates the taint. These are commonly used by - daemonsets to allow initialization and enforce startup ordering. StartupTaints are ignored for provisioning - purposes in that pods are not required to tolerate a StartupTaint in order to have nodes provisioned for them. - items: - description: |- - The node this Taint is attached to has the "effect" on - any pod that does not tolerate the Taint. - properties: - effect: - description: |- - Required. The effect of the taint on pods - that do not tolerate the taint. - Valid effects are NoSchedule, PreferNoSchedule and NoExecute. - type: string - enum: - - NoSchedule - - PreferNoSchedule - - NoExecute - key: - description: Required. The taint key to be applied to a node. - type: string - minLength: 1 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ - timeAdded: - description: |- - TimeAdded represents the time at which the taint was added. - It is only written for NoExecute taints. - format: date-time - type: string - value: - description: The taint value corresponding to the taint key. - type: string - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ - required: - - effect - - key - type: object - type: array - taints: - description: Taints will be applied to the NodeClaim's node. - items: - description: |- - The node this Taint is attached to has the "effect" on - any pod that does not tolerate the Taint. - properties: - effect: - description: |- - Required. The effect of the taint on pods - that do not tolerate the taint. - Valid effects are NoSchedule, PreferNoSchedule and NoExecute. - type: string - enum: - - NoSchedule - - PreferNoSchedule - - NoExecute - key: - description: Required. The taint key to be applied to a node. - type: string - minLength: 1 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ - timeAdded: - description: |- - TimeAdded represents the time at which the taint was added. - It is only written for NoExecute taints. - format: date-time - type: string - value: - description: The taint value corresponding to the taint key. - type: string - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ - required: - - effect - - key - type: object - type: array - required: - - nodeClassRef - - requirements - type: object - status: - description: NodeClaimStatus defines the observed state of NodeClaim - properties: - allocatable: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: Allocatable is the estimated allocatable capacity of the node - type: object - capacity: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: Capacity is the estimated full capacity of the node - type: object - conditions: - description: Conditions contains signals for health and readiness - items: - description: Condition aliases the upstream type and adds additional helper methods - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - pattern: ^([A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?|)$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - status - - type - type: object - type: array - imageID: - description: ImageID is an identifier for the image that runs on the node - type: string - nodeName: - description: NodeName is the name of the corresponding node object - type: string - providerID: - description: ProviderID of the corresponding node object - type: string - type: object - required: - - spec - type: object - served: true - storage: false - subresources: - status: {} conversion: strategy: Webhook webhook: diff --git a/pkg/apis/crds/karpenter.sh_nodepools.yaml b/pkg/apis/crds/karpenter.sh_nodepools.yaml index e61df31f2..2465ae81f 100644 --- a/pkg/apis/crds/karpenter.sh_nodepools.yaml +++ b/pkg/apis/crds/karpenter.sh_nodepools.yaml @@ -231,17 +231,31 @@ spec: description: API version of the referent pattern: ^[^/]*$ type: string + x-kubernetes-validations: + - message: group may not be empty + rule: self != '' kind: description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' type: string + x-kubernetes-validations: + - message: kind may not be empty + rule: self != '' name: description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' type: string + x-kubernetes-validations: + - message: name may not be empty + rule: self != '' required: - group - kind - name type: object + x-kubernetes-validations: + - message: nodeClassRef.group is immutable + rule: self.group == oldSelf.group + - message: nodeClassRef.kind is immutable + rule: self.kind == oldSelf.kind requirements: description: Requirements are layered with GetLabels and applied to every node. items: @@ -503,568 +517,6 @@ spec: storage: true subresources: status: {} - - additionalPrinterColumns: - - jsonPath: .spec.template.spec.nodeClassRef.name - name: NodeClass - type: string - - jsonPath: .spec.weight - name: Weight - priority: 1 - type: string - name: v1beta1 - schema: - openAPIV3Schema: - description: NodePool is the Schema for the NodePools API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: |- - NodePoolSpec is the top level nodepool specification. Nodepools - launch nodes in response to pods that are unschedulable. A single nodepool - is capable of managing a diverse set of nodes. Node properties are determined - from a combination of nodepool and pod scheduling constraints. - properties: - disruption: - default: - consolidationPolicy: WhenUnderutilized - expireAfter: 720h - description: Disruption contains the parameters that relate to Karpenter's disruption logic - properties: - budgets: - default: - - nodes: 10% - description: |- - Budgets is a list of Budgets. - If there are multiple active budgets, Karpenter uses - the most restrictive value. If left undefined, - this will default to one budget with a value to 10%. - items: - description: |- - Budget defines when Karpenter will restrict the - number of Node Claims that can be terminating simultaneously. - properties: - duration: - description: |- - Duration determines how long a Budget is active since each Schedule hit. - Only minutes and hours are accepted, as cron does not work in seconds. - If omitted, the budget is always active. - This is required if Schedule is set. - This regex has an optional 0s at the end since the duration.String() always adds - a 0s at the end. - pattern: ^((([0-9]+(h|m))|([0-9]+h[0-9]+m))(0s)?)$ - type: string - nodes: - default: 10% - description: |- - Nodes dictates the maximum number of NodeClaims owned by this NodePool - that can be terminating at once. This is calculated by counting nodes that - have a deletion timestamp set, or are actively being deleted by Karpenter. - This field is required when specifying a budget. - This cannot be of type intstr.IntOrString since kubebuilder doesn't support pattern - checking for int nodes for IntOrString nodes. - Ref: https://github.com/kubernetes-sigs/controller-tools/blob/55efe4be40394a288216dab63156b0a64fb82929/pkg/crd/markers/validation.go#L379-L388 - pattern: ^((100|[0-9]{1,2})%|[0-9]+)$ - type: string - schedule: - description: |- - Schedule specifies when a budget begins being active, following - the upstream cronjob syntax. If omitted, the budget is always active. - Timezones are not supported. - This field is required if Duration is set. - pattern: ^(@(annually|yearly|monthly|weekly|daily|midnight|hourly))|((.+)\s(.+)\s(.+)\s(.+)\s(.+))$ - type: string - required: - - nodes - type: object - maxItems: 50 - type: array - x-kubernetes-validations: - - message: '''schedule'' must be set with ''duration''' - rule: self.all(x, has(x.schedule) == has(x.duration)) - consolidateAfter: - description: |- - ConsolidateAfter is the duration the controller will wait - before attempting to terminate nodes that are underutilized. - Refer to ConsolidationPolicy for how underutilization is considered. - pattern: ^(([0-9]+(s|m|h))+)|(Never)$ - type: string - consolidationPolicy: - default: WhenUnderutilized - description: |- - ConsolidationPolicy describes which nodes Karpenter can disrupt through its consolidation - algorithm. This policy defaults to "WhenUnderutilized" if not specified - enum: - - WhenEmpty - - WhenUnderutilized - type: string - expireAfter: - default: 720h - description: |- - ExpireAfter is the duration the controller will wait - before terminating a node, measured from when the node is created. This - is useful to implement features like eventually consistent node upgrade, - memory leak protection, and disruption testing. - pattern: ^(([0-9]+(s|m|h))+)|(Never)$ - type: string - type: object - x-kubernetes-validations: - - message: consolidateAfter cannot be combined with consolidationPolicy=WhenUnderutilized - rule: 'has(self.consolidateAfter) ? self.consolidationPolicy != ''WhenUnderutilized'' || self.consolidateAfter == ''Never'' : true' - - message: consolidateAfter must be specified with consolidationPolicy=WhenEmpty - rule: 'self.consolidationPolicy == ''WhenEmpty'' ? has(self.consolidateAfter) : true' - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: Limits define a set of bounds for provisioning capacity. - type: object - template: - description: |- - Template contains the template of possibilities for the provisioning logic to launch a NodeClaim with. - NodeClaims launched from this NodePool will often be further constrained than the template specifies. - properties: - metadata: - properties: - annotations: - additionalProperties: - type: string - description: |- - Annotations is an unstructured key value map stored with a resource that may be - set by external tools to store and retrieve arbitrary metadata. They are not - queryable and should be preserved when modifying objects. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations - type: object - labels: - additionalProperties: - type: string - maxLength: 63 - pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ - description: |- - Map of string keys and values that can be used to organize and categorize - (scope and select) objects. May match selectors of replication controllers - and services. - More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels - type: object - maxProperties: 100 - x-kubernetes-validations: - - message: label domain "kubernetes.io" is restricted - rule: self.all(x, x in ["beta.kubernetes.io/instance-type", "failure-domain.beta.kubernetes.io/region", "beta.kubernetes.io/os", "beta.kubernetes.io/arch", "failure-domain.beta.kubernetes.io/zone", "topology.kubernetes.io/region", "kubernetes.io/arch", "kubernetes.io/os", "node.kubernetes.io/windows-build"] || x.find("^([^/]+)").endsWith("node.kubernetes.io") || x.find("^([^/]+)").endsWith("node-restriction.kubernetes.io") || !x.find("^([^/]+)").endsWith("kubernetes.io")) - - message: label domain "k8s.io" is restricted - rule: self.all(x, x.find("^([^/]+)").endsWith("kops.k8s.io") || !x.find("^([^/]+)").endsWith("k8s.io")) - - message: label domain "karpenter.sh" is restricted - rule: self.all(x, x in ["karpenter.sh/capacity-type", "karpenter.sh/nodepool"] || !x.find("^([^/]+)").endsWith("karpenter.sh")) - - message: label "karpenter.sh/nodepool" is restricted - rule: self.all(x, x != "karpenter.sh/nodepool") - - message: label "kubernetes.io/hostname" is restricted - rule: self.all(x, x != "kubernetes.io/hostname") - type: object - spec: - description: NodeClaimSpec describes the desired state of the NodeClaim - properties: - kubelet: - description: |- - Kubelet defines args to be used when configuring kubelet on provisioned nodes. - They are a subset of the upstream types, recognizing not all options may be supported. - Wherever possible, the types and names should reflect the upstream kubelet types. - properties: - clusterDNS: - description: |- - clusterDNS is a list of IP addresses for the cluster DNS server. - Note that not all providers may use all addresses. - items: - type: string - type: array - cpuCFSQuota: - description: CPUCFSQuota enables CPU CFS quota enforcement for containers that specify CPU limits. - type: boolean - evictionHard: - additionalProperties: - type: string - pattern: ^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$ - description: EvictionHard is the map of signal names to quantities that define hard eviction thresholds - type: object - x-kubernetes-validations: - - message: valid keys for evictionHard are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] - rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) - evictionMaxPodGracePeriod: - description: |- - EvictionMaxPodGracePeriod is the maximum allowed grace period (in seconds) to use when terminating pods in - response to soft eviction thresholds being met. - format: int32 - type: integer - evictionSoft: - additionalProperties: - type: string - pattern: ^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$ - description: EvictionSoft is the map of signal names to quantities that define soft eviction thresholds - type: object - x-kubernetes-validations: - - message: valid keys for evictionSoft are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] - rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) - evictionSoftGracePeriod: - additionalProperties: - type: string - description: EvictionSoftGracePeriod is the map of signal names to quantities that define grace periods for each eviction signal - type: object - x-kubernetes-validations: - - message: valid keys for evictionSoftGracePeriod are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] - rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) - imageGCHighThresholdPercent: - description: |- - ImageGCHighThresholdPercent is the percent of disk usage after which image - garbage collection is always run. The percent is calculated by dividing this - field value by 100, so this field must be between 0 and 100, inclusive. - When specified, the value must be greater than ImageGCLowThresholdPercent. - format: int32 - maximum: 100 - minimum: 0 - type: integer - imageGCLowThresholdPercent: - description: |- - ImageGCLowThresholdPercent is the percent of disk usage before which image - garbage collection is never run. Lowest disk usage to garbage collect to. - The percent is calculated by dividing this field value by 100, - so the field value must be between 0 and 100, inclusive. - When specified, the value must be less than imageGCHighThresholdPercent - format: int32 - maximum: 100 - minimum: 0 - type: integer - kubeReserved: - additionalProperties: - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - description: KubeReserved contains resources reserved for Kubernetes system components. - type: object - x-kubernetes-validations: - - message: valid keys for kubeReserved are ['cpu','memory','ephemeral-storage','pid'] - rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid') - - message: kubeReserved value cannot be a negative resource quantity - rule: self.all(x, !self[x].startsWith('-')) - maxPods: - description: |- - MaxPods is an override for the maximum number of pods that can run on - a worker node instance. - format: int32 - minimum: 0 - type: integer - podsPerCore: - description: |- - PodsPerCore is an override for the number of pods that can run on a worker node - instance based on the number of cpu cores. This value cannot exceed MaxPods, so, if - MaxPods is a lower value, that value will be used. - format: int32 - minimum: 0 - type: integer - systemReserved: - additionalProperties: - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - description: SystemReserved contains resources reserved for OS system daemons and kernel memory. - type: object - x-kubernetes-validations: - - message: valid keys for systemReserved are ['cpu','memory','ephemeral-storage','pid'] - rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid') - - message: systemReserved value cannot be a negative resource quantity - rule: self.all(x, !self[x].startsWith('-')) - type: object - x-kubernetes-validations: - - message: imageGCHighThresholdPercent must be greater than imageGCLowThresholdPercent - rule: 'has(self.imageGCHighThresholdPercent) && has(self.imageGCLowThresholdPercent) ? self.imageGCHighThresholdPercent > self.imageGCLowThresholdPercent : true' - - message: evictionSoft OwnerKey does not have a matching evictionSoftGracePeriod - rule: has(self.evictionSoft) ? self.evictionSoft.all(e, (e in self.evictionSoftGracePeriod)):true - - message: evictionSoftGracePeriod OwnerKey does not have a matching evictionSoft - rule: has(self.evictionSoftGracePeriod) ? self.evictionSoftGracePeriod.all(e, (e in self.evictionSoft)):true - nodeClassRef: - description: NodeClassRef is a reference to an object that defines provider specific configuration - properties: - apiVersion: - description: API version of the referent - type: string - kind: - description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' - type: string - name: - description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' - type: string - required: - - name - type: object - requirements: - description: Requirements are layered with GetLabels and applied to every node. - items: - description: |- - A node selector requirement with min values is a selector that contains values, a key, an operator that relates the key and values - and minValues that represent the requirement to have at least that many values. - properties: - key: - description: The label key that the selector applies to. - type: string - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ - x-kubernetes-validations: - - message: label domain "kubernetes.io" is restricted - rule: self in ["beta.kubernetes.io/instance-type", "failure-domain.beta.kubernetes.io/region", "beta.kubernetes.io/os", "beta.kubernetes.io/arch", "failure-domain.beta.kubernetes.io/zone", "topology.kubernetes.io/region", "node.kubernetes.io/instance-type", "kubernetes.io/arch", "kubernetes.io/os", "node.kubernetes.io/windows-build"] || self.find("^([^/]+)").endsWith("node.kubernetes.io") || self.find("^([^/]+)").endsWith("node-restriction.kubernetes.io") || !self.find("^([^/]+)").endsWith("kubernetes.io") - - message: label domain "k8s.io" is restricted - rule: self.find("^([^/]+)").endsWith("kops.k8s.io") || !self.find("^([^/]+)").endsWith("k8s.io") - - message: label domain "karpenter.sh" is restricted - rule: self in ["karpenter.sh/capacity-type", "karpenter.sh/nodepool"] || !self.find("^([^/]+)").endsWith("karpenter.sh") - - message: label "karpenter.sh/nodepool" is restricted - rule: self != "karpenter.sh/nodepool" - - message: label "kubernetes.io/hostname" is restricted - rule: self != "kubernetes.io/hostname" - minValues: - description: |- - This field is ALPHA and can be dropped or replaced at any time - MinValues is the minimum number of unique values required to define the flexibility of the specific requirement. - maximum: 50 - minimum: 1 - type: integer - operator: - description: |- - Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. - type: string - enum: - - In - - NotIn - - Exists - - DoesNotExist - - Gt - - Lt - values: - description: |- - An array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. If the operator is Gt or Lt, the values - array must have a single element, which will be interpreted as an integer. - This array is replaced during a strategic merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - maxLength: 63 - pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ - required: - - key - - operator - type: object - maxItems: 100 - type: array - x-kubernetes-validations: - - message: requirements with operator 'In' must have a value defined - rule: 'self.all(x, x.operator == ''In'' ? x.values.size() != 0 : true)' - - message: requirements operator 'Gt' or 'Lt' must have a single positive integer value - rule: 'self.all(x, (x.operator == ''Gt'' || x.operator == ''Lt'') ? (x.values.size() == 1 && int(x.values[0]) >= 0) : true)' - - message: requirements with 'minValues' must have at least that many values specified in the 'values' field - rule: 'self.all(x, (x.operator == ''In'' && has(x.minValues)) ? x.values.size() >= x.minValues : true)' - resources: - description: Resources models the resource requirements for the NodeClaim to launch - properties: - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: Requests describes the minimum required resources for the NodeClaim to launch - type: object - type: object - maxProperties: 0 - startupTaints: - description: |- - StartupTaints are taints that are applied to nodes upon startup which are expected to be removed automatically - within a short period of time, typically by a DaemonSet that tolerates the taint. These are commonly used by - daemonsets to allow initialization and enforce startup ordering. StartupTaints are ignored for provisioning - purposes in that pods are not required to tolerate a StartupTaint in order to have nodes provisioned for them. - items: - description: |- - The node this Taint is attached to has the "effect" on - any pod that does not tolerate the Taint. - properties: - effect: - description: |- - Required. The effect of the taint on pods - that do not tolerate the taint. - Valid effects are NoSchedule, PreferNoSchedule and NoExecute. - type: string - enum: - - NoSchedule - - PreferNoSchedule - - NoExecute - key: - description: Required. The taint key to be applied to a node. - type: string - minLength: 1 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ - timeAdded: - description: |- - TimeAdded represents the time at which the taint was added. - It is only written for NoExecute taints. - format: date-time - type: string - value: - description: The taint value corresponding to the taint key. - type: string - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ - required: - - effect - - key - type: object - type: array - taints: - description: Taints will be applied to the NodeClaim's node. - items: - description: |- - The node this Taint is attached to has the "effect" on - any pod that does not tolerate the Taint. - properties: - effect: - description: |- - Required. The effect of the taint on pods - that do not tolerate the taint. - Valid effects are NoSchedule, PreferNoSchedule and NoExecute. - type: string - enum: - - NoSchedule - - PreferNoSchedule - - NoExecute - key: - description: Required. The taint key to be applied to a node. - type: string - minLength: 1 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ - timeAdded: - description: |- - TimeAdded represents the time at which the taint was added. - It is only written for NoExecute taints. - format: date-time - type: string - value: - description: The taint value corresponding to the taint key. - type: string - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ - required: - - effect - - key - type: object - type: array - required: - - nodeClassRef - - requirements - type: object - required: - - spec - type: object - weight: - description: |- - Weight is the priority given to the nodepool during scheduling. A higher - numerical weight indicates that this nodepool will be ordered - ahead of other nodepools with lower weights. A nodepool with no weight - will be treated as if it is a nodepool with a weight of 0. - format: int32 - maximum: 100 - minimum: 1 - type: integer - required: - - template - type: object - status: - description: NodePoolStatus defines the observed state of NodePool - properties: - conditions: - description: Conditions contains signals for health and readiness - items: - description: Condition aliases the upstream type and adds additional helper methods - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - resources: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: Resources is the list of resources that have been provisioned. - type: object - type: object - required: - - spec - type: object - served: true - storage: false - subresources: - status: {} conversion: strategy: Webhook webhook: From a53cdd9cb78742ad9a7bf8b5e4cd1db716e71033 Mon Sep 17 00:00:00 2001 From: Alex Leites <18728999+tallaxes@users.noreply.github.com> Date: Tue, 3 Dec 2024 02:47:06 +0000 Subject: [PATCH 02/19] chore: update main and alt operator --- cmd/controller/main.go | 5 +- cmd/controller/main_ccp.go | 40 +------- .../karpenter-core/pkg/operator/operator.go | 94 ++++++++++--------- 3 files changed, 50 insertions(+), 89 deletions(-) diff --git a/cmd/controller/main.go b/cmd/controller/main.go index 24d9b5207..0eb292c15 100644 --- a/cmd/controller/main.go +++ b/cmd/controller/main.go @@ -28,7 +28,6 @@ import ( "sigs.k8s.io/karpenter/pkg/cloudprovider/metrics" corecontrollers "sigs.k8s.io/karpenter/pkg/controllers" coreoperator "sigs.k8s.io/karpenter/pkg/operator" - corewebhooks "sigs.k8s.io/karpenter/pkg/webhooks" ) func main() { @@ -53,7 +52,6 @@ func main() { op.EventRecorder, cloudProvider, )...). - WithWebhooks(ctx, corewebhooks.NewWebhooks()...). WithControllers(ctx, controllers.NewControllers( ctx, op.Manager, @@ -62,6 +60,5 @@ func main() { aksCloudProvider, op.InstanceProvider, )...). - //WithWebhooks(ctx, webhooks.NewWebhooks()...). - Start(ctx, cloudProvider) + Start(ctx) } diff --git a/cmd/controller/main_ccp.go b/cmd/controller/main_ccp.go index 6d95d2882..9ea232f53 100644 --- a/cmd/controller/main_ccp.go +++ b/cmd/controller/main_ccp.go @@ -19,23 +19,11 @@ limitations under the License. package main import ( - "context" - "github.com/samber/lo" - "go.uber.org/zap" - "knative.dev/pkg/logging" // Injection stuff - kubeclientinjection "knative.dev/pkg/client/injection/kube/client" - "knative.dev/pkg/configmap" - "knative.dev/pkg/controller" - knativeinjection "knative.dev/pkg/injection" - secretinformer "knative.dev/pkg/injection/clients/namespacedkube/informers/core/v1/secret" - kubeinformerfactory "knative.dev/pkg/injection/clients/namespacedkube/informers/factory" - "knative.dev/pkg/webhook/certificates" altOperator "github.com/Azure/karpenter-provider-azure/pkg/alt/karpenter-core/pkg/operator" - altwebhooks "github.com/Azure/karpenter-provider-azure/pkg/alt/karpenter-core/pkg/webhooks" "github.com/Azure/karpenter-provider-azure/pkg/cloudprovider" controllers "github.com/Azure/karpenter-provider-azure/pkg/controllers" "github.com/Azure/karpenter-provider-azure/pkg/operator" @@ -43,31 +31,6 @@ import ( corecontrollers "sigs.k8s.io/karpenter/pkg/controllers" ) -func newWebhooks(ctx context.Context) []knativeinjection.ControllerConstructor { - client := altOperator.GetCCPClient(ctx) - ccpInformerFactory := kubeinformerfactory.Get(ctx) - - secretInformer := ccpInformerFactory.Core().V1().Secrets() - ctx = context.WithValue(ctx, secretinformer.Key{}, secretInformer) - - logging.FromContext(ctx).Info("Starting horrible CCP informer") - if err := controller.StartInformers(ctx.Done(), secretInformer.Informer()); err != nil { - logging.FromContext(ctx).Fatalw("Failed to start horrible CCP informer", zap.Error(err)) - } - - return []knativeinjection.ControllerConstructor{ - func(ctx context.Context, watcher configmap.Watcher) *controller.Impl { - ctx = context.WithValue(ctx, secretinformer.Key{}, secretInformer) - ctx = context.WithValue(ctx, kubeclientinjection.Key{}, client) - return certificates.NewController(ctx, watcher) - }, - func(ctx context.Context, watcher configmap.Watcher) *controller.Impl { - ctx = context.WithValue(ctx, secretinformer.Key{}, secretInformer) - return altwebhooks.NewCRDConversionWebhook(ctx, watcher) - }, - } -} - func main() { //ctx, op := operator.NewOperator(coreoperator.NewOperator()) ctx, op := operator.NewOperator(altOperator.NewOperator()) @@ -91,7 +54,6 @@ func main() { op.EventRecorder, cloudProvider, )...). - WithWebhooks(ctx, newWebhooks(ctx)...). WithControllers(ctx, controllers.NewControllers( ctx, op.Manager, @@ -100,5 +62,5 @@ func main() { aksCloudProvider, op.InstanceProvider, )...). - Start(ctx, cloudProvider) + Start(ctx) } diff --git a/pkg/alt/karpenter-core/pkg/operator/operator.go b/pkg/alt/karpenter-core/pkg/operator/operator.go index c75c8b59d..f56132eaf 100644 --- a/pkg/alt/karpenter-core/pkg/operator/operator.go +++ b/pkg/alt/karpenter-core/pkg/operator/operator.go @@ -42,24 +42,22 @@ import ( "sigs.k8s.io/karpenter/pkg/operator/injection" "sigs.k8s.io/karpenter/pkg/operator/options" - "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" "k8s.io/client-go/tools/leaderelection/resourcelock" "k8s.io/client-go/util/flowcontrol" "k8s.io/utils/clock" - "knative.dev/pkg/controller" knativeinjection "knative.dev/pkg/injection" - kubeinformerfactory "knative.dev/pkg/injection/clients/namespacedkube/informers/factory" "knative.dev/pkg/signals" "knative.dev/pkg/system" - "knative.dev/pkg/webhook" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" "sigs.k8s.io/controller-runtime/pkg/healthz" + "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/karpenter/pkg/operator/logging" @@ -128,14 +126,6 @@ func NewOperator() (context.Context, *coreoperator.Operator) { log.SetLogger(logger) klog.SetLogger(logger) - // Webhook - ccPlaneCtx = webhook.WithOptions(ccPlaneCtx, webhook.Options{ - Port: options.FromContext(overlayCtx).WebhookPort, - ServiceName: options.FromContext(overlayCtx).ServiceName, - SecretName: fmt.Sprintf("%s-cert", options.FromContext(overlayCtx).ServiceName), - GracePeriod: 5 * time.Second, - }) - // Client Config ccPlaneConfig := lo.Must(rest.InClusterConfig()) overlayConfig := ctrl.GetConfigOrDie() @@ -150,24 +140,14 @@ func NewOperator() (context.Context, *coreoperator.Operator) { ccpKubernetesInterface := kubernetes.NewForConfigOrDie(ccPlaneConfig) ccPlaneCtx = withCCPClient(ccPlaneCtx, ccpKubernetesInterface) - ccpInformerFactory := informers.NewSharedInformerFactoryWithOptions( - ccpKubernetesInterface, - controller.GetResyncPeriod(ccPlaneCtx), - // This factory scopes things to the system namespace. - informers.WithNamespace(system.Namespace())) - // Also override the kubeinformerfactory because it's used by webhook.New() to construct - // a new secret client, see: https://github.com/knative/pkg/blob/main/webhook/webhook.go#L183 - ccPlaneCtx = context.WithValue(ccPlaneCtx, kubeinformerfactory.Key{}, ccpInformerFactory) - // Manager mgrOpts := ctrl.Options{ Logger: logging.IgnoreDebugEvents(logger), LeaderElection: !options.FromContext(overlayCtx).DisableLeaderElection, - LeaderElectionID: "karpenter-leader-election", - LeaderElectionResourceLock: resourcelock.LeasesResourceLock, + LeaderElectionID: options.FromContext(overlayCtx).LeaderElectionName, LeaderElectionNamespace: system.Namespace(), + LeaderElectionResourceLock: resourcelock.LeasesResourceLock, LeaderElectionReleaseOnCancel: true, - Scheme: scheme.Scheme, Metrics: server.Options{ BindAddress: fmt.Sprintf(":%d", options.FromContext(overlayCtx).MetricsPort), }, @@ -202,15 +182,49 @@ func NewOperator() (context.Context, *coreoperator.Operator) { mgr, err := ctrl.NewManager(overlayConfig, mgrOpts) mgr = lo.Must(mgr, err, "failed to setup manager") - lo.Must0(mgr.GetFieldIndexer().IndexField(overlayCtx, &corev1.Pod{}, "spec.nodeName", func(o client.Object) []string { + setupIndexers(overlayCtx, ccPlaneCtx, mgr) + + lo.Must0(mgr.AddReadyzCheck("manager", func(req *http.Request) error { + return lo.Ternary(mgr.GetCache().WaitForCacheSync(req.Context()), nil, fmt.Errorf("failed to sync caches")) + })) + lo.Must0(mgr.AddReadyzCheck("crd", func(_ *http.Request) error { + objects := []client.Object{&v1.NodePool{}, &v1.NodeClaim{}} + for _, obj := range objects { + gvk, err := apiutil.GVKForObject(obj, scheme.Scheme) + if err != nil { + return err + } + if _, err := mgr.GetRESTMapper().RESTMapping(gvk.GroupKind(), gvk.Version); err != nil { + return err + } + } + return nil + })) + lo.Must0(mgr.AddHealthzCheck("healthz", healthz.Ping)) + lo.Must0(mgr.AddReadyzCheck("readyz", healthz.Ping)) + + return ccPlaneCtx, &coreoperator.Operator{ + Manager: mgr, + KubernetesInterface: overlayKubernetesInterface, + EventRecorder: events.NewRecorder(mgr.GetEventRecorderFor(appName)), + Clock: clock.RealClock{}, + } +} + +func setupIndexers(ctx context.Context, ccPlaneCtx context.Context, mgr manager.Manager) { + lo.Must0(mgr.GetFieldIndexer().IndexField(ctx, &corev1.Pod{}, "spec.nodeName", func(o client.Object) []string { return []string{o.(*corev1.Pod).Spec.NodeName} }), "failed to setup pod indexer") - lo.Must0(mgr.GetFieldIndexer().IndexField(overlayCtx, &corev1.Node{}, "spec.providerID", func(o client.Object) []string { + lo.Must0(mgr.GetFieldIndexer().IndexField(ctx, &corev1.Node{}, "spec.providerID", func(o client.Object) []string { return []string{o.(*corev1.Node).Spec.ProviderID} }), "failed to setup node provider id indexer") + lo.Must0(mgr.GetFieldIndexer().IndexField(ctx, &storagev1.VolumeAttachment{}, "spec.nodeName", func(o client.Object) []string { + return []string{o.(*storagev1.VolumeAttachment).Spec.NodeName} + }), "failed to setup volumeattachment indexer") + lo.Must0(func() error { _, _, err := lo.AttemptWithDelay(42, 10*time.Second, func(index int, duration time.Duration) error { - err := mgr.GetFieldIndexer().IndexField(overlayCtx, &v1.NodeClaim{}, "status.providerID", func(o client.Object) []string { + err := mgr.GetFieldIndexer().IndexField(ctx, &v1.NodeClaim{}, "status.providerID", func(o client.Object) []string { return []string{o.(*v1.NodeClaim).Status.ProviderID} }) if err != nil { @@ -221,35 +235,23 @@ func NewOperator() (context.Context, *coreoperator.Operator) { return err }(), "failed to setup nodeclaim provider id indexer, all attempts used") - lo.Must0(mgr.GetFieldIndexer().IndexField(overlayCtx, &v1.NodeClaim{}, "spec.nodeClassRef.group", func(o client.Object) []string { + lo.Must0(mgr.GetFieldIndexer().IndexField(ctx, &v1.NodeClaim{}, "spec.nodeClassRef.group", func(o client.Object) []string { return []string{o.(*v1.NodeClaim).Spec.NodeClassRef.Group} }), "failed to setup nodeclaim nodeclassref apiversion indexer") - lo.Must0(mgr.GetFieldIndexer().IndexField(overlayCtx, &v1.NodeClaim{}, "spec.nodeClassRef.kind", func(o client.Object) []string { + lo.Must0(mgr.GetFieldIndexer().IndexField(ctx, &v1.NodeClaim{}, "spec.nodeClassRef.kind", func(o client.Object) []string { return []string{o.(*v1.NodeClaim).Spec.NodeClassRef.Kind} }), "failed to setup nodeclaim nodeclassref kind indexer") - lo.Must0(mgr.GetFieldIndexer().IndexField(overlayCtx, &v1.NodeClaim{}, "spec.nodeClassRef.name", func(o client.Object) []string { + lo.Must0(mgr.GetFieldIndexer().IndexField(ctx, &v1.NodeClaim{}, "spec.nodeClassRef.name", func(o client.Object) []string { return []string{o.(*v1.NodeClaim).Spec.NodeClassRef.Name} }), "failed to setup nodeclaim nodeclassref name indexer") - lo.Must0(mgr.GetFieldIndexer().IndexField(overlayCtx, &v1.NodePool{}, "spec.template.spec.nodeClassRef.group", func(o client.Object) []string { + + lo.Must0(mgr.GetFieldIndexer().IndexField(ctx, &v1.NodePool{}, "spec.template.spec.nodeClassRef.group", func(o client.Object) []string { return []string{o.(*v1.NodePool).Spec.Template.Spec.NodeClassRef.Group} }), "failed to setup nodepool nodeclassref apiversion indexer") - lo.Must0(mgr.GetFieldIndexer().IndexField(overlayCtx, &v1.NodePool{}, "spec.template.spec.nodeClassRef.kind", func(o client.Object) []string { + lo.Must0(mgr.GetFieldIndexer().IndexField(ctx, &v1.NodePool{}, "spec.template.spec.nodeClassRef.kind", func(o client.Object) []string { return []string{o.(*v1.NodePool).Spec.Template.Spec.NodeClassRef.Kind} }), "failed to setup nodepool nodeclassref kind indexer") - lo.Must0(mgr.GetFieldIndexer().IndexField(overlayCtx, &v1.NodePool{}, "spec.template.spec.nodeClassRef.name", func(o client.Object) []string { + lo.Must0(mgr.GetFieldIndexer().IndexField(ctx, &v1.NodePool{}, "spec.template.spec.nodeClassRef.name", func(o client.Object) []string { return []string{o.(*v1.NodePool).Spec.Template.Spec.NodeClassRef.Name} }), "failed to setup nodepool nodeclassref name indexer") - lo.Must0(mgr.GetFieldIndexer().IndexField(overlayCtx, &storagev1.VolumeAttachment{}, "spec.nodeName", func(o client.Object) []string { - return []string{o.(*storagev1.VolumeAttachment).Spec.NodeName} - }), "failed to setup volumeattachment indexer") - - lo.Must0(mgr.AddHealthzCheck("healthz", healthz.Ping)) - lo.Must0(mgr.AddReadyzCheck("readyz", healthz.Ping)) - - return ccPlaneCtx, &coreoperator.Operator{ - Manager: mgr, - KubernetesInterface: overlayKubernetesInterface, - EventRecorder: events.NewRecorder(mgr.GetEventRecorderFor(appName)), - Clock: clock.RealClock{}, - } } From abc980723a2dd97d5b56e51b25ab7591456e9979 Mon Sep 17 00:00:00 2001 From: Alex Leites <18728999+tallaxes@users.noreply.github.com> Date: Tue, 3 Dec 2024 02:51:14 +0000 Subject: [PATCH 03/19] chore: remove webhooks wiring --- pkg/alt/karpenter-core/pkg/operator/logger.go | 59 -- pkg/alt/karpenter-core/pkg/webhooks/README.md | 4 - .../karpenter-core/pkg/webhooks/webhooks.go | 205 ------ .../resourcesemantics/conversion/README.md | 10 - .../conversion/controller.go | 159 ----- .../conversion/conversion.go | 206 ------ .../conversion/conversion_test.go | 630 ------------------ .../conversion/internal/types.go | 278 -------- .../internal/zz_generated.deepcopy.go | 132 ---- .../resourcesemantics/conversion/options.go | 46 -- .../conversion/options_test.go | 38 -- .../conversion/reconciler.go | 123 ---- .../conversion/table_test.go | 290 -------- 13 files changed, 2180 deletions(-) delete mode 100644 pkg/alt/karpenter-core/pkg/operator/logger.go delete mode 100644 pkg/alt/karpenter-core/pkg/webhooks/README.md delete mode 100644 pkg/alt/karpenter-core/pkg/webhooks/webhooks.go delete mode 100644 pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/README.md delete mode 100644 pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/controller.go delete mode 100644 pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/conversion.go delete mode 100644 pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/conversion_test.go delete mode 100644 pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/internal/types.go delete mode 100644 pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/internal/zz_generated.deepcopy.go delete mode 100644 pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/options.go delete mode 100644 pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/options_test.go delete mode 100644 pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/reconciler.go delete mode 100644 pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/table_test.go diff --git a/pkg/alt/karpenter-core/pkg/operator/logger.go b/pkg/alt/karpenter-core/pkg/operator/logger.go deleted file mode 100644 index 5e184b6b4..000000000 --- a/pkg/alt/karpenter-core/pkg/operator/logger.go +++ /dev/null @@ -1,59 +0,0 @@ -/* -Portions Copyright (c) Microsoft Corporation. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Unmodified copy of ignoreDebugEventsSink from -// https://github.com/kubernetes-sigs/karpenter/blob/v0.30.0/pkg/operator/logger.go - -package operator - -import ( - "github.com/go-logr/logr" -) - -type ignoreDebugEventsSink struct { - name string - sink logr.LogSink -} - -func (i ignoreDebugEventsSink) Init(ri logr.RuntimeInfo) { - i.sink.Init(ri) -} - -func (i ignoreDebugEventsSink) Enabled(level int) bool { return i.sink.Enabled(level) } - -func (i ignoreDebugEventsSink) Info(level int, msg string, keysAndValues ...interface{}) { - // ignore debug "events" logs - if level == 1 && i.name == "events" { - return - } - i.sink.Info(level, msg, keysAndValues...) -} - -func (i ignoreDebugEventsSink) Error(err error, msg string, keysAndValues ...interface{}) { - i.sink.Error(err, msg, keysAndValues...) -} - -func (i ignoreDebugEventsSink) WithValues(keysAndValues ...interface{}) logr.LogSink { - return i.sink.WithValues(keysAndValues...) -} - -func (i ignoreDebugEventsSink) WithName(name string) logr.LogSink { - return &ignoreDebugEventsSink{name: name, sink: i.sink.WithName(name)} -} - -func ignoreDebugEvents(logger logr.Logger) logr.Logger { - return logr.New(&ignoreDebugEventsSink{sink: logger.GetSink()}) -} diff --git a/pkg/alt/karpenter-core/pkg/webhooks/README.md b/pkg/alt/karpenter-core/pkg/webhooks/README.md deleted file mode 100644 index 6236d4dd8..000000000 --- a/pkg/alt/karpenter-core/pkg/webhooks/README.md +++ /dev/null @@ -1,4 +0,0 @@ -This is copied from https://github.com/kubernetes-sigs/karpenter/blob/v1.0.4/pkg/webhooks/webhooks.go - -Some modifications have been made to cater to the deployment model in AKS (multiple API servers). -Look for the sections that start with `// AKS customized` to understand the changes. diff --git a/pkg/alt/karpenter-core/pkg/webhooks/webhooks.go b/pkg/alt/karpenter-core/pkg/webhooks/webhooks.go deleted file mode 100644 index 6fef858b2..000000000 --- a/pkg/alt/karpenter-core/pkg/webhooks/webhooks.go +++ /dev/null @@ -1,205 +0,0 @@ -/* -Portions Copyright (c) Microsoft Corporation. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -package webhooks - -import ( - "context" - "crypto/tls" - "errors" - "fmt" - "io" - "net/http" - "strings" - - "github.com/Azure/karpenter-provider-azure/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion" - "github.com/awslabs/operatorpkg/object" - "github.com/samber/lo" - "go.uber.org/zap" - "golang.org/x/sync/errgroup" - "k8s.io/apimachinery/pkg/runtime/schema" - informers "k8s.io/client-go/informers" - "k8s.io/client-go/rest" - "knative.dev/pkg/configmap" - "knative.dev/pkg/controller" - knativeinjection "knative.dev/pkg/injection" - kubeinformerfactory "knative.dev/pkg/injection/clients/namespacedkube/informers/factory" - "knative.dev/pkg/injection/sharedmain" - knativelogging "knative.dev/pkg/logging" - "knative.dev/pkg/metrics" - "knative.dev/pkg/webhook" - "knative.dev/pkg/webhook/certificates" - "sigs.k8s.io/controller-runtime/pkg/healthz" - - v1 "sigs.k8s.io/karpenter/pkg/apis/v1" - "sigs.k8s.io/karpenter/pkg/apis/v1beta1" - "sigs.k8s.io/karpenter/pkg/operator/injection" - "sigs.k8s.io/karpenter/pkg/operator/logging" - "sigs.k8s.io/karpenter/pkg/operator/options" -) - -const component = "webhook" - -var ( - // TODO: Remove conversion webhooks once support for the v1beta1 APIs is dropped - ConversionResource = map[schema.GroupKind]conversion.GroupKindConversion{ - object.GVK(&v1beta1.NodePool{}).GroupKind(): { - DefinitionName: "nodepools.karpenter.sh", - HubVersion: "v1", - Zygotes: map[string]conversion.ConvertibleObject{ - "v1": &v1.NodePool{}, - "v1beta1": &v1beta1.NodePool{}, - }, - }, - object.GVK(&v1beta1.NodeClaim{}).GroupKind(): { - DefinitionName: "nodeclaims.karpenter.sh", - HubVersion: "v1", - Zygotes: map[string]conversion.ConvertibleObject{ - "v1": &v1.NodeClaim{}, - "v1beta1": &v1beta1.NodeClaim{}, - }, - }, - } -) - -func NewWebhooks() []knativeinjection.ControllerConstructor { - return []knativeinjection.ControllerConstructor{ - certificates.NewController, - NewCRDConversionWebhook, - } -} - -func NewCRDConversionWebhook(ctx context.Context, _ configmap.Watcher) *controller.Impl { - nodeclassCtx := injection.GetNodeClasses(ctx) - client := injection.GetClient(ctx) - return conversion.NewConversionController( - ctx, - "/conversion/karpenter.sh", - ConversionResource, - func(ctx context.Context) context.Context { - return injection.WithClient(injection.WithNodeClasses(ctx, nodeclassCtx), client) - }, - ) -} - -// Start copies the relevant portions for starting the webhooks from sharedmain.MainWithConfig -// https://github.com/knative/pkg/blob/0f52db700d63/injection/sharedmain/main.go#L227 -func Start(ctx context.Context, cfg *rest.Config, ctors ...knativeinjection.ControllerConstructor) { - logger := logging.NewLogger(ctx, component).Sugar() - ctx = knativelogging.WithLogger(ctx, logger) - - // AKS customized - // This section is customized for AKS use-case to allow overriding the informer factory. - // Overriding the informer factory is required to make webhooks work, as it is used - // in webhook.New() to construct a new secret client, see: - // https://github.com/knative/pkg/blob/main/webhook/webhook.go#L183 - var overriddenInformerFactory informers.SharedInformerFactory - if ctx.Value(kubeinformerfactory.Key{}) != nil { - overriddenInformerFactory = kubeinformerfactory.Get(ctx) - } - // end customization - - ctx, startInformers := knativeinjection.EnableInjectionOrDie(ctx, cfg) - cmw := sharedmain.SetupConfigMapWatchOrDie(ctx, logger) - controllers, webhooks := sharedmain.ControllersAndWebhooksFromCtors(ctx, cmw, ctors...) - - // AKS customized - if overriddenInformerFactory != nil { - ctx = context.WithValue(ctx, kubeinformerfactory.Key{}, overriddenInformerFactory) - } - // end customization - - // Many of the webhooks rely on configuration, e.g. configurable defaults, feature flags. - // So make sure that we have synchronized our configuration state before launching the - // webhooks, so that things are properly initialized. - logger.Info("Starting configuration manager...") - if err := cmw.Start(ctx.Done()); err != nil { - logger.Fatalw("Failed to start configuration manager", zap.Error(err)) - } - - // If we have one or more admission controllers, then start the webhook - // and pass them in. - var wh *webhook.Webhook - var err error - eg, egCtx := errgroup.WithContext(ctx) - if len(webhooks) > 0 { - // Update the metric exporter to point to a prometheus endpoint - lo.Must0(metrics.UpdateExporter(ctx, metrics.ExporterOptions{ - Component: strings.ReplaceAll(component, "-", "_"), - ConfigMap: lo.Must(metrics.NewObservabilityConfigFromConfigMap(nil)).GetConfigMap().Data, - Secrets: sharedmain.SecretFetcher(ctx), - PrometheusPort: options.FromContext(ctx).WebhookMetricsPort, - }, logger)) - // Register webhook metrics - webhook.RegisterMetrics() - - wh, err = webhook.New(ctx, webhooks) - if err != nil { - logger.Fatalw("Failed to create webhook", zap.Error(err)) - } - eg.Go(func() error { - return wh.Run(ctx.Done()) - }) - } - - // Start the injection clients and informers. - startInformers() - - // Wait for webhook informers to sync. - if wh != nil { - wh.InformersHaveSynced() - } - logger.Info("Starting controllers...") - eg.Go(func() error { - return controller.StartAll(ctx, controllers...) - }) - // This will block until either a signal arrives or one of the grouped functions - // returns an error. - <-egCtx.Done() - - // Don't forward ErrServerClosed as that indicates we're already shutting down. - if err := eg.Wait(); err != nil && !errors.Is(err, http.ErrServerClosed) { - logger.Errorw("Error while running server", zap.Error(err)) - } -} - -func HealthProbe(ctx context.Context) healthz.Checker { - // Create new transport that doesn't validate the TLS certificate - // This transport is just polling so validating the server certificate isn't necessary - transport := http.DefaultTransport.(*http.Transport).Clone() - transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} // nolint:gosec - client := &http.Client{Transport: transport} - - // TODO: Add knative health check port for webhooks when health port can be configured - // Issue: https://github.com/knative/pkg/issues/2765 - return func(req *http.Request) (err error) { - res, err := client.Get(fmt.Sprintf("https://localhost:%d", options.FromContext(ctx).WebhookPort)) - // If the webhook connection errors out, liveness/readiness should fail - if err != nil { - return err - } - // Close the body to avoid leaking file descriptors - // Always read the body so we can re-use the connection: https://stackoverflow.com/questions/17948827/reusing-http-connections-in-go - _, _ = io.ReadAll(res.Body) - res.Body.Close() - - // If there is a server-side error or path not found, - // consider liveness to have failed - if res.StatusCode >= 500 || res.StatusCode == 404 { - return fmt.Errorf("webhook probe failed with status code %d", res.StatusCode) - } - return nil - } -} diff --git a/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/README.md b/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/README.md deleted file mode 100644 index 7424c3768..000000000 --- a/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/README.md +++ /dev/null @@ -1,10 +0,0 @@ -This code is all copied from https://github.com/knative/pkg/tree/main/webhook/resourcesemantics/conversion -at commit fdbc0b5adde7 (this is the commit used by our knative dependency v0.0.0-20240910170930-fdbc0b5adde7 -as of Oct 23, 2024). - -The only changes were made to reconciler.go to support `crd.spec.conversion.webhook.clientConfig.url` in addition -to `crd.spec.conversion.webhook.clientConfig.service`. - -Look for the sections that start with `// AKS customized` to understand the changes. - -Also note that the zz_generated.deepcopy.go isn't currently actually autogenerated it's just copied from upstream. diff --git a/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/controller.go b/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/controller.go deleted file mode 100644 index fbb9a8e2c..000000000 --- a/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/controller.go +++ /dev/null @@ -1,159 +0,0 @@ -// Portions Copyright (c) Microsoft Corporation. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package conversion - -import ( - "context" - - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/tools/cache" - "knative.dev/pkg/apis" - apixclient "knative.dev/pkg/client/injection/apiextensions/client" - crdinformer "knative.dev/pkg/client/injection/apiextensions/informers/apiextensions/v1/customresourcedefinition" - "knative.dev/pkg/controller" - secretinformer "knative.dev/pkg/injection/clients/namespacedkube/informers/core/v1/secret" - "knative.dev/pkg/logging" - pkgreconciler "knative.dev/pkg/reconciler" - "knative.dev/pkg/system" - "knative.dev/pkg/webhook" -) - -// ConvertibleObject defines the functionality our API types -// are required to implement in order to be convertible from -// one version to another -// -// Optionally if the object implements apis.Defaultable the -// ConversionController will apply defaults before returning -// the response -type ConvertibleObject interface { - // ConvertTo(ctx, to) - // ConvertFrom(ctx, from) - apis.Convertible - - // DeepCopyObject() - // GetObjectKind() => SetGroupVersionKind(gvk) - runtime.Object -} - -// GroupKindConversion specifies how a specific Kind for a given -// group should be converted -type GroupKindConversion struct { - // DefinitionName specifies the CustomResourceDefinition that should - // be reconciled with by the controller. - // - // The conversion webhook configuration will be updated - // when the CA bundle changes - DefinitionName string - - // HubVersion specifies which version of the CustomResource supports - // conversions to and from all types - // - // It is expected that the Zygotes map contains an entry for the - // specified HubVersion - HubVersion string - - // Zygotes contains a map of version strings (ie. v1, v2) to empty - // ConvertibleObject objects - // - // During a conversion request these zygotes will be deep copied - // and manipulated using the apis.Convertible interface - Zygotes map[string]ConvertibleObject -} - -// NewConversionController returns a K8s controller that will -// will reconcile CustomResourceDefinitions and update their -// conversion webhook attributes such as path & CA bundle. -// -// Additionally the controller's Reconciler implements -// webhook.ConversionController for the purposes of converting -// resources between different versions -func NewConversionController( - ctx context.Context, - path string, - kinds map[schema.GroupKind]GroupKindConversion, - withContext func(context.Context) context.Context, -) *controller.Impl { - opts := []OptionFunc{ - WithPath(path), - WithWrapContext(withContext), - WithKinds(kinds), - } - - return newController(ctx, opts...) -} - -func newController(ctx context.Context, optsFunc ...OptionFunc) *controller.Impl { - secretInformer := secretinformer.Get(ctx) - crdInformer := crdinformer.Get(ctx) - client := apixclient.Get(ctx) - woptions := webhook.GetOptions(ctx) - - opts := &options{} - - for _, f := range optsFunc { - f(opts) - } - - r := &reconciler{ - LeaderAwareFuncs: pkgreconciler.LeaderAwareFuncs{ - // Have this reconciler enqueue our types whenever it becomes leader. - PromoteFunc: func(bkt pkgreconciler.Bucket, enq func(pkgreconciler.Bucket, types.NamespacedName)) error { - for _, gkc := range opts.kinds { - name := gkc.DefinitionName - enq(bkt, types.NamespacedName{Name: name}) - } - return nil - }, - }, - - kinds: opts.kinds, - path: opts.path, - secretName: woptions.SecretName, - withContext: opts.wc, - - client: client, - secretLister: secretInformer.Lister(), - crdLister: crdInformer.Lister(), - } - - logger := logging.FromContext(ctx) - controllerOptions := woptions.ControllerOptions - if controllerOptions == nil { - const queueName = "ConversionWebhook" - controllerOptions = &controller.ControllerOptions{WorkQueueName: queueName, Logger: logger.Named(queueName)} - } - c := controller.NewContext(ctx, r, *controllerOptions) - - // Reconciler when the named CRDs change. - for _, gkc := range opts.kinds { - name := gkc.DefinitionName - - crdInformer.Informer().AddEventHandler(cache.FilteringResourceEventHandler{ - FilterFunc: controller.FilterWithName(name), - Handler: controller.HandleAll(c.Enqueue), - }) - - sentinel := c.EnqueueSentinel(types.NamespacedName{Name: name}) - - // Reconcile when the cert bundle changes. - secretInformer.Informer().AddEventHandler(cache.FilteringResourceEventHandler{ - FilterFunc: controller.FilterWithNameAndNamespace(system.Namespace(), woptions.SecretName), - Handler: controller.HandleAll(sentinel), - }) - } - - return c -} diff --git a/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/conversion.go b/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/conversion.go deleted file mode 100644 index 761acf81c..000000000 --- a/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/conversion.go +++ /dev/null @@ -1,206 +0,0 @@ -/* -Portions Copyright (c) Microsoft Corporation. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package conversion - -import ( - "context" - "encoding/json" - "fmt" - - "go.uber.org/zap" - - apixv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - - "knative.dev/pkg/apis" - "knative.dev/pkg/kmeta" - "knative.dev/pkg/logging" - "knative.dev/pkg/logging/logkey" -) - -// Convert implements webhook.ConversionController -func (r *reconciler) Convert( - ctx context.Context, - req *apixv1.ConversionRequest, -) *apixv1.ConversionResponse { - if r.withContext != nil { - ctx = r.withContext(ctx) - } - - res := &apixv1.ConversionResponse{ - UID: req.UID, - Result: metav1.Status{ - Status: metav1.StatusSuccess, - }, - } - - result := make([]runtime.RawExtension, 0, len(req.Objects)) - - for _, obj := range req.Objects { - converted, err := r.convert(ctx, obj, req.DesiredAPIVersion) - if err != nil { - logging.FromContext(ctx).Errorw("Conversion failed", zap.Error(err)) - res.Result.Status = metav1.StatusFailure - res.Result.Message = err.Error() - break - } - - result = append(result, converted) - } - - res.ConvertedObjects = result - return res -} - -func (r *reconciler) convert( - ctx context.Context, - inRaw runtime.RawExtension, - targetVersion string, -) (runtime.RawExtension, error) { - logger := logging.FromContext(ctx) - var ret runtime.RawExtension - - inGVK, err := parseGVK(inRaw) - if err != nil { - return ret, err - } - - inGK := inGVK.GroupKind() - conv, ok := r.kinds[inGK] - if !ok { - return ret, fmt.Errorf("no conversion support for type %s", formatGK(inGVK.GroupKind())) - } - - outGVK, err := parseAPIVersion(targetVersion, inGK.Kind) - if err != nil { - return ret, err - } - - inZygote, ok := conv.Zygotes[inGVK.Version] - if !ok { - return ret, fmt.Errorf("conversion not supported for type %s", formatGVK(inGVK)) - } - outZygote, ok := conv.Zygotes[outGVK.Version] - if !ok { - return ret, fmt.Errorf("conversion not supported for type %s", formatGVK(outGVK)) - } - hubZygote, ok := conv.Zygotes[conv.HubVersion] - if !ok { - return ret, fmt.Errorf("conversion not supported for type %s", formatGK(inGVK.GroupKind())) - } - - in := inZygote.DeepCopyObject().(ConvertibleObject) - hub := hubZygote.DeepCopyObject().(ConvertibleObject) - out := outZygote.DeepCopyObject().(ConvertibleObject) - - hubGVK := inGVK.GroupKind().WithVersion(conv.HubVersion) - - logger = logger.With( - zap.String("inputType", formatGVK(inGVK)), - zap.String("outputType", formatGVK(outGVK)), - zap.String("hubType", formatGVK(hubGVK)), - ) - - // TODO(dprotaso) - potentially error on unknown fields - if err = json.Unmarshal(inRaw.Raw, &in); err != nil { - return ret, fmt.Errorf("unable to unmarshal input: %w", err) - } - - if acc, err := kmeta.DeletionHandlingAccessor(in); err == nil { - // TODO: right now we don't convert any non-namespaced objects. If we ever do that - // this needs to updated to deal with it. - logger = logger.With(zap.String(logkey.Key, acc.GetNamespace()+"/"+acc.GetName())) - } else { - logger.Infof("Could not get Accessor for %s: %v", formatGK(inGVK.GroupKind()), err) - } - ctx = logging.WithLogger(ctx, logger) - - if inGVK.Version == conv.HubVersion { - hub = in - } else if err = hub.ConvertFrom(ctx, in); err != nil { - return ret, fmt.Errorf("conversion failed to version %s for type %s - %w", outGVK.Version, formatGVK(inGVK), err) - } - - if outGVK.Version == conv.HubVersion { - out = hub - } else if err = hub.ConvertTo(ctx, out); err != nil { - return ret, fmt.Errorf("conversion failed to version %s for type %s - %w", outGVK.Version, formatGVK(inGVK), err) - } - - out.GetObjectKind().SetGroupVersionKind(outGVK) - - if defaultable, ok := out.(apis.Defaultable); ok { - defaultable.SetDefaults(ctx) - } - - if ret.Raw, err = json.Marshal(out); err != nil { - return ret, fmt.Errorf("unable to marshal output: %w", err) - } - return ret, nil -} - -func parseGVK(in runtime.RawExtension) (schema.GroupVersionKind, error) { - var ( - typeMeta metav1.TypeMeta - gvk schema.GroupVersionKind - ) - - if err := json.Unmarshal(in.Raw, &typeMeta); err != nil { - return gvk, fmt.Errorf("error parsing type meta %q - %w", string(in.Raw), err) - } - - gv, err := schema.ParseGroupVersion(typeMeta.APIVersion) - if err != nil { - return gvk, fmt.Errorf("error parsing GV %q: %w", typeMeta.APIVersion, err) - } - gvk = gv.WithKind(typeMeta.Kind) - - if gvk.Group == "" || gvk.Version == "" || gvk.Kind == "" { - return gvk, fmt.Errorf("invalid GroupVersionKind %v", gvk) - } - - return gvk, nil -} - -func parseAPIVersion(apiVersion string, kind string) (schema.GroupVersionKind, error) { - gv, err := schema.ParseGroupVersion(apiVersion) - if err != nil { - err = fmt.Errorf("desired API version %q is not valid", apiVersion) - return schema.GroupVersionKind{}, err - } - - if !isValidGV(gv) { - err = fmt.Errorf("desired API version %q is not valid", apiVersion) - return schema.GroupVersionKind{}, err - } - - return gv.WithKind(kind), nil -} - -func formatGVK(gvk schema.GroupVersionKind) string { - return fmt.Sprintf("[kind=%s group=%s version=%s]", gvk.Kind, gvk.Group, gvk.Version) -} - -func formatGK(gk schema.GroupKind) string { - return fmt.Sprintf("[kind=%s group=%s]", gk.Kind, gk.Group) -} - -func isValidGV(gk schema.GroupVersion) bool { - return gk.Group != "" && gk.Version != "" -} diff --git a/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/conversion_test.go b/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/conversion_test.go deleted file mode 100644 index 362d12b3d..000000000 --- a/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/conversion_test.go +++ /dev/null @@ -1,630 +0,0 @@ -/* -Portions Copyright (c) Microsoft Corporation. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package conversion - -import ( - "context" - "encoding/json" - "fmt" - "strings" - "testing" - - // injection - _ "knative.dev/pkg/client/injection/apiextensions/informers/apiextensions/v1/customresourcedefinition/fake" - _ "knative.dev/pkg/injection/clients/namespacedkube/informers/core/v1/secret/fake" - - "github.com/Azure/karpenter-provider-azure/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/internal" - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" - apixv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "knative.dev/pkg/webhook" - - . "knative.dev/pkg/reconciler/testing" -) - -var ( - webhookPath = "/convert" - testGK = schema.GroupKind{ - Group: internal.Group, - Kind: internal.Kind, - } - - zygotes = map[string]ConvertibleObject{ - "v1": &internal.V1Resource{}, - "v2": &internal.V2Resource{}, - "v3": &internal.V3Resource{}, - "error": &internal.ErrorResource{}, - } - - kinds = map[schema.GroupKind]GroupKindConversion{ - testGK: { - DefinitionName: "resource.webhook.pkg.knative.dev", - HubVersion: "v1", - Zygotes: zygotes, - }, - } - - rawOpt = cmp.Transformer("raw", func(res []runtime.RawExtension) []string { - result := make([]string, 0, len(res)) - for _, re := range res { - result = append(result, string(re.Raw)) - } - return result - }) - - cmpOpts = []cmp.Option{ - rawOpt, - } -) - -func testAPIVersion(version string) string { - return testGK.WithVersion(version).GroupVersion().String() -} - -func TestWebhookPath(t *testing.T) { - ctx, _ := SetupFakeContext(t) - ctx = webhook.WithOptions(ctx, webhook.Options{ - SecretName: "webhook-secret", - }) - - controller := NewConversionController(ctx, "/some-path", nil, nil) - conversion := controller.Reconciler.(webhook.ConversionController) - - if got, want := conversion.Path(), "/some-path"; got != want { - t.Errorf("expected controller to return provided path got: %q, want: %q", got, want) - } -} - -func TestConversionToHub(t *testing.T) { - ctx, conversion := newConversion(t) - - req := &apixv1.ConversionRequest{ - UID: "some-uid", - DesiredAPIVersion: testAPIVersion("v1"), - Objects: []runtime.RawExtension{ - toRaw(t, internal.NewV2("bing")), - toRaw(t, internal.NewV3("bang")), - }, - } - - want := &apixv1.ConversionResponse{ - UID: "some-uid", - Result: metav1.Status{Status: metav1.StatusSuccess}, - ConvertedObjects: []runtime.RawExtension{ - toRaw(t, internal.NewV1("bing")), - toRaw(t, internal.NewV1("bang")), - }, - } - - got := conversion.Convert(ctx, req) - if diff := cmp.Diff(want, got, cmpOpts...); diff != "" { - t.Error("unexpected response:", diff) - } -} - -func TestConversionFromHub(t *testing.T) { - tests := []struct { - version string - in runtime.Object - out runtime.Object - }{{ - version: "v2", - in: internal.NewV1("bing"), - out: internal.NewV2("bing"), - }, { - version: "v3", - in: internal.NewV1("bing"), - out: internal.NewV3("bing"), - }} - - for _, test := range tests { - t.Run(test.version, func(t *testing.T) { - ctx, conversion := newConversion(t) - req := &apixv1.ConversionRequest{ - UID: "some-uid", - DesiredAPIVersion: testAPIVersion(test.version), - Objects: []runtime.RawExtension{ - toRaw(t, test.in), - }, - } - - want := &apixv1.ConversionResponse{ - UID: "some-uid", - Result: metav1.Status{Status: metav1.StatusSuccess}, - ConvertedObjects: []runtime.RawExtension{ - toRaw(t, test.out), - }, - } - - got := conversion.Convert(ctx, req) - if diff := cmp.Diff(want, got, cmpOpts...); diff != "" { - t.Error("unexpected response:", diff) - } - }) - } -} - -func TestConversionThroughHub(t *testing.T) { - tests := []struct { - name string - version string - in runtime.Object - out runtime.Object - }{{ - name: "v3 to v2", - version: "v2", - in: internal.NewV3("bing"), - out: internal.NewV2("bing"), - }, { - name: "v2 to v3", - version: "v3", - in: internal.NewV2("bang"), - out: internal.NewV3("bang"), - }} - - for _, test := range tests { - t.Run(test.version, func(t *testing.T) { - ctx, conversion := newConversion(t) - - req := &apixv1.ConversionRequest{ - UID: "some-uid", - DesiredAPIVersion: testAPIVersion(test.version), - Objects: []runtime.RawExtension{ - toRaw(t, test.in), - }, - } - - want := &apixv1.ConversionResponse{ - UID: "some-uid", - Result: metav1.Status{Status: metav1.StatusSuccess}, - ConvertedObjects: []runtime.RawExtension{ - toRaw(t, test.out), - }, - } - - got := conversion.Convert(ctx, req) - if diff := cmp.Diff(want, got, cmpOpts...); diff != "" { - t.Error("unexpected response:", diff) - } - }) - } -} - -func TestConversionErrorBadGVK(t *testing.T) { - tests := []struct { - name string - gvk schema.GroupVersionKind - }{{ - name: "empty group", - gvk: schema.GroupVersionKind{ - Version: "v1", - Kind: "Resource", - }, - }, { - name: "empty version", - gvk: schema.GroupVersionKind{ - Group: "webhook.pkg.knative.dev", - Kind: "Resource", - }, - }, { - name: "empty kind", - gvk: schema.GroupVersionKind{ - Group: "webhook.pkg.knative.dev", - Version: "v1", - }, - }} - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - obj := internal.NewV2("bing") - obj.SetGroupVersionKind(test.gvk) - - ctx, conversion := newConversion(t) - - req := &apixv1.ConversionRequest{ - UID: "some-uid", - DesiredAPIVersion: testAPIVersion("v1"), - Objects: []runtime.RawExtension{ - toRaw(t, obj), - }, - } - - want := &apixv1.ConversionResponse{ - UID: "some-uid", - Result: metav1.Status{ - Status: metav1.StatusFailure, - }, - } - - cmpOpts := []cmp.Option{ - cmpopts.IgnoreFields(metav1.Status{}, "Message"), - cmpopts.EquateEmpty(), - } - - got := conversion.Convert(ctx, req) - if diff := cmp.Diff(want, got, cmpOpts...); diff != "" { - t.Error("unexpected response:", diff) - } - - if !strings.HasPrefix(got.Result.Message, "invalid GroupVersionKind") { - t.Errorf("expected message to start with 'invalid GroupVersionKind' got %q", got.Result.Message) - } - }) - } -} - -func TestConversionUnknownInputGVK(t *testing.T) { - unknownObj := &unstructured.Unstructured{} - unknownObj.SetGroupVersionKind(schema.GroupVersionKind{ - Group: "some.api.group.dev", - Version: "v1", - Kind: "Resource", - }) - - ctx, conversion := newConversion(t) - - req := &apixv1.ConversionRequest{ - UID: "some-uid", - DesiredAPIVersion: testAPIVersion("v3"), - Objects: []runtime.RawExtension{ - toRaw(t, unknownObj), - }, - } - - want := &apixv1.ConversionResponse{ - UID: "some-uid", - Result: metav1.Status{ - Message: "no conversion support for type [kind=Resource group=some.api.group.dev]", - Status: metav1.StatusFailure, - }, - } - - got := conversion.Convert(ctx, req) - if diff := cmp.Diff(want, got, cmpOpts...); diff != "" { - t.Error("unexpected response:", diff) - } -} - -func TestConversionInvalidTypeMeta(t *testing.T) { - ctx, conversion := newConversionWithKinds(t, nil) - - req := &apixv1.ConversionRequest{ - UID: "some-uid", - DesiredAPIVersion: "some-version", - Objects: []runtime.RawExtension{ - {Raw: []byte("}")}, - }, - } - - want := &apixv1.ConversionResponse{ - UID: "some-uid", - Result: metav1.Status{ - Status: metav1.StatusFailure, - }, - } - - cmpOpts := []cmp.Option{ - cmpopts.IgnoreFields(metav1.Status{}, "Message"), - cmpopts.EquateEmpty(), - } - - got := conversion.Convert(ctx, req) - if diff := cmp.Diff(want, got, cmpOpts...); diff != "" { - t.Error("unexpected response:", diff) - } - - if !strings.HasPrefix(got.Result.Message, "error parsing type meta") { - t.Errorf("expected message to start with 'error parsing type meta' got %q", got.Result.Message) - } -} - -func TestConversionFailureToUnmarshalInput(t *testing.T) { - ctx, conversion := newConversion(t) - - req := &apixv1.ConversionRequest{ - UID: "some-uid", - DesiredAPIVersion: testAPIVersion("v1"), - Objects: []runtime.RawExtension{ - toRaw(t, internal.NewErrorResource(internal.ErrorUnmarshal)), - }, - } - - want := &apixv1.ConversionResponse{ - UID: "some-uid", - Result: metav1.Status{ - Status: metav1.StatusFailure, - }, - } - - cmpOpts := []cmp.Option{ - cmpopts.IgnoreFields(metav1.Status{}, "Message"), - cmpopts.EquateEmpty(), - } - - got := conversion.Convert(ctx, req) - if diff := cmp.Diff(want, got, cmpOpts...); diff != "" { - t.Error("unexpected response:", diff) - } - - if !strings.HasPrefix(got.Result.Message, "unable to unmarshal input") { - t.Errorf("expected message to start with 'unable to unmarshal input' got %q", got.Result.Message) - } -} - -func TestConversionFailureToMarshalOutput(t *testing.T) { - ctx, conversion := newConversion(t) - - req := &apixv1.ConversionRequest{ - UID: "some-uid", - DesiredAPIVersion: testAPIVersion("error"), - Objects: []runtime.RawExtension{ - // This property should make the Marshal on the - // ErrorResource to fail - toRaw(t, internal.NewV1(internal.ErrorMarshal)), - }, - } - - want := &apixv1.ConversionResponse{ - UID: "some-uid", - Result: metav1.Status{ - Status: metav1.StatusFailure, - }, - } - - cmpOpts := []cmp.Option{ - cmpopts.IgnoreFields(metav1.Status{}, "Message"), - cmpopts.EquateEmpty(), - } - - got := conversion.Convert(ctx, req) - if diff := cmp.Diff(want, got, cmpOpts...); diff != "" { - t.Error("unexpected response:", diff) - } - - if !strings.HasPrefix(got.Result.Message, "unable to marshal output") { - t.Errorf("expected message to start with 'unable to marshal output' got %q", got.Result.Message) - } -} - -func TestConversionFailureToConvert(t *testing.T) { - // v1 => error resource => v3 - kinds := map[schema.GroupKind]GroupKindConversion{ - testGK: { - DefinitionName: "resource.webhook.pkg.knative.dev", - HubVersion: "error", - Zygotes: zygotes, - }, - } - - tests := []struct { - name string - errorOn string - }{{ - name: "error converting from", - errorOn: internal.ErrorConvertFrom, - }, { - name: "error converting to", - errorOn: internal.ErrorConvertTo, - }} - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - ctx, conversion := newConversionWithKinds(t, kinds) - req := &apixv1.ConversionRequest{ - UID: "some-uid", - DesiredAPIVersion: testAPIVersion("v3"), - Objects: []runtime.RawExtension{ - // Insert failure here - toRaw(t, internal.NewV1(test.errorOn)), - }, - } - - want := &apixv1.ConversionResponse{ - UID: "some-uid", - Result: metav1.Status{ - Status: metav1.StatusFailure, - }, - } - - cmpOpts := []cmp.Option{ - cmpopts.IgnoreFields(metav1.Status{}, "Message"), - rawOpt, - } - - got := conversion.Convert(ctx, req) - if diff := cmp.Diff(want, got, cmpOpts...); diff != "" { - t.Error("unexpected response:", diff) - } - - if !strings.HasPrefix(got.Result.Message, "conversion failed") { - t.Errorf("expected message to start with 'conversion failed' got %q", got.Result.Message) - } - }) - } -} - -func TestConversionFailureInvalidDesiredAPIVersion(t *testing.T) { - tests := []struct { - name string - version string - }{{ - name: "multiple path segments", - version: "bad-api-version/v1/v2", - }, { - name: "empty", - version: "", - }, { - name: "path segment", - version: "/", - }, { - name: "no version", - version: "some.api.group", - }} - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - ctx, conversion := newConversion(t) - - req := &apixv1.ConversionRequest{ - UID: "some-uid", - DesiredAPIVersion: test.version, - Objects: []runtime.RawExtension{ - toRaw(t, internal.NewV1("bing")), - }, - } - - want := &apixv1.ConversionResponse{ - UID: "some-uid", - Result: metav1.Status{ - Message: fmt.Sprintf("desired API version %q is not valid", test.version), - Status: metav1.StatusFailure, - }, - } - - got := conversion.Convert(ctx, req) - if diff := cmp.Diff(want, got, cmpOpts...); diff != "" { - t.Error("unexpected response:", diff) - } - }) - } -} - -func TestConversionMissingZygotes(t *testing.T) { - // Assume we're converting from - // v2 (input) => v1 (hub) => v3 (output) - tests := []struct { - name string - zygotes map[string]ConvertibleObject - }{{ - name: "missing input", - zygotes: map[string]ConvertibleObject{ - "v1": &internal.V1Resource{}, - "v3": &internal.V3Resource{}, - }, - }, { - name: "missing output", - zygotes: map[string]ConvertibleObject{ - "v1": &internal.V1Resource{}, - "v2": &internal.V2Resource{}, - }, - }, { - name: "missing hub", - zygotes: map[string]ConvertibleObject{ - "v2": &internal.V2Resource{}, - "v3": &internal.V3Resource{}, - }, - }} - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - kinds = map[schema.GroupKind]GroupKindConversion{ - testGK: { - DefinitionName: "resource.webhook.pkg.knative.dev", - HubVersion: "v1", - Zygotes: test.zygotes, - }, - } - - ctx, conversion := newConversionWithKinds(t, kinds) - - req := &apixv1.ConversionRequest{ - UID: "some-uid", - DesiredAPIVersion: testAPIVersion("v3"), - Objects: []runtime.RawExtension{ - toRaw(t, internal.NewV2("bing")), - }, - } - - want := &apixv1.ConversionResponse{ - UID: "some-uid", - Result: metav1.Status{ - Status: metav1.StatusFailure, - }, - } - - cmpOpts := []cmp.Option{ - cmpopts.IgnoreFields(metav1.Status{}, "Message"), - cmpopts.EquateEmpty(), - } - - got := conversion.Convert(ctx, req) - if diff := cmp.Diff(want, got, cmpOpts...); diff != "" { - t.Error("unexpected response:", diff) - } - - if !strings.HasPrefix(got.Result.Message, "conversion not supported") { - t.Errorf("expected message to start with 'conversion not supported' got %q", got.Result.Message) - } - }) - } -} - -func TestContextDecoration(t *testing.T) { - ctx, _ := SetupFakeContext(t) - ctx = webhook.WithOptions(ctx, webhook.Options{ - SecretName: "webhook-secret", - }) - - decoratorCalled := false - decorator := func(ctx context.Context) context.Context { - decoratorCalled = true - return ctx - } - - controller := NewConversionController(ctx, webhookPath, kinds, decorator) - r := controller.Reconciler.(*reconciler) - r.Convert(ctx, &apixv1.ConversionRequest{}) - - if !decoratorCalled { - t.Errorf("context decorator was not invoked") - } -} - -func toRaw(t *testing.T, obj runtime.Object) runtime.RawExtension { - t.Helper() - - raw, err := json.Marshal(obj) - if err != nil { - t.Fatal("unable to marshal resource:", err) - } - - return runtime.RawExtension{Raw: raw} -} - -func newConversion(t *testing.T) (context.Context, webhook.ConversionController) { - return newConversionWithKinds(t, kinds) -} - -func newConversionWithKinds( - t *testing.T, - kinds map[schema.GroupKind]GroupKindConversion, -) ( - context.Context, - webhook.ConversionController, -) { - ctx, _ := SetupFakeContext(t) - ctx = webhook.WithOptions(ctx, webhook.Options{ - SecretName: "webhook-secret", - }) - - controller := NewConversionController(ctx, webhookPath, kinds, nil) - return ctx, controller.Reconciler.(*reconciler) -} diff --git a/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/internal/types.go b/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/internal/types.go deleted file mode 100644 index 14c56acd1..000000000 --- a/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/internal/types.go +++ /dev/null @@ -1,278 +0,0 @@ -/* -Portions Copyright (c) Microsoft Corporation. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package internal - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "strings" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "knative.dev/pkg/apis" -) - -const ( - // Group specifies the group of the test resource - Group = "webhook.pkg.knative.dev" - - // Kind specifies the kind of the test resource - Kind = "Resource" - - // ErrorMarshal when assigned to the Spec.Property of the ErrorResource - // will cause json marshaling of the resource to fail - ErrorMarshal = "marshal" - - // ErrorUnmarshal when assigned to the Spec.Property of the ErrorResource - // will cause json unmarshalling of the resource to fail - ErrorUnmarshal = "unmarshal" - - // ErrorConvertTo when assigned to the Spec.Property of the ErrorResource - // will cause ConvertTo to fail - ErrorConvertTo = "convertTo" - - // ErrorConvertFrom when assigned to the Spec.Property of the ErrorResource - // will cause ConvertFrom to fail - ErrorConvertFrom = "convertFrom" -) - -type ( - // V1Resource will never has a prefix or suffix on Spec.Property - // This type is used for testing conversion webhooks - // - // +k8s:deepcopy-gen=true - // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object - V1Resource struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` - Spec Spec `json:"spec"` - } - - // V2Resource will always have a 'prefix/' in front of it's property - // This type is used for testing conversion webhooks - // - // +k8s:deepcopy-gen=true - // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object - V2Resource struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` - Spec Spec `json:"spec"` - } - - // V3Resource will always have a '/suffix' in front of it's property - // This type is used for testing conversion webhooks - // - // +k8s:deepcopy-gen=true - // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object - V3Resource struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` - Spec SpecWithDefault `json:"spec"` - } - - // ErrorResource explodes in various settings depending on the property - // set. Use the Error* constants - // - // This type is used for testing conversion webhooks - // - // +k8s:deepcopy-gen=true - // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object - ErrorResource struct { - // We embed the V1Resource as an easy way to still marshal & unmarshal - // this type without infinite loops - since we override the methods - // in order to induce failures - V1Resource `json:",inline"` - } - - // Spec holds our fancy string property - Spec struct { - Property string `json:"prop"` - } - - // SpecWithDefault holds two fancy string properties - SpecWithDefault struct { - Property string `json:"prop"` - NewProperty string `json:"defaulted_prop"` - } -) - -var ( - _ apis.Convertible = (*V1Resource)(nil) - _ apis.Convertible = (*V2Resource)(nil) - _ apis.Convertible = (*V3Resource)(nil) - _ apis.Convertible = (*ErrorResource)(nil) - - _ apis.Defaultable = (*V3Resource)(nil) -) - -// NewV1 returns a V1Resource with Spec.Property set -// to prop -func NewV1(prop string) *V1Resource { - return &V1Resource{ - TypeMeta: metav1.TypeMeta{ - Kind: Kind, - APIVersion: Group + "/v1", - }, - Spec: Spec{ - Property: prop, - }, - } -} - -// NewV2 returns a V2Resource with Spec.Property set -// to 'prefix/' + prop -func NewV2(prop string) *V2Resource { - return &V2Resource{ - TypeMeta: metav1.TypeMeta{ - Kind: Kind, - APIVersion: Group + "/v2", - }, - Spec: Spec{ - Property: "prefix/" + prop, - }, - } -} - -// NewV3 returns a V3Resource with Spec.Property set -// to prop + '/suffix' -func NewV3(prop string) *V3Resource { - v3 := &V3Resource{ - TypeMeta: metav1.TypeMeta{ - Kind: Kind, - APIVersion: Group + "/v3", - }, - Spec: SpecWithDefault{ - Property: prop + "/suffix", - }, - } - v3.SetDefaults(context.Background()) - return v3 -} - -// NewErrorResource returns an ErrorResource with Spec.Property set -// to failure -func NewErrorResource(failure string) *ErrorResource { - return &ErrorResource{ - V1Resource: V1Resource{ - TypeMeta: metav1.TypeMeta{ - Kind: Kind, - APIVersion: Group + "/error", - }, - Spec: Spec{ - Property: failure, - }, - }, - } -} - -// ConvertTo implements apis.Convertible -func (r *V1Resource) ConvertTo(ctx context.Context, to apis.Convertible) error { - switch sink := to.(type) { - case *V2Resource: - sink.Spec.Property = "prefix/" + r.Spec.Property - case *V3Resource: - sink.Spec.Property = r.Spec.Property + "/suffix" - case *ErrorResource: - sink.Spec.Property = r.Spec.Property - case *V1Resource: - sink.Spec.Property = r.Spec.Property - default: - return fmt.Errorf("unsupported type %T", sink) - } - return nil -} - -// ConvertFrom implements apis.Convertible -func (r *V1Resource) ConvertFrom(ctx context.Context, from apis.Convertible) error { - switch source := from.(type) { - case *V2Resource: - r.Spec.Property = strings.TrimPrefix(source.Spec.Property, "prefix/") - case *V3Resource: - r.Spec.Property = strings.TrimSuffix(source.Spec.Property, "/suffix") - case *ErrorResource: - r.Spec.Property = source.Spec.Property - case *V1Resource: - r.Spec.Property = source.Spec.Property - default: - return fmt.Errorf("unsupported type %T", source) - } - return nil -} - -// SetDefaults implements apis.Defaultable -func (r *V3Resource) SetDefaults(ctx context.Context) { - if r.Spec.NewProperty == "" { - r.Spec.NewProperty = "defaulted" - } -} - -// ConvertTo implements apis.Convertible -func (*V2Resource) ConvertTo(ctx context.Context, to apis.Convertible) error { - panic("unimplemented") -} - -// ConvertFrom implements apis.Convertible -func (*V2Resource) ConvertFrom(ctx context.Context, from apis.Convertible) error { - panic("unimplemented") -} - -// ConvertTo implements apis.Convertible -func (*V3Resource) ConvertTo(ctx context.Context, to apis.Convertible) error { - panic("unimplemented") -} - -// ConvertFrom implements apis.Convertible -func (*V3Resource) ConvertFrom(ctx context.Context, from apis.Convertible) error { - panic("unimplemented") -} - -// ConvertTo implements apis.Convertible -func (e *ErrorResource) ConvertTo(ctx context.Context, to apis.Convertible) error { - if e.Spec.Property == ErrorConvertTo { - return errors.New("boooom - convert up") - } - - return e.V1Resource.ConvertTo(ctx, to) -} - -// ConvertFrom implements apis.Convertible -func (e *ErrorResource) ConvertFrom(ctx context.Context, from apis.Convertible) error { - err := e.V1Resource.ConvertFrom(ctx, from) - - if err == nil && e.Spec.Property == ErrorConvertFrom { - err = errors.New("boooom - convert down") - } - return err -} - -// UnmarshalJSON implements json.Unmarshaler -func (e *ErrorResource) UnmarshalJSON(data []byte) (err error) { - err = json.Unmarshal(data, &e.V1Resource) - if err == nil && e.Spec.Property == ErrorUnmarshal { - err = errors.New("boooom - unmarshal json") - } - return -} - -// MarshalJSON implements json.Marshaler -func (e *ErrorResource) MarshalJSON() ([]byte, error) { - if e.Spec.Property == ErrorMarshal { - return nil, errors.New("boooom - marshal json") - } - return json.Marshal(e.V1Resource) -} diff --git a/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/internal/zz_generated.deepcopy.go b/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/internal/zz_generated.deepcopy.go deleted file mode 100644 index 6ea895a85..000000000 --- a/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/internal/zz_generated.deepcopy.go +++ /dev/null @@ -1,132 +0,0 @@ -//go:build !ignore_autogenerated -// +build !ignore_autogenerated - -/* -Copyright 2022 The Knative Authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by deepcopy-gen. DO NOT EDIT. - -package internal - -import ( - runtime "k8s.io/apimachinery/pkg/runtime" -) - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ErrorResource) DeepCopyInto(out *ErrorResource) { - *out = *in - in.V1Resource.DeepCopyInto(&out.V1Resource) - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ErrorResource. -func (in *ErrorResource) DeepCopy() *ErrorResource { - if in == nil { - return nil - } - out := new(ErrorResource) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *ErrorResource) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *V1Resource) DeepCopyInto(out *V1Resource) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - out.Spec = in.Spec - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new V1Resource. -func (in *V1Resource) DeepCopy() *V1Resource { - if in == nil { - return nil - } - out := new(V1Resource) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *V1Resource) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *V2Resource) DeepCopyInto(out *V2Resource) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - out.Spec = in.Spec - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new V2Resource. -func (in *V2Resource) DeepCopy() *V2Resource { - if in == nil { - return nil - } - out := new(V2Resource) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *V2Resource) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *V3Resource) DeepCopyInto(out *V3Resource) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - out.Spec = in.Spec - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new V3Resource. -func (in *V3Resource) DeepCopy() *V3Resource { - if in == nil { - return nil - } - out := new(V3Resource) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *V3Resource) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} diff --git a/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/options.go b/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/options.go deleted file mode 100644 index 488d54987..000000000 --- a/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/options.go +++ /dev/null @@ -1,46 +0,0 @@ -// Portions Copyright (c) Microsoft Corporation. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package conversion - -import ( - "context" - - "k8s.io/apimachinery/pkg/runtime/schema" -) - -type options struct { - path string - wc func(context.Context) context.Context - kinds map[schema.GroupKind]GroupKindConversion -} - -type OptionFunc func(*options) - -func WithKinds(kinds map[schema.GroupKind]GroupKindConversion) OptionFunc { - return func(o *options) { - o.kinds = kinds - } -} - -func WithPath(path string) OptionFunc { - return func(o *options) { - o.path = path - } -} - -func WithWrapContext(f func(context.Context) context.Context) OptionFunc { - return func(o *options) { - o.wc = f - } -} diff --git a/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/options_test.go b/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/options_test.go deleted file mode 100644 index 438fffd91..000000000 --- a/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/options_test.go +++ /dev/null @@ -1,38 +0,0 @@ -/* -Portions Copyright (c) Microsoft Corporation. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package conversion - -import ( - "reflect" - "testing" -) - -func TestOptions(t *testing.T) { - got := &options{} - WithPath("path")(got) - - want := &options{ - path: "path", - // we can't compare wc as functions are not - // comparable in golang (thus it needs to be - // done indirectly) - } - - if !reflect.DeepEqual(got, want) { - t.Error("option was not applied") - } -} diff --git a/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/reconciler.go b/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/reconciler.go deleted file mode 100644 index ed0fadc28..000000000 --- a/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/reconciler.go +++ /dev/null @@ -1,123 +0,0 @@ -// Portions Copyright (c) Microsoft Corporation. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package conversion - -import ( - "context" - "fmt" - - "go.uber.org/zap" - apixv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - apixclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" - apixlisters "k8s.io/apiextensions-apiserver/pkg/client/listers/apiextensions/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/types" - corelisters "k8s.io/client-go/listers/core/v1" - "knative.dev/pkg/controller" - "knative.dev/pkg/kmp" - "knative.dev/pkg/logging" - "knative.dev/pkg/ptr" - pkgreconciler "knative.dev/pkg/reconciler" - "knative.dev/pkg/system" - "knative.dev/pkg/webhook" - certresources "knative.dev/pkg/webhook/certificates/resources" -) - -type reconciler struct { - pkgreconciler.LeaderAwareFuncs - - kinds map[schema.GroupKind]GroupKindConversion - path string - secretName string - withContext func(context.Context) context.Context - - secretLister corelisters.SecretLister - crdLister apixlisters.CustomResourceDefinitionLister - client apixclient.Interface -} - -var ( - _ webhook.ConversionController = (*reconciler)(nil) - _ controller.Reconciler = (*reconciler)(nil) - _ pkgreconciler.LeaderAware = (*reconciler)(nil) -) - -// Path implements webhook.ConversionController -func (r *reconciler) Path() string { - return r.path -} - -// Reconciler implements controller.Reconciler -func (r *reconciler) Reconcile(ctx context.Context, key string) error { - logger := logging.FromContext(ctx) - - if !r.IsLeaderFor(types.NamespacedName{Name: key}) { - return controller.NewSkipKey(key) - } - - // Look up the webhook secret, and fetch the CA cert bundle. - secret, err := r.secretLister.Secrets(system.Namespace()).Get(r.secretName) - if err != nil { - logger.Errorw("Error fetching secret", zap.Error(err)) - return err - } - - cacert, ok := secret.Data[certresources.CACert] - if !ok { - return fmt.Errorf("secret %q is missing %q key", r.secretName, certresources.CACert) - } - - return r.reconcileCRD(ctx, cacert, key) -} - -func (r *reconciler) reconcileCRD(ctx context.Context, cacert []byte, key string) error { - logger := logging.FromContext(ctx) - - configuredCRD, err := r.crdLister.Get(key) - if err != nil { - return fmt.Errorf("error retrieving crd: %w", err) - } - - crd := configuredCRD.DeepCopy() - - if crd.Spec.Conversion == nil || - crd.Spec.Conversion.Strategy != apixv1.WebhookConverter || - crd.Spec.Conversion.Webhook.ClientConfig == nil || - (crd.Spec.Conversion.Webhook.ClientConfig.Service == nil && crd.Spec.Conversion.Webhook.ClientConfig.URL == nil) { // AKS customized - return fmt.Errorf("custom resource %q isn't configured for webhook conversion", key) - } - - crd.Spec.Conversion.Webhook.ClientConfig.CABundle = cacert - - // AKS customized - if crd.Spec.Conversion.Webhook.ClientConfig.Service != nil { // This line is changed from upstream - crd.Spec.Conversion.Webhook.ClientConfig.Service.Path = ptr.String(r.path) - } - // end customization - - if ok, err := kmp.SafeEqual(configuredCRD, crd); err != nil { - return fmt.Errorf("error diffing custom resource definitions: %w", err) - } else if !ok { - logger.Infof("updating CRD") - crdClient := r.client.ApiextensionsV1().CustomResourceDefinitions() - if _, err := crdClient.Update(ctx, crd, metav1.UpdateOptions{}); err != nil { - return fmt.Errorf("failed to update webhook: %w", err) - } - } else { - logger.Info("CRD is up to date") - } - - return nil -} diff --git a/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/table_test.go b/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/table_test.go deleted file mode 100644 index dc812302c..000000000 --- a/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/table_test.go +++ /dev/null @@ -1,290 +0,0 @@ -/* -Portions Copyright (c) Microsoft Corporation. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package conversion - -import ( - "context" - "testing" - - corev1 "k8s.io/api/core/v1" - apixv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - clientgotesting "k8s.io/client-go/testing" - apixclient "knative.dev/pkg/client/injection/apiextensions/client/fake" - "knative.dev/pkg/configmap" - "knative.dev/pkg/controller" - "knative.dev/pkg/ptr" - "knative.dev/pkg/system" - certresources "knative.dev/pkg/webhook/certificates/resources" - - . "knative.dev/pkg/reconciler/testing" - . "knative.dev/pkg/webhook/testing" -) - -func TestReconcile(t *testing.T) { - key := "some.crd.group.dev" - path := "/some/path" - secretName := "webhook-secret" - - secret := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: secretName, - Namespace: system.Namespace(), - }, - Data: map[string][]byte{ - certresources.ServerKey: []byte("present"), - certresources.ServerCert: []byte("present"), - certresources.CACert: []byte("present"), - }, - } - - table := TableTest{{ - Name: "no secret", - Key: key, - WantErr: true, - }, { - Name: "secret missing CA Cert", - Key: key, - Objects: []runtime.Object{&corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: secretName, - Namespace: system.Namespace(), - }, - Data: map[string][]byte{ - certresources.ServerKey: []byte("present"), - certresources.ServerCert: []byte("present"), - // certresources.CACert: []byte("missing"), - }, - }}, - WantErr: true, - }, { - Name: "secret exists, but CRD does not", - Key: key, - Objects: []runtime.Object{secret}, - WantErr: true, - }, { - Name: "secret and CRD exist, missing service reference", - Key: key, - Objects: []runtime.Object{ - secret, - &apixv1.CustomResourceDefinition{ - ObjectMeta: metav1.ObjectMeta{ - Name: key, - }, - Spec: apixv1.CustomResourceDefinitionSpec{ - Conversion: &apixv1.CustomResourceConversion{}, - }, - }, - }, - WantErr: true, - }, { - Name: "secret and CRD exist, missing other stuff", - Key: key, - Objects: []runtime.Object{ - secret, - &apixv1.CustomResourceDefinition{ - ObjectMeta: metav1.ObjectMeta{ - Name: key, - }, - Spec: apixv1.CustomResourceDefinitionSpec{ - Conversion: &apixv1.CustomResourceConversion{ - Strategy: apixv1.WebhookConverter, - Webhook: &apixv1.WebhookConversion{ - ClientConfig: &apixv1.WebhookClientConfig{ - Service: &apixv1.ServiceReference{ - Namespace: system.Namespace(), - Name: "webhook", - }, - }, - }, - }, - }, - }, - }, - WantUpdates: []clientgotesting.UpdateActionImpl{{ - Object: &apixv1.CustomResourceDefinition{ - ObjectMeta: metav1.ObjectMeta{ - Name: key, - }, - Spec: apixv1.CustomResourceDefinitionSpec{ - Conversion: &apixv1.CustomResourceConversion{ - Strategy: apixv1.WebhookConverter, - Webhook: &apixv1.WebhookConversion{ - ClientConfig: &apixv1.WebhookClientConfig{ - Service: &apixv1.ServiceReference{ - Namespace: system.Namespace(), - Name: "webhook", - // Path is added. - Path: ptr.String(path), - }, - // CABundle is added. - CABundle: []byte("present"), - }, - }, - }, - }, - }, - }}, - }, { - Name: "secret and CRD exist, incorrect fields", - Key: key, - Objects: []runtime.Object{ - secret, - &apixv1.CustomResourceDefinition{ - ObjectMeta: metav1.ObjectMeta{ - Name: key, - }, - Spec: apixv1.CustomResourceDefinitionSpec{ - Conversion: &apixv1.CustomResourceConversion{ - Strategy: apixv1.WebhookConverter, - Webhook: &apixv1.WebhookConversion{ - ClientConfig: &apixv1.WebhookClientConfig{ - Service: &apixv1.ServiceReference{ - Namespace: system.Namespace(), - Name: "webhook", - // Incorrect path - Path: ptr.String("/incorrect"), - }, - // CABundle is added. - CABundle: []byte("incorrect"), - }, - }, - }, - }, - }, - }, - WantUpdates: []clientgotesting.UpdateActionImpl{{ - Object: &apixv1.CustomResourceDefinition{ - ObjectMeta: metav1.ObjectMeta{ - Name: key, - }, - Spec: apixv1.CustomResourceDefinitionSpec{ - Conversion: &apixv1.CustomResourceConversion{ - Strategy: apixv1.WebhookConverter, - Webhook: &apixv1.WebhookConversion{ - ClientConfig: &apixv1.WebhookClientConfig{ - Service: &apixv1.ServiceReference{ - Namespace: system.Namespace(), - Name: "webhook", - // Path is added. - Path: ptr.String(path), - }, - // CABundle is added. - CABundle: []byte("present"), - }, - }, - }, - }, - }, - }}, - }, { - Name: "failed to update custom resource definition", - Key: key, - WantErr: true, - WithReactors: []clientgotesting.ReactionFunc{ - InduceFailure("update", "customresourcedefinitions"), - }, - Objects: []runtime.Object{ - secret, - &apixv1.CustomResourceDefinition{ - ObjectMeta: metav1.ObjectMeta{ - Name: key, - }, - Spec: apixv1.CustomResourceDefinitionSpec{ - Conversion: &apixv1.CustomResourceConversion{ - Strategy: apixv1.WebhookConverter, - Webhook: &apixv1.WebhookConversion{ - ClientConfig: &apixv1.WebhookClientConfig{ - Service: &apixv1.ServiceReference{ - Namespace: system.Namespace(), - Name: "webhook", - // Incorrect path - Path: ptr.String("/incorrect"), - }, - // CABundle is added. - CABundle: []byte("incorrect"), - }, - }, - }, - }, - }, - }, - WantUpdates: []clientgotesting.UpdateActionImpl{{ - Object: &apixv1.CustomResourceDefinition{ - ObjectMeta: metav1.ObjectMeta{ - Name: key, - }, - Spec: apixv1.CustomResourceDefinitionSpec{ - Conversion: &apixv1.CustomResourceConversion{ - Strategy: apixv1.WebhookConverter, - Webhook: &apixv1.WebhookConversion{ - ClientConfig: &apixv1.WebhookClientConfig{ - Service: &apixv1.ServiceReference{ - Namespace: system.Namespace(), - Name: "webhook", - // Path is added. - Path: ptr.String(path), - }, - // CABundle is added. - CABundle: []byte("present"), - }, - }, - }, - }, - }, - }}, - }, { - Name: "stable", - Key: key, - Objects: []runtime.Object{ - secret, - &apixv1.CustomResourceDefinition{ - ObjectMeta: metav1.ObjectMeta{ - Name: key, - }, - Spec: apixv1.CustomResourceDefinitionSpec{ - Conversion: &apixv1.CustomResourceConversion{ - Strategy: apixv1.WebhookConverter, - Webhook: &apixv1.WebhookConversion{ - ClientConfig: &apixv1.WebhookClientConfig{ - Service: &apixv1.ServiceReference{ - Namespace: system.Namespace(), - Name: "webhook", - Path: ptr.String(path), - }, - // CABundle is added. - CABundle: []byte("present"), - }, - }, - }, - }, - }, - }, - }} - - table.Test(t, MakeFactory(func(ctx context.Context, listers *Listers, cmw configmap.Watcher) controller.Reconciler { - return &reconciler{ - kinds: kinds, - path: path, - secretName: secretName, - secretLister: listers.GetSecretLister(), - crdLister: listers.GetCustomResourceDefinitionLister(), - client: apixclient.Get(ctx), - } - })) -} From 74ea94d9c07d9a330007eca0cf90b17d4b5e060a Mon Sep 17 00:00:00 2001 From: Alex Leites <18728999+tallaxes@users.noreply.github.com> Date: Tue, 3 Dec 2024 02:55:08 +0000 Subject: [PATCH 04/19] chore: update operator --- pkg/operator/operator.go | 60 ---------------------------------------- 1 file changed, 60 deletions(-) diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index 0cf48ff27..042bf8b3c 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -20,30 +20,18 @@ import ( "context" "encoding/base64" "fmt" - "sync" - "github.com/awslabs/operatorpkg/controller" - "github.com/awslabs/operatorpkg/object" - "github.com/awslabs/operatorpkg/status" "github.com/patrickmn/go-cache" "github.com/samber/lo" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/rest" "k8s.io/client-go/transport" - knativeinjection "knative.dev/pkg/injection" "knative.dev/pkg/ptr" - "sigs.k8s.io/controller-runtime/pkg/log" karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" - karpv1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" - "sigs.k8s.io/karpenter/pkg/cloudprovider" - "sigs.k8s.io/karpenter/pkg/operator/injection" - karpenteroptions "sigs.k8s.io/karpenter/pkg/operator/options" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" - webhooksalt "github.com/Azure/karpenter-provider-azure/pkg/alt/karpenter-core/pkg/webhooks" "github.com/Azure/karpenter-provider-azure/pkg/auth" azurecache "github.com/Azure/karpenter-provider-azure/pkg/cache" @@ -61,7 +49,6 @@ import ( func init() { karpv1.NormalizedLabels = lo.Assign(karpv1.NormalizedLabels, map[string]string{"topology.disk.csi.azure.com/zone": corev1.LabelTopologyZone}) - karpv1beta1.NormalizedLabels = lo.Assign(karpv1.NormalizedLabels, map[string]string{"topology.disk.csi.azure.com/zone": corev1.LabelTopologyZone}) } type Operator struct { @@ -76,9 +63,6 @@ type Operator struct { InstanceTypesProvider instancetype.Provider InstanceProvider *instance.DefaultProvider LoadBalancerProvider *loadbalancer.Provider - - // Copied from the core Operator because we control our own webhooks - webhooks []knativeinjection.ControllerConstructor } func NewOperator(ctx context.Context, operator *operator.Operator) (context.Context, *Operator) { @@ -161,50 +145,6 @@ func NewOperator(ctx context.Context, operator *operator.Operator) (context.Cont } } -// Copied from karpenter-core pkg/operator/operator.go, needed for webhooks -func (o *Operator) WithControllers(ctx context.Context, controllers ...controller.Controller) *Operator { - for _, c := range controllers { - lo.Must0(c.Register(ctx, o.Manager)) - } - return o -} - -// Copied from karpenter-core pkg/operator/operator.go, needed for webhooks -func (o *Operator) WithWebhooks(ctx context.Context, ctors ...knativeinjection.ControllerConstructor) *Operator { - if !karpenteroptions.FromContext(ctx).DisableWebhook { - o.webhooks = append(o.webhooks, ctors...) - lo.Must0(o.Manager.AddReadyzCheck("webhooks", webhooksalt.HealthProbe(ctx))) - lo.Must0(o.Manager.AddHealthzCheck("webhooks", webhooksalt.HealthProbe(ctx))) - } - return o -} - -// Copied from karpenter-core pkg/operator/operator.go, needed for webhooks -func (o *Operator) Start(ctx context.Context, cp cloudprovider.CloudProvider) { - wg := &sync.WaitGroup{} - wg.Add(1) - go func() { - defer wg.Done() - lo.Must0(o.Manager.Start(ctx)) - }() - if karpenteroptions.FromContext(ctx).DisableWebhook { - log.FromContext(ctx).Info("conversion webhooks are disabled") - } else { - wg.Add(1) - go func() { - defer wg.Done() - // Taking the first supported NodeClass to be the default NodeClass - gvk := lo.Map(cp.GetSupportedNodeClasses(), func(nc status.Object, _ int) schema.GroupVersionKind { - return object.GVK(nc) - }) - ctx = injection.WithNodeClasses(ctx, gvk) - ctx = injection.WithClient(ctx, o.GetClient()) - webhooksalt.Start(ctx, o.GetConfig(), o.webhooks...) // This is our alt copy of webhooks that can support multiple apiservers - }() - } - wg.Wait() -} - func GetAZConfig() (*auth.Config, error) { cfg, err := auth.BuildAzureConfig() if err != nil { From 0a003604b2b8b0750366f28e64d2f1655564ec23 Mon Sep 17 00:00:00 2001 From: Alex Leites <18728999+tallaxes@users.noreply.github.com> Date: Tue, 3 Dec 2024 02:55:42 +0000 Subject: [PATCH 05/19] chore: update cloudprovider --- pkg/cloudprovider/cloudprovider.go | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/pkg/cloudprovider/cloudprovider.go b/pkg/cloudprovider/cloudprovider.go index b9c9764b3..f1be96f97 100644 --- a/pkg/cloudprovider/cloudprovider.go +++ b/pkg/cloudprovider/cloudprovider.go @@ -26,7 +26,7 @@ import ( "github.com/awslabs/operatorpkg/status" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -240,6 +240,18 @@ func (c *CloudProvider) GetSupportedNodeClasses() []status.Object { return []status.Object{&v1alpha2.AKSNodeClass{}} } +// TODO: review repair policies +func (c *CloudProvider) RepairPolicies() []cloudprovider.RepairPolicy { + return []cloudprovider.RepairPolicy{ + // Supported Kubelet fields + { + ConditionType: corev1.NodeReady, + ConditionStatus: corev1.ConditionFalse, + TolerationDuration: 30 * time.Minute, + }, + } +} + func (c *CloudProvider) resolveNodeClassFromNodeClaim(ctx context.Context, nodeClaim *karpv1.NodeClaim) (*v1alpha2.AKSNodeClass, error) { nodeClass := &v1alpha2.AKSNodeClass{} if err := c.kubeClient.Get(ctx, types.NamespacedName{Name: nodeClaim.Spec.NodeClassRef.Name}, nodeClass); err != nil { @@ -324,8 +336,8 @@ func (c *CloudProvider) instanceToNodeClaim(ctx context.Context, vm *armcompute. if instanceType != nil { labels = instance.GetAllSingleValuedRequirementLabels(instanceType) - nodeClaim.Status.Capacity = lo.PickBy(instanceType.Capacity, func(_ v1.ResourceName, v resource.Quantity) bool { return !resources.IsZero(v) }) - nodeClaim.Status.Allocatable = lo.PickBy(instanceType.Allocatable(), func(_ v1.ResourceName, v resource.Quantity) bool { return !resources.IsZero(v) }) + nodeClaim.Status.Capacity = lo.PickBy(instanceType.Capacity, func(_ corev1.ResourceName, v resource.Quantity) bool { return !resources.IsZero(v) }) + nodeClaim.Status.Allocatable = lo.PickBy(instanceType.Allocatable(), func(_ corev1.ResourceName, v resource.Quantity) bool { return !resources.IsZero(v) }) } // TODO: review logic for determining zone (AWS uses Zone from subnet resolved and aviailable from NodeClass conditions ...) From ec6baf72670d95bb4acdb26ec32ee699e920f114 Mon Sep 17 00:00:00 2001 From: Alex Leites <18728999+tallaxes@users.noreply.github.com> Date: Tue, 3 Dec 2024 02:56:25 +0000 Subject: [PATCH 06/19] chore: fix tests and update expectation helpers --- pkg/cloudprovider/suite_test.go | 6 +- .../nodeclaim/garbagecollection/suite_test.go | 2 +- pkg/providers/instance/suite_test.go | 6 +- pkg/providers/instancetype/suite_test.go | 31 +++-- pkg/test/environment.go | 2 - test/pkg/environment/common/expectations.go | 130 +++++++++++++++--- test/suites/nodeclaim/nodeclaim_test.go | 17 ++- 7 files changed, 156 insertions(+), 38 deletions(-) diff --git a/pkg/cloudprovider/suite_test.go b/pkg/cloudprovider/suite_test.go index b4cde4dee..8009f0442 100644 --- a/pkg/cloudprovider/suite_test.go +++ b/pkg/cloudprovider/suite_test.go @@ -83,7 +83,7 @@ var _ = BeforeSuite(func() { fakeClock = clock.NewFakeClock(time.Now()) recorder = events.NewRecorder(&record.FakeRecorder{}) cloudProvider = New(azureEnv.InstanceTypesProvider, azureEnv.InstanceProvider, recorder, env.Client, azureEnv.ImageProvider) - cluster = state.NewCluster(fakeClock, env.Client) + cluster = state.NewCluster(fakeClock, env.Client, cloudProvider) prov = provisioning.NewProvisioner(env.Client, recorder, cloudProvider, cluster, fakeClock) }) @@ -195,7 +195,9 @@ var _ = Describe("CloudProvider", func() { }, Spec: karpv1.NodeClaimSpec{ NodeClassRef: &karpv1.NodeClassReference{ - Name: nodeClass.Name, + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, }, }, }) diff --git a/pkg/controllers/nodeclaim/garbagecollection/suite_test.go b/pkg/controllers/nodeclaim/garbagecollection/suite_test.go index 5e9a7edee..7df69fb96 100644 --- a/pkg/controllers/nodeclaim/garbagecollection/suite_test.go +++ b/pkg/controllers/nodeclaim/garbagecollection/suite_test.go @@ -82,7 +82,7 @@ var _ = BeforeSuite(func() { cloudProvider = cloudprovider.New(azureEnv.InstanceTypesProvider, azureEnv.InstanceProvider, events.NewRecorder(&record.FakeRecorder{}), env.Client, azureEnv.ImageProvider) garbageCollectionController = garbagecollection.NewController(env.Client, cloudProvider) fakeClock = &clock.FakeClock{} - cluster = state.NewCluster(fakeClock, env.Client) + cluster = state.NewCluster(fakeClock, env.Client, cloudProvider) prov = provisioning.NewProvisioner(env.Client, events.NewRecorder(&record.FakeRecorder{}), cloudProvider, cluster, fakeClock) }) diff --git a/pkg/providers/instance/suite_test.go b/pkg/providers/instance/suite_test.go index 923f24e17..1142ce0d2 100644 --- a/pkg/providers/instance/suite_test.go +++ b/pkg/providers/instance/suite_test.go @@ -79,7 +79,7 @@ func TestAzure(t *testing.T) { cloudProvider = cloudprovider.New(azureEnv.InstanceTypesProvider, azureEnv.InstanceProvider, events.NewRecorder(&record.FakeRecorder{}), env.Client, azureEnv.ImageProvider) cloudProviderNonZonal = cloudprovider.New(azureEnvNonZonal.InstanceTypesProvider, azureEnvNonZonal.InstanceProvider, events.NewRecorder(&record.FakeRecorder{}), env.Client, azureEnvNonZonal.ImageProvider) fakeClock = &clock.FakeClock{} - cluster = state.NewCluster(fakeClock, env.Client) + cluster = state.NewCluster(fakeClock, env.Client, cloudProvider) coreProvisioner = provisioning.NewProvisioner(env.Client, events.NewRecorder(&record.FakeRecorder{}), cloudProvider, cluster, fakeClock) RunSpecs(t, "Provider/Azure") } @@ -119,7 +119,9 @@ var _ = Describe("InstanceProvider", func() { }, Spec: karpv1.NodeClaimSpec{ NodeClassRef: &karpv1.NodeClassReference{ - Name: nodeClass.Name, + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, }, }, }) diff --git a/pkg/providers/instancetype/suite_test.go b/pkg/providers/instancetype/suite_test.go index 6c4a5b85c..a55f1a88f 100644 --- a/pkg/providers/instancetype/suite_test.go +++ b/pkg/providers/instancetype/suite_test.go @@ -25,6 +25,7 @@ import ( "strings" "testing" + "github.com/awslabs/operatorpkg/object" "github.com/awslabs/operatorpkg/status" "github.com/blang/semver/v4" . "github.com/onsi/ginkgo/v2" @@ -93,8 +94,8 @@ func TestAzure(t *testing.T) { cloudProvider = cloudprovider.New(azureEnv.InstanceTypesProvider, azureEnv.InstanceProvider, events.NewRecorder(&record.FakeRecorder{}), env.Client, azureEnv.ImageProvider) cloudProviderNonZonal = cloudprovider.New(azureEnvNonZonal.InstanceTypesProvider, azureEnvNonZonal.InstanceProvider, events.NewRecorder(&record.FakeRecorder{}), env.Client, azureEnvNonZonal.ImageProvider) - cluster = state.NewCluster(fakeClock, env.Client) - clusterNonZonal = state.NewCluster(fakeClock, env.Client) + cluster = state.NewCluster(fakeClock, env.Client, cloudProvider) + clusterNonZonal = state.NewCluster(fakeClock, env.Client, cloudProviderNonZonal) coreProvisioner = provisioning.NewProvisioner(env.Client, events.NewRecorder(&record.FakeRecorder{}), cloudProvider, cluster, fakeClock) coreProvisionerNonZonal = provisioning.NewProvisioner(env.Client, events.NewRecorder(&record.FakeRecorder{}), cloudProviderNonZonal, clusterNonZonal, fakeClock) @@ -121,7 +122,9 @@ var _ = Describe("InstanceType Provider", func() { Template: karpv1.NodeClaimTemplate{ Spec: karpv1.NodeClaimTemplateSpec{ NodeClassRef: &karpv1.NodeClassReference{ - Name: nodeClass.Name, + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, }, }, }, @@ -313,7 +316,9 @@ var _ = Describe("InstanceType Provider", func() { }, Spec: karpv1.NodeClaimSpec{ NodeClassRef: &karpv1.NodeClassReference{ - Name: nodeClass.Name, + Name: nodeClass.Name, + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, }, }, }) @@ -382,7 +387,9 @@ var _ = Describe("InstanceType Provider", func() { Values: []string{"Standard_D64s_v3"}, }}) np.Spec.Template.Spec.NodeClassRef = &karpv1.NodeClassReference{ - Name: nodeClass.Name, + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, } ExpectApplied(ctx, env.Client, np, nodeClass) @@ -413,7 +420,9 @@ var _ = Describe("InstanceType Provider", func() { Values: []string{"Standard_D64s_v3"}, }}) np.Spec.Template.Spec.NodeClassRef = &karpv1.NodeClassReference{ - Name: nodeClass.Name, + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, } ExpectApplied(ctx, env.Client, np, nodeClass) @@ -440,7 +449,9 @@ var _ = Describe("InstanceType Provider", func() { Values: []string{"Standard_D2s_v3"}, }}) np.Spec.Template.Spec.NodeClassRef = &karpv1.NodeClassReference{ - Name: nodeClass.Name, + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, } ExpectApplied(ctx, env.Client, np, nodeClass) @@ -892,7 +903,11 @@ var _ = Describe("InstanceType Provider", func() { Operator: v1.NodeSelectorOpIn, Values: []string{instanceType}, }}) - nodePool.Spec.Template.Spec.NodeClassRef = &karpv1.NodeClassReference{Name: nodeClass.Name} + nodePool.Spec.Template.Spec.NodeClassRef = &karpv1.NodeClassReference{ + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, + } ExpectApplied(ctx, env.Client, nodePool, nodeClass) pod := coretest.UnschedulablePod(coretest.PodOptions{}) ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, coreProvisioner, pod) diff --git a/pkg/test/environment.go b/pkg/test/environment.go index 30b6777bf..36d5fb102 100644 --- a/pkg/test/environment.go +++ b/pkg/test/environment.go @@ -23,7 +23,6 @@ import ( corev1 "k8s.io/api/core/v1" karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" - karpv1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" azurecache "github.com/Azure/karpenter-provider-azure/pkg/cache" "github.com/Azure/karpenter-provider-azure/pkg/fake" @@ -40,7 +39,6 @@ import ( ) func init() { - karpv1beta1.NormalizedLabels = lo.Assign(karpv1beta1.NormalizedLabels, map[string]string{"topology.disk.csi.azure.com/zone": corev1.LabelTopologyZone}) karpv1.NormalizedLabels = lo.Assign(karpv1.NormalizedLabels, map[string]string{"topology.disk.csi.azure.com/zone": corev1.LabelTopologyZone}) } diff --git a/test/pkg/environment/common/expectations.go b/test/pkg/environment/common/expectations.go index 754d1eb5f..af3876eee 100644 --- a/test/pkg/environment/common/expectations.go +++ b/test/pkg/environment/common/expectations.go @@ -91,6 +91,36 @@ func (env *Environment) ExpectUpdated(objects ...client.Object) { } } +// ExpectStatusUpdated will update objects in the cluster to match the inputs. +// WARNING: This ignores the resource version check, which can result in +// overwriting changes made by other controllers in the cluster. +// This is useful in ensuring that we can clean up resources by patching +// out finalizers. +// Grab the object before making the updates to reduce the chance of this race. +func (env *Environment) ExpectStatusUpdated(objects ...client.Object) { + GinkgoHelper() + for _, o := range objects { + Eventually(func(g Gomega) { + current := o.DeepCopyObject().(client.Object) + g.Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(current), current)).To(Succeed()) + if current.GetResourceVersion() != o.GetResourceVersion() { + log.FromContext(env).Info(fmt.Sprintf("detected an update to an object (%s) with an outdated resource version, did you get the latest version of the object before patching?", lo.Must(apiutil.GVKForObject(o, env.Client.Scheme())))) + } + o.SetResourceVersion(current.GetResourceVersion()) + g.Expect(env.Client.Status().Update(env.Context, o)).To(Succeed()) + }).WithTimeout(time.Second * 10).Should(Succeed()) + } +} + +func ReplaceNodeConditions(node *corev1.Node, conds ...corev1.NodeCondition) *corev1.Node { + keys := sets.New[string](lo.Map(conds, func(c corev1.NodeCondition, _ int) string { return string(c.Type) })...) + node.Status.Conditions = lo.Reject(node.Status.Conditions, func(c corev1.NodeCondition, _ int) bool { + return keys.Has(string(c.Type)) + }) + node.Status.Conditions = append(node.Status.Conditions, conds...) + return node +} + // ExpectCreatedOrUpdated can update objects in the cluster to match the inputs. // WARNING: ExpectUpdated ignores the resource version check, which can result in // overwriting changes made by other controllers in the cluster. @@ -272,6 +302,17 @@ func (env *Environment) EventuallyExpectTerminatingWithTimeout(timeout time.Dura }).WithTimeout(timeout).Should(Succeed()) } +func (env *Environment) EventuallyExpectNoLeakedKubeNodeLease() { + GinkgoHelper() + // expect no kube node lease to be leaked + leases := &coordinationv1.LeaseList{} + Expect(env.Client.List(env.Context, leases, client.InNamespace("kube-node-lease"))).To(Succeed()) + leakedLeases := lo.Filter(leases.Items, func(l coordinationv1.Lease, _ int) bool { + return l.OwnerReferences == nil + }) + Expect(leakedLeases).To(HaveLen(0)) +} + func (env *Environment) EventuallyExpectHealthyWithTimeout(timeout time.Duration, pods ...*corev1.Pod) { GinkgoHelper() Eventually(func(g Gomega) { @@ -296,6 +337,17 @@ func (env *Environment) ConsistentlyExpectTerminatingPods(duration time.Duration }, duration.String()).Should(Succeed()) } +func (env *Environment) ConsistentlyExpectActivePods(duration time.Duration, pods ...*corev1.Pod) { + GinkgoHelper() + By(fmt.Sprintf("expecting %d pods to be live for %s", len(pods), duration)) + Consistently(func(g Gomega) { + for _, pod := range pods { + g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(pod), pod)).To(Succeed()) + g.Expect(pod.DeletionTimestamp.IsZero()).To(BeTrue()) + } + }, duration.String()).Should(Succeed()) +} + func (env *Environment) ConsistentlyExpectHealthyPods(duration time.Duration, pods ...*corev1.Pod) { GinkgoHelper() By(fmt.Sprintf("expecting %d pods to be ready for %s", len(pods), duration)) @@ -462,16 +514,13 @@ func (env *Environment) eventuallyExpectScaleDown() { func (env *Environment) EventuallyExpectNotFound(objects ...client.Object) { GinkgoHelper() - env.EventuallyExpectNotFoundAssertion(objects...).Should(Succeed()) -} -func (env *Environment) EventuallyExpectNotFoundAssertion(objects ...client.Object) AsyncAssertion { - return Eventually(func(g Gomega) { + Eventually(func(g Gomega) { for _, object := range objects { err := env.Client.Get(env, client.ObjectKeyFromObject(object), object) g.Expect(errors.IsNotFound(err)).To(BeTrue()) } - }) + }).Should(Succeed()) } func (env *Environment) ExpectCreatedNodeCount(comparator string, count int) []*corev1.Node { @@ -524,34 +573,77 @@ func (env *Environment) ConsistentlyExpectNodeCount(comparator string, count int return lo.ToSlicePtr(nodeList.Items) } -func (env *Environment) ConsistentlyExpectNoDisruptions(nodeCount int, duration time.Duration) (taintedNodes []*corev1.Node) { +// ConsistentlyExpectNoDisruptions asserts that the number of tainted nodes remains the same. +// And that the number of nodeclaims remains the same. +func (env *Environment) ConsistentlyExpectNoDisruptions(nodeCount int, duration time.Duration) { GinkgoHelper() - return env.ConsistentlyExpectDisruptionsWithNodeCount(0, nodeCount, duration) + Consistently(func(g Gomega) { + nodeClaimList := &karpv1.NodeClaimList{} + g.Expect(env.Client.List(env, nodeClaimList, client.HasLabels{test.DiscoveryLabel})).To(Succeed()) + g.Expect(nodeClaimList.Items).To(HaveLen(nodeCount)) + nodeList := &corev1.NodeList{} + g.Expect(env.Client.List(env, nodeList, client.HasLabels{test.DiscoveryLabel})).To(Succeed()) + g.Expect(nodeList.Items).To(HaveLen(nodeCount)) + nodeList.Items = lo.Filter(nodeList.Items, func(n corev1.Node, _ int) bool { + _, ok := lo.Find(n.Spec.Taints, func(t corev1.Taint) bool { + return t.MatchTaint(&karpv1.DisruptedNoScheduleTaint) + }) + return ok + }) + g.Expect(nodeList.Items).To(HaveLen(0)) + }, duration).Should(Succeed()) } -// ConsistentlyExpectDisruptionsWithNodeCount will continually ensure that there are exactly disruptingNodes with totalNodes (including replacements and existing nodes) -func (env *Environment) ConsistentlyExpectDisruptionsWithNodeCount(disruptingNodes, totalNodes int, duration time.Duration) (taintedNodes []*corev1.Node) { +// ConsistentlyExpectDisruptionsUntilNoneLeft consistently ensures a max on number of concurrently disrupting and non-terminating nodes. +// This actually uses an Eventually() under the hood so that when we reach 0 tainted nodes we exit early. +// We use the StopTrying() so that we can exit the Eventually() if we've breached an assertion on total concurrency of disruptions. +// For example: if we have 5 nodes, with a budget of 2 nodes, we ensure that `disruptingNodes <= maxNodesDisrupting=2` +// We use nodesAtStart+maxNodesDisrupting to assert that we're not creating too many instances in replacement. +func (env *Environment) ConsistentlyExpectDisruptionsUntilNoneLeft(nodesAtStart, maxNodesDisrupting int, timeout time.Duration) { GinkgoHelper() nodes := []corev1.Node{} - Consistently(func(g Gomega) { - // Ensure we don't change our NodeClaims + // We use an eventually to exit when we detect the number of tainted/disrupted nodes matches our target. + Eventually(func(g Gomega) { + // Grab Nodes and NodeClaims nodeClaimList := &karpv1.NodeClaimList{} - g.Expect(env.Client.List(env, nodeClaimList, client.HasLabels{test.DiscoveryLabel})).To(Succeed()) - g.Expect(nodeClaimList.Items).To(HaveLen(totalNodes)) - nodeList := &corev1.NodeList{} + g.Expect(env.Client.List(env, nodeClaimList, client.HasLabels{test.DiscoveryLabel})).To(Succeed()) g.Expect(env.Client.List(env, nodeList, client.HasLabels{test.DiscoveryLabel})).To(Succeed()) - g.Expect(nodeList.Items).To(HaveLen(totalNodes)) + // Don't include NodeClaims with the `Terminating` status condition, as they're not included in budgets + removedProviderIDs := sets.Set[string]{} + nodeClaimList.Items = lo.Filter(nodeClaimList.Items, func(nc karpv1.NodeClaim, _ int) bool { + if !nc.StatusConditions().IsTrue(karpv1.ConditionTypeInstanceTerminating) { + return true + } + removedProviderIDs.Insert(nc.Status.ProviderID) + return false + }) + if len(nodeClaimList.Items) > nodesAtStart+maxNodesDisrupting { + StopTrying(fmt.Sprintf("Too many nodeclaims created. Expected no more than %d, got %d", nodesAtStart+maxNodesDisrupting, len(nodeClaimList.Items))).Now() + } + + // Don't include Nodes whose NodeClaims have been ignored + nodeList.Items = lo.Filter(nodeList.Items, func(n corev1.Node, _ int) bool { + return !removedProviderIDs.Has(n.Spec.ProviderID) + }) + if len(nodeList.Items) > nodesAtStart+maxNodesDisrupting { + StopTrying(fmt.Sprintf("Too many nodes created. Expected no more than %d, got %d", nodesAtStart+maxNodesDisrupting, len(nodeList.Items))).Now() + } + + // Filter further by the number of tainted nodes to get the number of nodes that are disrupting nodes = lo.Filter(nodeList.Items, func(n corev1.Node, _ int) bool { _, ok := lo.Find(n.Spec.Taints, func(t corev1.Taint) bool { - return karpv1.IsDisruptingTaint(t) + return t.MatchTaint(&karpv1.DisruptedNoScheduleTaint) }) return ok }) - g.Expect(nodes).To(HaveLen(disruptingNodes)) - }, duration).Should(Succeed()) - return lo.ToSlicePtr(nodes) + if len(nodes) > maxNodesDisrupting { + StopTrying(fmt.Sprintf("Too many disruptions detected. Expected no more than %d, got %d", maxNodesDisrupting, len(nodeList.Items))).Now() + } + + g.Expect(nodes).To(HaveLen(0)) + }).WithTimeout(timeout).WithPolling(5 * time.Second).Should(Succeed()) } func (env *Environment) EventuallyExpectTaintedNodeCount(comparator string, count int) []*corev1.Node { diff --git a/test/suites/nodeclaim/nodeclaim_test.go b/test/suites/nodeclaim/nodeclaim_test.go index 9b9b32d78..2d35a176a 100644 --- a/test/suites/nodeclaim/nodeclaim_test.go +++ b/test/suites/nodeclaim/nodeclaim_test.go @@ -18,6 +18,7 @@ package nodeclaim_test import ( "github.com/Azure/karpenter-provider-azure/pkg/apis/v1alpha2" + "github.com/awslabs/operatorpkg/object" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" v1 "k8s.io/api/core/v1" @@ -46,7 +47,9 @@ var _ = Describe("StandaloneNodeClaim", func() { }}, }, NodeClassRef: &karpv1.NodeClassReference{ - Name: nodeClass.Name, + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, }, }, }) @@ -66,7 +69,9 @@ var _ = Describe("StandaloneNodeClaim", func() { }, }, NodeClassRef: &karpv1.NodeClassReference{ - Name: nodeClass.Name, + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, }, }, }) @@ -100,7 +105,9 @@ var _ = Describe("StandaloneNodeClaim", func() { }, }, NodeClassRef: &karpv1.NodeClassReference{ - Name: nodeClass.Name, + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, }, }, }) @@ -139,7 +146,9 @@ var _ = Describe("StandaloneNodeClaim", func() { }}, }, NodeClassRef: &karpv1.NodeClassReference{ - Name: nodeClass.Name, + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, }, }, }) From 71158c91b791d20f899afc145b94c3ffb2f1fd53 Mon Sep 17 00:00:00 2001 From: Alex Leites <18728999+tallaxes@users.noreply.github.com> Date: Tue, 3 Dec 2024 02:56:49 +0000 Subject: [PATCH 07/19] chore: add repair policy test --- test/suites/integration/repair_policy_test.go | 127 ++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 test/suites/integration/repair_policy_test.go diff --git a/test/suites/integration/repair_policy_test.go b/test/suites/integration/repair_policy_test.go new file mode 100644 index 000000000..e40422be8 --- /dev/null +++ b/test/suites/integration/repair_policy_test.go @@ -0,0 +1,127 @@ +// Portions Copyright (c) Microsoft Corporation. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package integration_test + +import ( + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + karpenterv1 "sigs.k8s.io/karpenter/pkg/apis/v1" + coretest "sigs.k8s.io/karpenter/pkg/test" + + "github.com/Azure/karpenter-provider-azure/test/pkg/environment/common" + + . "github.com/onsi/ginkgo/v2" + "github.com/samber/lo" +) + +var _ = Describe("Repair Policy", func() { + var selector labels.Selector + var dep *appsv1.Deployment + var numPods int + var unhealthyCondition corev1.NodeCondition + + BeforeEach(func() { + unhealthyCondition = corev1.NodeCondition{ + Type: corev1.NodeReady, + Status: corev1.ConditionFalse, + LastTransitionTime: metav1.Time{Time: time.Now().Add(-31 * time.Minute)}, + } + numPods = 1 + // Add pods with a do-not-disrupt annotation so that we can check node metadata before we disrupt + dep = coretest.Deployment(coretest.DeploymentOptions{ + Replicas: int32(numPods), + PodOptions: coretest.PodOptions{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "my-app", + }, + Annotations: map[string]string{ + karpenterv1.DoNotDisruptAnnotationKey: "true", + }, + }, + TerminationGracePeriodSeconds: lo.ToPtr[int64](0), + }, + }) + selector = labels.SelectorFromSet(dep.Spec.Selector.MatchLabels) + }) + + DescribeTable("Conditions", func(unhealthyCondition corev1.NodeCondition) { + env.ExpectCreated(nodeClass, nodePool, dep) + pod := env.EventuallyExpectHealthyPodCount(selector, numPods)[0] + node := env.ExpectCreatedNodeCount("==", 1)[0] + env.EventuallyExpectInitializedNodeCount("==", 1) + + node = common.ReplaceNodeConditions(node, unhealthyCondition) + env.ExpectStatusUpdated(node) + + env.EventuallyExpectNotFound(pod, node) + env.EventuallyExpectHealthyPodCount(selector, numPods) + }, + Entry("Readiness", corev1.NodeCondition{ + Type: corev1.NodeReady, + Status: corev1.ConditionFalse, + LastTransitionTime: metav1.Time{Time: time.Now().Add(-31 * time.Minute)}, + }), + ) + It("should ignore disruption budgets", func() { + nodePool.Spec.Disruption.Budgets = []karpenterv1.Budget{ + { + Nodes: "0", + }, + } + env.ExpectCreated(nodeClass, nodePool, dep) + pod := env.EventuallyExpectHealthyPodCount(selector, numPods)[0] + node := env.ExpectCreatedNodeCount("==", 1)[0] + env.EventuallyExpectInitializedNodeCount("==", 1) + + node = common.ReplaceNodeConditions(node, unhealthyCondition) + env.ExpectStatusUpdated(node) + + env.EventuallyExpectNotFound(pod, node) + env.EventuallyExpectHealthyPodCount(selector, numPods) + }) + It("should ignore do-not-disrupt annotation on node", func() { + env.ExpectCreated(nodeClass, nodePool, dep) + pod := env.EventuallyExpectHealthyPodCount(selector, numPods)[0] + node := env.ExpectCreatedNodeCount("==", 1)[0] + env.EventuallyExpectInitializedNodeCount("==", 1) + + node.Annotations[karpenterv1.DoNotDisruptAnnotationKey] = "true" + env.ExpectUpdated(node) + + node = common.ReplaceNodeConditions(node, unhealthyCondition) + env.ExpectStatusUpdated(node) + + env.EventuallyExpectNotFound(pod, node) + env.EventuallyExpectHealthyPodCount(selector, numPods) + }) + It("should ignore terminationGracePeriod on the nodepool", func() { + nodePool.Spec.Template.Spec.TerminationGracePeriod = &metav1.Duration{Duration: time.Hour} + env.ExpectCreated(nodeClass, nodePool, dep) + pod := env.EventuallyExpectHealthyPodCount(selector, numPods)[0] + node := env.ExpectCreatedNodeCount("==", 1)[0] + env.EventuallyExpectInitializedNodeCount("==", 1) + + node = common.ReplaceNodeConditions(node, unhealthyCondition) + env.ExpectStatusUpdated(node) + + env.EventuallyExpectNotFound(pod, node) + env.EventuallyExpectHealthyPodCount(selector, numPods) + }) +}) From f4fbb635fa801c03230da30c1a847dccb8e64f04 Mon Sep 17 00:00:00 2001 From: Alex Leites <18728999+tallaxes@users.noreply.github.com> Date: Tue, 3 Dec 2024 02:57:23 +0000 Subject: [PATCH 08/19] chore: add node repair feature gate --- charts/karpenter/templates/deployment.yaml | 3 +-- charts/karpenter/values.yaml | 3 +++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/charts/karpenter/templates/deployment.yaml b/charts/karpenter/templates/deployment.yaml index 5b258ff4f..bd0c0551a 100644 --- a/charts/karpenter/templates/deployment.yaml +++ b/charts/karpenter/templates/deployment.yaml @@ -103,8 +103,7 @@ spec: divisor: "0" resource: limits.memory - name: FEATURE_GATES - value: "SpotToSpotConsolidation={{ .Values.settings.featureGates.spotToSpotConsolidation }}" - {{- with .Values.settings.batchMaxDuration }} + value: "SpotToSpotConsolidation={{ .Values.settings.featureGates.spotToSpotConsolidation }},NodeRepair={{ .Values.settings.featureGates.nodeRepair }}" {{- with .Values.settings.batchMaxDuration }} - name: BATCH_MAX_DURATION value: "{{ . }}" {{- end }} diff --git a/charts/karpenter/values.yaml b/charts/karpenter/values.yaml index 4c4260d46..7fe6e8b87 100644 --- a/charts/karpenter/values.yaml +++ b/charts/karpenter/values.yaml @@ -174,3 +174,6 @@ settings: # -- spotToSpotConsolidation is ALPHA and is disabled by default. # Setting this to true will enable spot replacement consolidation for both single and multi-node consolidation. spotToSpotConsolidation: false + # -- nodeRepair is ALPHA and is disabled by default. + # Setting this to true will enable node repair. + nodeRepair: false From 31d672d80c2533f3193a893f7b7581cf20f289c8 Mon Sep 17 00:00:00 2001 From: Alex Leites <18728999+tallaxes@users.noreply.github.com> Date: Tue, 3 Dec 2024 07:40:05 +0000 Subject: [PATCH 09/19] chore: fix install-deps action and dependabot config --- .github/actions/install-deps/{action.yml => action.yaml} | 9 +++++---- .github/dependabot.yaml | 8 ++++++++ 2 files changed, 13 insertions(+), 4 deletions(-) rename .github/actions/install-deps/{action.yml => action.yaml} (79%) diff --git a/.github/actions/install-deps/action.yml b/.github/actions/install-deps/action.yaml similarity index 79% rename from .github/actions/install-deps/action.yml rename to .github/actions/install-deps/action.yaml index 47068256c..b1182941d 100644 --- a/.github/actions/install-deps/action.yml +++ b/.github/actions/install-deps/action.yaml @@ -3,11 +3,12 @@ description: 'Installs Go Downloads and installs Karpenter Dependencies' inputs: k8sVersion: description: Kubernetes version to use when installing the toolchain - default: "1.27.x" + default: "1.31.x" runs: using: "composite" steps: - - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 + - uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0 + id: setup-go with: go-version-file: go.mod check-latest: true @@ -15,7 +16,7 @@ runs: # Root path permission workaround for caching https://github.com/actions/cache/issues/845#issuecomment-1252594999 - run: sudo chown "$USER" /usr/local shell: bash - - uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 + - uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 id: cache-toolchain with: path: | @@ -26,4 +27,4 @@ runs: shell: bash env: K8S_VERSION: ${{ inputs.k8sVersion }} - run: make toolchain \ No newline at end of file + run: make toolchain diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml index a458ffa76..2ae5745c1 100644 --- a/.github/dependabot.yaml +++ b/.github/dependabot.yaml @@ -123,3 +123,11 @@ updates: action-deps: patterns: - '*' + - package-ecosystem: github-actions + directory: .github/actions/install-deps + schedule: + interval: weekly + groups: + action-deps: + patterns: + - '*' From 67b61d73c46f33e438046dc91773019534c50d15 Mon Sep 17 00:00:00 2001 From: Alex Leites <18728999+tallaxes@users.noreply.github.com> Date: Wed, 4 Dec 2024 02:24:40 +0000 Subject: [PATCH 10/19] chore: remove go toolchain --- go.mod | 2 -- 1 file changed, 2 deletions(-) diff --git a/go.mod b/go.mod index ca025ef6e..070b9f624 100644 --- a/go.mod +++ b/go.mod @@ -2,8 +2,6 @@ module github.com/Azure/karpenter-provider-azure go 1.23.2 -toolchain go1.23.3 - require ( github.com/Azure/azure-kusto-go v0.16.1 github.com/Azure/azure-sdk-for-go v68.0.0+incompatible From 7374529c55063739fc33994960ecdf4d9871145e Mon Sep 17 00:00:00 2001 From: Alex Leites <18728999+tallaxes@users.noreply.github.com> Date: Wed, 4 Dec 2024 04:34:44 +0000 Subject: [PATCH 11/19] chore: fix ci --- Makefile | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index 966c2279b..56ee0e18d 100644 --- a/Makefile +++ b/Makefile @@ -23,20 +23,16 @@ help: ## Display help presubmit: verify test ## Run all steps in the developer loop -ci-test: battletest coverage ## Runs tests and submits coverage +ci-test: test coverage ## Runs tests and submits coverage ci-non-test: verify vulncheck ## Runs checks other than tests test: ## Run tests - ginkgo -v --focus="${FOCUS}" ./pkg/$(shell echo $(TEST_SUITE) | tr A-Z a-z) - -battletest: ## Run randomized, racing, code-covered tests - ginkgo -v \ + ginkgo -vv \ -race \ -cover -coverprofile=coverage.out -output-dir=. -coverpkg=./pkg/... \ --focus="${FOCUS}" \ --randomize-all \ - -tags random_test_delay \ ./pkg/... e2etests: ## Run the e2e suite against your local cluster @@ -59,10 +55,7 @@ e2etests: ## Run the e2e suite against your local cluster benchmark: go test -tags=test_performance -run=NoTests -bench=. ./... -deflake: ## Run randomized, racing, code-covered tests to deflake failures - for i in $(shell seq 1 5); do make battletest || exit 1; done - -deflake-until-it-fails: ## Run randomized, racing tests until the test fails to catch flakes +deflake: ## Run randomized, racing tests until the test fails to catch flakes ginkgo \ --race \ --focus="${FOCUS}" \ From 17c854a2fc3521c5e425c88ba8cc86f278c001ea Mon Sep 17 00:00:00 2001 From: Alex Leites <18728999+tallaxes@users.noreply.github.com> Date: Wed, 4 Dec 2024 05:12:24 +0000 Subject: [PATCH 12/19] chore: update Makefile --- Makefile | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index 56ee0e18d..7652d48ae 100644 --- a/Makefile +++ b/Makefile @@ -25,16 +25,24 @@ presubmit: verify test ## Run all steps in the developer loop ci-test: test coverage ## Runs tests and submits coverage -ci-non-test: verify vulncheck ## Runs checks other than tests +ci-non-test: verify licenses vulncheck ## Runs checks other than tests test: ## Run tests ginkgo -vv \ - -race \ -cover -coverprofile=coverage.out -output-dir=. -coverpkg=./pkg/... \ --focus="${FOCUS}" \ --randomize-all \ ./pkg/... +deflake: ## Run randomized, racing tests until the test fails to catch flakes + ginkgo \ + --race \ + --focus="${FOCUS}" \ + --randomize-all \ + --until-it-fails \ + -v \ + ./pkg/... + e2etests: ## Run the e2e suite against your local cluster # Notes: # -p: the number of programs, such as build commands or test binaries, that can be run in parallel? @@ -55,15 +63,6 @@ e2etests: ## Run the e2e suite against your local cluster benchmark: go test -tags=test_performance -run=NoTests -bench=. ./... -deflake: ## Run randomized, racing tests until the test fails to catch flakes - ginkgo \ - --race \ - --focus="${FOCUS}" \ - --randomize-all \ - --until-it-fails \ - -v \ - ./pkg/... - coverage: go tool cover -html coverage.out -o coverage.html @@ -95,6 +94,9 @@ verify: toolchain tidy download ## Verify code. Includes dependencies, linting, vulncheck: ## Verify code vulnerabilities @govulncheck ./pkg/... +licenses: download ## Verifies dependency licenses + ! go-licenses csv ./... | grep -v -e 'MIT' -e 'Apache-2.0' -e 'BSD-3-Clause' -e 'BSD-2-Clause' -e 'ISC' -e 'MPL-2.0' + codegen: ## Auto generate files based on Azure API responses ./hack/codegen.sh @@ -113,7 +115,7 @@ tidy: ## Recursively "go mod tidy" on all directories where go.mod exists download: ## Recursively "go mod download" on all directories where go.mod exists $(foreach dir,$(MOD_DIRS),cd $(dir) && go mod download $(newline)) -.PHONY: help test battletest e2etests verify tidy download codegen toolchain vulncheck snapshot release +.PHONY: help presubmit ci-test ci-non-test test deflake e2etests coverage verify vulncheck licenses codegen snapshot release toolchain tidy download define newline From c370ef2f6d00b452a28625153746f506b498e1bd Mon Sep 17 00:00:00 2001 From: Alex Leites <18728999+tallaxes@users.noreply.github.com> Date: Wed, 4 Dec 2024 06:57:27 +0000 Subject: [PATCH 13/19] fix: skip debug controller name validation --- test/pkg/debug/nodeclaim.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/pkg/debug/nodeclaim.go b/test/pkg/debug/nodeclaim.go index 635fb0268..1d65bf9b0 100644 --- a/test/pkg/debug/nodeclaim.go +++ b/test/pkg/debug/nodeclaim.go @@ -21,6 +21,7 @@ import ( "fmt" "time" + "github.com/samber/lo" "k8s.io/apimachinery/pkg/api/errors" controllerruntime "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -75,6 +76,6 @@ func (c *NodeClaimController) Register(_ context.Context, m manager.Manager) err return c.GetInfo(oldNodeClaim) != c.GetInfo(newNodeClaim) }, }). - WithOptions(controller.Options{MaxConcurrentReconciles: 10}). + WithOptions(controller.Options{MaxConcurrentReconciles: 10, SkipNameValidation: lo.ToPtr(true)}). Complete(c) } From 35cdaba56a0e8a14ff40820ebbf11ed07e965457 Mon Sep 17 00:00:00 2001 From: Alex Leites <18728999+tallaxes@users.noreply.github.com> Date: Wed, 4 Dec 2024 17:38:43 +0000 Subject: [PATCH 14/19] fix: skip debug controller name validation --- test/pkg/debug/node.go | 3 ++- test/pkg/debug/pod.go | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/test/pkg/debug/node.go b/test/pkg/debug/node.go index 385628eed..ddea2719f 100644 --- a/test/pkg/debug/node.go +++ b/test/pkg/debug/node.go @@ -21,6 +21,7 @@ import ( "fmt" "time" + "github.com/samber/lo" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" controllerruntime "sigs.k8s.io/controller-runtime" @@ -79,6 +80,6 @@ func (c *NodeController) Register(ctx context.Context, m manager.Manager) error return o.GetLabels()[karpv1.NodePoolLabelKey] != "" }), )). - WithOptions(controller.Options{MaxConcurrentReconciles: 10}). + WithOptions(controller.Options{MaxConcurrentReconciles: 10, SkipNameValidation: lo.ToPtr(true)}). Complete(c) } diff --git a/test/pkg/debug/pod.go b/test/pkg/debug/pod.go index 2f27a7d41..1772b0790 100644 --- a/test/pkg/debug/pod.go +++ b/test/pkg/debug/pod.go @@ -86,6 +86,6 @@ func (c *PodController) Register(_ context.Context, m manager.Manager) error { return o.GetNamespace() != "kube-system" }), )). - WithOptions(controller.Options{MaxConcurrentReconciles: 10}). + WithOptions(controller.Options{MaxConcurrentReconciles: 10, SkipNameValidation: lo.ToPtr(true)}). Complete(c) } From ca7d505822a47fa9dc245274d63a103249b64e38 Mon Sep 17 00:00:00 2001 From: Alex Leites <18728999+tallaxes@users.noreply.github.com> Date: Wed, 8 Jan 2025 00:52:41 +0000 Subject: [PATCH 15/19] chore: remove comment --- pkg/providers/instancetype/suite_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/providers/instancetype/suite_test.go b/pkg/providers/instancetype/suite_test.go index 16553d8fb..617dee1c5 100644 --- a/pkg/providers/instancetype/suite_test.go +++ b/pkg/providers/instancetype/suite_test.go @@ -407,7 +407,6 @@ var _ = Describe("InstanceType Provider", func() { It("should use ephemeral disk if supported, and set disk size to OSDiskSizeGB from node class", func() { // Create a Nodepool that selects a sku that supports ephemeral // SKU Standard_D64s_v3 has 1600GB of CacheDisk space, so we expect we can create an ephemeral disk with size 256GB - // nodeClass = test.AKSNodeClass() nodeClass.Spec.OSDiskSizeGB = lo.ToPtr[int32](256) nodePool.Spec.Template.Spec.Requirements = append(nodePool.Spec.Template.Spec.Requirements, karpv1.NodeSelectorRequirementWithMinValues{ NodeSelectorRequirement: v1.NodeSelectorRequirement{ From 15966099c4f71e95916630dc846f9d0dbc92bc53 Mon Sep 17 00:00:00 2001 From: Alex Leites <18728999+tallaxes@users.noreply.github.com> Date: Wed, 8 Jan 2025 01:01:31 +0000 Subject: [PATCH 16/19] chore: update sig.k8s.io/karpenter to v1.1.1 --- go.mod | 7 +++---- go.sum | 14 ++++++-------- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 9f0bfce49..d659b0512 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/Azure/skewer v0.0.19 github.com/Pallinder/go-randomdata v1.2.0 github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 - github.com/awslabs/operatorpkg v0.0.0-20241125173122-bef8fba1bdf6 + github.com/awslabs/operatorpkg v0.0.0-20241205163410-0fff9f28d115 github.com/blang/semver/v4 v4.0.0 github.com/go-logr/zapr v1.3.0 github.com/go-openapi/errors v0.22.0 @@ -46,8 +46,8 @@ require ( k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 knative.dev/pkg v0.0.0-20240910170930-fdbc0b5adde7 sigs.k8s.io/cloud-provider-azure v1.29.3 - sigs.k8s.io/controller-runtime v0.19.2 - sigs.k8s.io/karpenter v1.1.0 + sigs.k8s.io/controller-runtime v0.19.3 + sigs.k8s.io/karpenter v1.1.1 ) require ( @@ -81,7 +81,6 @@ require ( github.com/emicklei/go-restful/v3 v3.12.1 // indirect github.com/evanphx/json-patch/v5 v5.9.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/gabriel-vasile/mimetype v1.4.3 // indirect github.com/go-kit/log v0.2.1 // indirect diff --git a/go.sum b/go.sum index d938c6266..c29dc6158 100644 --- a/go.sum +++ b/go.sum @@ -125,8 +125,8 @@ github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3d github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0= github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY= -github.com/awslabs/operatorpkg v0.0.0-20241125173122-bef8fba1bdf6 h1:O32mqYd4TFNsiZP93RSPfkAKaskV0vP8pBcwTkR9p2M= -github.com/awslabs/operatorpkg v0.0.0-20241125173122-bef8fba1bdf6/go.mod h1:jina2fQk+b3oa9r5bRuMDbpy6mfhTGuruh05oVVWwnk= +github.com/awslabs/operatorpkg v0.0.0-20241205163410-0fff9f28d115 h1:9nhjY3dzCpEmhpQ0vMlhB7wqucAiftLjAIEQu8uT2J4= +github.com/awslabs/operatorpkg v0.0.0-20241205163410-0fff9f28d115/go.mod h1:TTs6HGuqmgdNyNlbdv29v1OoON+kQKVPojZgJaJVtNk= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -166,8 +166,6 @@ github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0 github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= -github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= @@ -891,12 +889,12 @@ sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.0.26 h1:BHauRhfjzs4UWu/yiLw82WK sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.0.26/go.mod h1:02JRJ7ioAoT9PZzIxlR4Kw7WbejsMIy1eeDyYX8sgvk= sigs.k8s.io/cloud-provider-azure/pkg/azclient/configloader v0.0.1 h1:Lp0nALZmvMJoiVeVV6XjnZv1uClfArnThhuDAjaqE5A= sigs.k8s.io/cloud-provider-azure/pkg/azclient/configloader v0.0.1/go.mod h1:pPkJPx/eMVWP3R+LhPoOYGoY7lywcMJev5L2uSfH+Jo= -sigs.k8s.io/controller-runtime v0.19.2 h1:3sPrF58XQEPzbE8T81TN6selQIMGbtYwuaJ6eDssDF8= -sigs.k8s.io/controller-runtime v0.19.2/go.mod h1:iRmWllt8IlaLjvTTDLhRBXIEtkCK6hwVBJJsYS9Ajf4= +sigs.k8s.io/controller-runtime v0.19.3 h1:XO2GvC9OPftRst6xWCpTgBZO04S2cbp0Qqkj8bX1sPw= +sigs.k8s.io/controller-runtime v0.19.3/go.mod h1:j4j87DqtsThvwTv5/Tc5NFRyyF/RF0ip4+62tbTSIUM= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/karpenter v1.1.0 h1:4SdCXihNKH4Y5xLwtKh7sqvCPYTQ3vCvCTPFRaNPQ5Y= -sigs.k8s.io/karpenter v1.1.0/go.mod h1:LPVsUkC/MdmIjCBcmNVzy1wLp86xRKTRKFGPSNAW+YE= +sigs.k8s.io/karpenter v1.1.1 h1:QPpVC8DsaLgJ/YWcFpZKE4m3jD+Qp88/GtSPvMfffck= +sigs.k8s.io/karpenter v1.1.1/go.mod h1:NQouOJNK6s1d4EIKa5cY7nAV3IG74qZ6gPzHBeCZNPw= sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= From 011ed1b2160e45a73759bf0f233d43e1b419c53f Mon Sep 17 00:00:00 2001 From: Alex Leites <18728999+tallaxes@users.noreply.github.com> Date: Wed, 8 Jan 2025 03:34:50 +0000 Subject: [PATCH 17/19] chore: update comment --- pkg/providers/instancetype/suite_test.go | 2 +- test/suites/integration/termination_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/providers/instancetype/suite_test.go b/pkg/providers/instancetype/suite_test.go index 617dee1c5..8d1e60fc8 100644 --- a/pkg/providers/instancetype/suite_test.go +++ b/pkg/providers/instancetype/suite_test.go @@ -381,7 +381,7 @@ var _ = Describe("InstanceType Provider", func() { Context("Ephemeral Disk", func() { It("should use ephemeral disk if supported, and has space of at least 128GB by default", func() { - // Create a Provisioner that selects a sku that supports ephemeral + // Create a NodePool that selects a sku that supports ephemeral // SKU Standard_D64s_v3 has 1600GB of CacheDisk space, so we expect we can create an ephemeral disk with size 128GB nodePool.Spec.Template.Spec.Requirements = append(nodePool.Spec.Template.Spec.Requirements, karpv1.NodeSelectorRequirementWithMinValues{ NodeSelectorRequirement: v1.NodeSelectorRequirement{ diff --git a/test/suites/integration/termination_test.go b/test/suites/integration/termination_test.go index 83bdd1a0f..44e4611d6 100644 --- a/test/suites/integration/termination_test.go +++ b/test/suites/integration/termination_test.go @@ -32,7 +32,7 @@ var _ = Describe("Termination", func() { nodes := env.Monitor.CreatedNodes() // Pod is deleted so that we don't re-provision after node deletion - // NOTE: We have to do this right now to deal with a race condition in provisioner ownership + // NOTE: We have to do this right now to deal with a race condition in nodepool ownership // This can be removed once this race is resolved with the Machine env.ExpectDeleted(pod) From 6e66e9aa152c1c2b8e3fb95f8db46200eaa7dc38 Mon Sep 17 00:00:00 2001 From: Alex Leites <18728999+tallaxes@users.noreply.github.com> Date: Wed, 8 Jan 2025 09:20:48 +0000 Subject: [PATCH 18/19] fix: allow nodeclass label --- hack/validation/labels.sh | 1 + hack/validation/requirements.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/hack/validation/labels.sh b/hack/validation/labels.sh index 8bbeed5c0..56ceacb62 100755 --- a/hack/validation/labels.sh +++ b/hack/validation/labels.sh @@ -6,6 +6,7 @@ set -euo pipefail rule=$'self.all(x, x in [ + "karpenter.azure.com/aksnodeclass", "karpenter.azure.com/sku-name", "karpenter.azure.com/sku-family", "karpenter.azure.com/sku-version", diff --git a/hack/validation/requirements.sh b/hack/validation/requirements.sh index 29acd5410..989cc7310 100755 --- a/hack/validation/requirements.sh +++ b/hack/validation/requirements.sh @@ -6,6 +6,7 @@ set -euo pipefail rule=$'self in [ + "karpenter.azure.com/aksnodeclass", "karpenter.azure.com/sku-name", "karpenter.azure.com/sku-family", "karpenter.azure.com/sku-version", From c79549775b0ec5978944ef8cf81e3c7bdfd1cdf2 Mon Sep 17 00:00:00 2001 From: Alex Leites <18728999+tallaxes@users.noreply.github.com> Date: Wed, 8 Jan 2025 09:21:42 +0000 Subject: [PATCH 19/19] chore: regen CRDs --- charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml | 2 +- charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml | 4 ++-- pkg/apis/crds/karpenter.sh_nodeclaims.yaml | 2 +- pkg/apis/crds/karpenter.sh_nodepools.yaml | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml b/charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml index b12e4f7aa..e8fb08d58 100644 --- a/charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml +++ b/charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml @@ -130,7 +130,7 @@ spec: - message: label "kubernetes.io/hostname" is restricted rule: self != "kubernetes.io/hostname" - message: label domain "karpenter.azure.com" is restricted - rule: self in [ "karpenter.azure.com/sku-name", "karpenter.azure.com/sku-family", "karpenter.azure.com/sku-version", "karpenter.azure.com/sku-cpu", "karpenter.azure.com/sku-memory", "karpenter.azure.com/sku-accelerator", "karpenter.azure.com/sku-networking-accelerated", "karpenter.azure.com/sku-storage-premium-capable", "karpenter.azure.com/sku-storage-ephemeralos-maxsize", "karpenter.azure.com/sku-encryptionathost-capable", "karpenter.azure.com/sku-gpu-name", "karpenter.azure.com/sku-gpu-manufacturer", "karpenter.azure.com/sku-gpu-count" ] || !self.find("^([^/]+)").endsWith("karpenter.azure.com") + rule: self in [ "karpenter.azure.com/aksnodeclass", "karpenter.azure.com/sku-name", "karpenter.azure.com/sku-family", "karpenter.azure.com/sku-version", "karpenter.azure.com/sku-cpu", "karpenter.azure.com/sku-memory", "karpenter.azure.com/sku-accelerator", "karpenter.azure.com/sku-networking-accelerated", "karpenter.azure.com/sku-storage-premium-capable", "karpenter.azure.com/sku-storage-ephemeralos-maxsize", "karpenter.azure.com/sku-encryptionathost-capable", "karpenter.azure.com/sku-gpu-name", "karpenter.azure.com/sku-gpu-manufacturer", "karpenter.azure.com/sku-gpu-count" ] || !self.find("^([^/]+)").endsWith("karpenter.azure.com") minValues: description: |- This field is ALPHA and can be dropped or replaced at any time diff --git a/charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml b/charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml index d8a0c9a31..d1da5dd18 100644 --- a/charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml +++ b/charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml @@ -207,7 +207,7 @@ spec: - message: label "kubernetes.io/hostname" is restricted rule: self.all(x, x != "kubernetes.io/hostname") - message: label domain "karpenter.azure.com" is restricted - rule: self.all(x, x in [ "karpenter.azure.com/sku-name", "karpenter.azure.com/sku-family", "karpenter.azure.com/sku-version", "karpenter.azure.com/sku-cpu", "karpenter.azure.com/sku-memory", "karpenter.azure.com/sku-accelerator", "karpenter.azure.com/sku-networking-accelerated", "karpenter.azure.com/sku-storage-premium-capable", "karpenter.azure.com/sku-storage-ephemeralos-maxsize", "karpenter.azure.com/sku-encryptionathost-capable", "karpenter.azure.com/sku-gpu-name", "karpenter.azure.com/sku-gpu-manufacturer", "karpenter.azure.com/sku-gpu-count" ] || !x.find("^([^/]+)").endsWith("karpenter.azure.com")) + rule: self.all(x, x in [ "karpenter.azure.com/aksnodeclass", "karpenter.azure.com/sku-name", "karpenter.azure.com/sku-family", "karpenter.azure.com/sku-version", "karpenter.azure.com/sku-cpu", "karpenter.azure.com/sku-memory", "karpenter.azure.com/sku-accelerator", "karpenter.azure.com/sku-networking-accelerated", "karpenter.azure.com/sku-storage-premium-capable", "karpenter.azure.com/sku-storage-ephemeralos-maxsize", "karpenter.azure.com/sku-encryptionathost-capable", "karpenter.azure.com/sku-gpu-name", "karpenter.azure.com/sku-gpu-manufacturer", "karpenter.azure.com/sku-gpu-count" ] || !x.find("^([^/]+)").endsWith("karpenter.azure.com")) type: object spec: description: |- @@ -280,7 +280,7 @@ spec: - message: label "kubernetes.io/hostname" is restricted rule: self != "kubernetes.io/hostname" - message: label domain "karpenter.azure.com" is restricted - rule: self in [ "karpenter.azure.com/sku-name", "karpenter.azure.com/sku-family", "karpenter.azure.com/sku-version", "karpenter.azure.com/sku-cpu", "karpenter.azure.com/sku-memory", "karpenter.azure.com/sku-accelerator", "karpenter.azure.com/sku-networking-accelerated", "karpenter.azure.com/sku-storage-premium-capable", "karpenter.azure.com/sku-storage-ephemeralos-maxsize", "karpenter.azure.com/sku-encryptionathost-capable", "karpenter.azure.com/sku-gpu-name", "karpenter.azure.com/sku-gpu-manufacturer", "karpenter.azure.com/sku-gpu-count" ] || !self.find("^([^/]+)").endsWith("karpenter.azure.com") + rule: self in [ "karpenter.azure.com/aksnodeclass", "karpenter.azure.com/sku-name", "karpenter.azure.com/sku-family", "karpenter.azure.com/sku-version", "karpenter.azure.com/sku-cpu", "karpenter.azure.com/sku-memory", "karpenter.azure.com/sku-accelerator", "karpenter.azure.com/sku-networking-accelerated", "karpenter.azure.com/sku-storage-premium-capable", "karpenter.azure.com/sku-storage-ephemeralos-maxsize", "karpenter.azure.com/sku-encryptionathost-capable", "karpenter.azure.com/sku-gpu-name", "karpenter.azure.com/sku-gpu-manufacturer", "karpenter.azure.com/sku-gpu-count" ] || !self.find("^([^/]+)").endsWith("karpenter.azure.com") minValues: description: |- This field is ALPHA and can be dropped or replaced at any time diff --git a/pkg/apis/crds/karpenter.sh_nodeclaims.yaml b/pkg/apis/crds/karpenter.sh_nodeclaims.yaml index 0e17daa43..0d1977902 100644 --- a/pkg/apis/crds/karpenter.sh_nodeclaims.yaml +++ b/pkg/apis/crds/karpenter.sh_nodeclaims.yaml @@ -130,7 +130,7 @@ spec: - message: label "kubernetes.io/hostname" is restricted rule: self != "kubernetes.io/hostname" - message: label domain "karpenter.azure.com" is restricted - rule: self in [ "karpenter.azure.com/sku-name", "karpenter.azure.com/sku-family", "karpenter.azure.com/sku-version", "karpenter.azure.com/sku-cpu", "karpenter.azure.com/sku-memory", "karpenter.azure.com/sku-accelerator", "karpenter.azure.com/sku-networking-accelerated", "karpenter.azure.com/sku-storage-premium-capable", "karpenter.azure.com/sku-storage-ephemeralos-maxsize", "karpenter.azure.com/sku-encryptionathost-capable", "karpenter.azure.com/sku-gpu-name", "karpenter.azure.com/sku-gpu-manufacturer", "karpenter.azure.com/sku-gpu-count" ] || !self.find("^([^/]+)").endsWith("karpenter.azure.com") + rule: self in [ "karpenter.azure.com/aksnodeclass", "karpenter.azure.com/sku-name", "karpenter.azure.com/sku-family", "karpenter.azure.com/sku-version", "karpenter.azure.com/sku-cpu", "karpenter.azure.com/sku-memory", "karpenter.azure.com/sku-accelerator", "karpenter.azure.com/sku-networking-accelerated", "karpenter.azure.com/sku-storage-premium-capable", "karpenter.azure.com/sku-storage-ephemeralos-maxsize", "karpenter.azure.com/sku-encryptionathost-capable", "karpenter.azure.com/sku-gpu-name", "karpenter.azure.com/sku-gpu-manufacturer", "karpenter.azure.com/sku-gpu-count" ] || !self.find("^([^/]+)").endsWith("karpenter.azure.com") minValues: description: |- This field is ALPHA and can be dropped or replaced at any time diff --git a/pkg/apis/crds/karpenter.sh_nodepools.yaml b/pkg/apis/crds/karpenter.sh_nodepools.yaml index 2465ae81f..8acfcfe93 100644 --- a/pkg/apis/crds/karpenter.sh_nodepools.yaml +++ b/pkg/apis/crds/karpenter.sh_nodepools.yaml @@ -207,7 +207,7 @@ spec: - message: label "kubernetes.io/hostname" is restricted rule: self.all(x, x != "kubernetes.io/hostname") - message: label domain "karpenter.azure.com" is restricted - rule: self.all(x, x in [ "karpenter.azure.com/sku-name", "karpenter.azure.com/sku-family", "karpenter.azure.com/sku-version", "karpenter.azure.com/sku-cpu", "karpenter.azure.com/sku-memory", "karpenter.azure.com/sku-accelerator", "karpenter.azure.com/sku-networking-accelerated", "karpenter.azure.com/sku-storage-premium-capable", "karpenter.azure.com/sku-storage-ephemeralos-maxsize", "karpenter.azure.com/sku-encryptionathost-capable", "karpenter.azure.com/sku-gpu-name", "karpenter.azure.com/sku-gpu-manufacturer", "karpenter.azure.com/sku-gpu-count" ] || !x.find("^([^/]+)").endsWith("karpenter.azure.com")) + rule: self.all(x, x in [ "karpenter.azure.com/aksnodeclass", "karpenter.azure.com/sku-name", "karpenter.azure.com/sku-family", "karpenter.azure.com/sku-version", "karpenter.azure.com/sku-cpu", "karpenter.azure.com/sku-memory", "karpenter.azure.com/sku-accelerator", "karpenter.azure.com/sku-networking-accelerated", "karpenter.azure.com/sku-storage-premium-capable", "karpenter.azure.com/sku-storage-ephemeralos-maxsize", "karpenter.azure.com/sku-encryptionathost-capable", "karpenter.azure.com/sku-gpu-name", "karpenter.azure.com/sku-gpu-manufacturer", "karpenter.azure.com/sku-gpu-count" ] || !x.find("^([^/]+)").endsWith("karpenter.azure.com")) type: object spec: description: |- @@ -280,7 +280,7 @@ spec: - message: label "kubernetes.io/hostname" is restricted rule: self != "kubernetes.io/hostname" - message: label domain "karpenter.azure.com" is restricted - rule: self in [ "karpenter.azure.com/sku-name", "karpenter.azure.com/sku-family", "karpenter.azure.com/sku-version", "karpenter.azure.com/sku-cpu", "karpenter.azure.com/sku-memory", "karpenter.azure.com/sku-accelerator", "karpenter.azure.com/sku-networking-accelerated", "karpenter.azure.com/sku-storage-premium-capable", "karpenter.azure.com/sku-storage-ephemeralos-maxsize", "karpenter.azure.com/sku-encryptionathost-capable", "karpenter.azure.com/sku-gpu-name", "karpenter.azure.com/sku-gpu-manufacturer", "karpenter.azure.com/sku-gpu-count" ] || !self.find("^([^/]+)").endsWith("karpenter.azure.com") + rule: self in [ "karpenter.azure.com/aksnodeclass", "karpenter.azure.com/sku-name", "karpenter.azure.com/sku-family", "karpenter.azure.com/sku-version", "karpenter.azure.com/sku-cpu", "karpenter.azure.com/sku-memory", "karpenter.azure.com/sku-accelerator", "karpenter.azure.com/sku-networking-accelerated", "karpenter.azure.com/sku-storage-premium-capable", "karpenter.azure.com/sku-storage-ephemeralos-maxsize", "karpenter.azure.com/sku-encryptionathost-capable", "karpenter.azure.com/sku-gpu-name", "karpenter.azure.com/sku-gpu-manufacturer", "karpenter.azure.com/sku-gpu-count" ] || !self.find("^([^/]+)").endsWith("karpenter.azure.com") minValues: description: |- This field is ALPHA and can be dropped or replaced at any time