koordlet: tc plugin for netqos

Signed-off-by: lucming <[email protected]>

.github/workflows/ci.yaml

@@ -18,9 +18,9 @@ jobs:
           cache: false
           go-version-file: go.mod
       - run: sudo apt update && sudo apt install -y libpfm4 libpfm4-dev
-      - uses: golangci/golangci-lint-action@v5
+      - uses: golangci/golangci-lint-action@v5.3.0
         with:
-          version: v1.47.3
+          version: v1.55.2
 
   unit-tests:
     strategy:

.github/workflows/e2e-k8s-1.22.yaml

@@ -10,7 +10,7 @@ on:
 
 env:
   # Common versions
-  GO_VERSION: '1.19'
+  GO_VERSION: '1.20'
   KIND_ACTION_VERSION: 'v1.5.0'
   KIND_VERSION: 'v0.20.0'
   KIND_IMAGE: 'kindest/node:v1.22.17'

.github/workflows/e2e-k8s-1.24.yaml

@@ -10,7 +10,7 @@ on:
 
 env:
   # Common versions
-  GO_VERSION: '1.19'
+  GO_VERSION: '1.20'
   KIND_ACTION_VERSION: 'v1.5.0'
   KIND_VERSION: 'v0.20.0'
   KIND_IMAGE: 'kindest/node:v1.24.15'

.github/workflows/e2e-k8s-1.28.yaml

@@ -0,0 +1,106 @@
+name: E2E-K8S-1.28
+
+on:
+  push:
+    branches:
+      - main
+      - release-*
+  pull_request: {}
+  workflow_dispatch: {}
+
+env:
+  # Common versions
+  GO_VERSION: '1.20'
+  KIND_ACTION_VERSION: 'v1.5.0'
+  KIND_VERSION: 'v0.22.0'
+  KIND_IMAGE: 'kindest/node:v1.28.7'
+  KIND_CLUSTER_NAME: 'ci-testing'
+  COMPONENT_NS: "koordinator-system"
+
+jobs:
+
+  slo-controller:
+    continue-on-error: true
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/[email protected]
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export MANAGER_IMAGE="koordinator-sh/koord-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull . -t ${MANAGER_IMAGE} -f docker/koord-manager.dockerfile
+          export KOORDLET_IMAGE="koordinator-sh/koordlet:e2e-${GITHUB_RUN_ID}"
+          docker build --pull . -t ${KOORDLET_IMAGE} -f docker/koordlet.dockerfile
+          kind load docker-image --name=${KIND_CLUSTER_NAME} ${MANAGER_IMAGE} || { echo >&2 "kind not installed or error loading image: ${MANAGER_IMAGE}"; exit 1; }
+          kind load docker-image --name=${KIND_CLUSTER_NAME} ${KOORDLET_IMAGE} || { echo >&2 "kind not installed or error loading image: ${KOORDLET_IMAGE}"; exit 1; }
+      - name: Check host environment
+        run: |
+          set -ex
+          kubectl version --short
+          kubectl get pods -A
+          kubectl get nodes -o yaml
+          tree -L 2 /sys/
+          tree -L 2 /sys/fs/cgroup
+          cat /proc/cpuinfo
+      - name: Install Koordinator
+        run: |
+          set -ex
+          kubectl cluster-info
+          MANAGER_IMG=koordinator-sh/koord-manager:e2e-${GITHUB_RUN_ID} KOORDLET_IMG=koordinator-sh/koordlet:e2e-${GITHUB_RUN_ID} ./hack/deploy_kind.sh
+          NODES=$(kubectl get node | wc -l)
+          for ((i=1;i<10;i++));
+          do
+            set +e
+            PODS=$(kubectl get pod -n ${COMPONENT_NS} | grep "koord-manager\|koordlet" | grep '1/1' | wc -l)
+            set -e
+            if [ "$PODS" -ge "$NODES" ]; then
+              break
+            fi
+            sleep 6
+          done
+          set +e
+          PODS=$(kubectl get pod -n ${COMPONENT_NS} | grep "koord-manager\|koordlet" | grep '1/1' | wc -l)
+          kubectl get pod -A
+          kubectl get node -o yaml
+          kubectl get all -n ${COMPONENT_NS} -o wide
+          kubectl get pod -n ${COMPONENT_NS} --no-headers | grep koord-manager | head -n 1 | awk '{print $1}' | xargs kubectl logs -n ${COMPONENT_NS} --tail=100
+          kubectl get pod -n ${COMPONENT_NS} --no-headers | grep koord-scheduler | awk '{print $1}' | xargs kubectl logs -n ${COMPONENT_NS} --tail=100
+          kubectl get pod -n ${COMPONENT_NS} --no-headers | grep koordlet | head -n 1 | awk '{print $1}' | xargs -L 1 kubectl logs -n ${COMPONENT_NS}
+          kubectl get pod -n ${COMPONENT_NS} -o wide
+          set -e
+          if [ "$PODS" -ge "$NODES" ]; then
+            echo "Wait for koord-manager and koordlet ready successfully"
+          else
+            echo "Timeout to wait for koord-manager and koordlet ready"
+            exit 1
+          fi
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          make ginkgo
+          set +e
+          EXTRA_ARGS="-koordinator-component-namespace=${COMPONENT_NS} -allowed-not-ready-nodes=1 -system-pods-startup-timeout=10s -e2e-verify-service-account=false"
+          ./bin/ginkgo -timeout 60m -v --focus='slo-controller' test/e2e -- ${EXTRA_ARGS}
+          retVal=$?
+          restartCount=$(kubectl get pod -n ${COMPONENT_NS} -l koord-app=koord-manager --no-headers | head -n 1 | awk '{print $4}')
+          if [ "${restartCount}" -eq "0" ];then
+              echo "koord-manager has not restarted"
+          else
+              kubectl get pod -n ${COMPONENT_NS} -l koord-app=koord-manager --no-headers
+              echo "koord-manager has restarted, abort!!!"
+              kubectl get pod -n ${COMPONENT_NS} --no-headers -l koord-app=koord-manager | head -n 1 | awk '{print $1}' | xargs kubectl logs -p -n ${COMPONENT_NS}
+              exit 1
+          fi
+          exit $retVal

.github/workflows/e2e-k8s-latest.yaml

@@ -10,7 +10,7 @@ on:
 
 env:
   # Common versions
-  GO_VERSION: '1.19'
+  GO_VERSION: '1.20'
   KIND_ACTION_VERSION: 'v1.5.0'
   KIND_VERSION: 'v0.20.0'
   KIND_CLUSTER_NAME: 'ci-testing'

.golangci.yml

@@ -58,7 +58,7 @@ linters:
   fast: false
   disable-all: true
   enable:
-    - deadcode
+#    - unused
     - gofmt
     - govet
     - goimports

Makefile

@@ -15,7 +15,7 @@ KOORD_SCHEDULER_IMG ?= "${REG}/${REG_NS}/koord-scheduler:${GIT_BRANCH}-${GIT_COM
 KOORD_DESCHEDULER_IMG ?= "${REG}/${REG_NS}/koord-descheduler:${GIT_BRANCH}-${GIT_COMMIT_ID}"
 
 # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.
-ENVTEST_K8S_VERSION = 1.22
+ENVTEST_K8S_VERSION = 1.28
 
 AGENT_MODE ?= hostMode
 # Set license header files.
@@ -65,7 +65,6 @@ help: ## Display this help.
 .PHONY: manifests
 manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
 	$(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases
-	@hack/fix_crd_plural.sh
 
 .PHONY: generate
 generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
@@ -217,8 +216,8 @@ HACK_DIR ?= $(PWD)/hack
 
 ## Tool Versions
 KUSTOMIZE_VERSION ?= v3.8.7
-CONTROLLER_TOOLS_VERSION ?= v0.9.0
-GOLANGCILINT_VERSION ?= v1.47.3
+CONTROLLER_TOOLS_VERSION ?= v0.14.0
+GOLANGCILINT_VERSION ?= v1.55.2
 GINKGO_VERSION ?= v1.16.4
 
 KUSTOMIZE_INSTALL_SCRIPT ?= "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh"

apis/extension/elastic_quota.go

@@ -22,7 +22,8 @@ import (
 
 	corev1 "k8s.io/api/core/v1"
 	v1 "k8s.io/apiserver/pkg/quota/v1"
-	"sigs.k8s.io/scheduler-plugins/pkg/apis/scheduling/v1alpha1"
+
+	"github.com/koordinator-sh/koordinator/apis/thirdparty/scheduler-plugins/pkg/apis/scheduling/v1alpha1"
 )
 
 // RootQuotaName means quotaTree's root\head.

apis/extension/numa_aware.go

@@ -26,6 +26,9 @@ import (
 
 // Defines the pod level annotations and labels
 const (
+	// AnnotationNUMATopologySpec represents numa allocation API defined by Koordinator.
+	// The user specifies the desired numa policy by setting the annotation.
+	AnnotationNUMATopologySpec = SchedulingDomainPrefix + "/numa-topology-spec"
 	// AnnotationResourceSpec represents resource allocation API defined by Koordinator.
 	// The user specifies the desired CPU orchestration policy by setting the annotation.
 	AnnotationResourceSpec = SchedulingDomainPrefix + "/resource-spec"
@@ -67,6 +70,15 @@ type ResourceSpec struct {
 	PreferredCPUExclusivePolicy CPUExclusivePolicy `json:"preferredCPUExclusivePolicy,omitempty"`
 }
 
+type NUMATopologySpec struct {
+	// NUMATopologyPolicy represents the numa topology policy when schedule pod
+	NUMATopologyPolicy NUMATopologyPolicy `json:"numaTopologyPolicy,omitempty"`
+	// SingleNUMANodeExclusive represents whether a Pod that will use a single NUMA node/multiple NUMA nodes
+	// on a NUMA node can be scheduled to use the NUMA node when another Pod that uses multiple NUMA nodes/a single NUMA node
+	// is already running on the same node.
+	SingleNUMANodeExclusive NumaTopologyExclusive `json:"singleNUMANodeExclusive,omitempty"`
+}
+
 // ResourceStatus describes resource allocation result, such as how to bind CPU.
 type ResourceStatus struct {
 	// CPUSet represents the allocated CPUs. It is Linux CPU list formatted string.
@@ -135,6 +147,21 @@ const (
 	NodeNUMAAllocateStrategyMostAllocated  = NUMAMostAllocated
 )
 
+type NumaTopologyExclusive string
+
+const (
+	NumaTopologyExclusivePreferred NumaTopologyExclusive = "Preferred"
+	NumaTopologyExclusiveRequired  NumaTopologyExclusive = "Required"
+)
+
+type NumaNodeStatus string
+
+const (
+	NumaNodeStatusIdle   NumaNodeStatus = "idle"
+	NumaNodeStatusShared NumaNodeStatus = "shared"
+	NumaNodeStatusSingle NumaNodeStatus = "single"
+)
+
 type NUMATopologyPolicy string
 
 const (
@@ -187,6 +214,19 @@ type KubeletCPUManagerPolicy struct {
 	ReservedCPUs string            `json:"reservedCPUs,omitempty"`
 }
 
+func GetNUMATopologySpec(annotations map[string]string) (*NUMATopologySpec, error) {
+	numaSpec := &NUMATopologySpec{}
+	data, ok := annotations[AnnotationNUMATopologySpec]
+	if !ok {
+		return numaSpec, nil
+	}
+	err := json.Unmarshal([]byte(data), numaSpec)
+	if err != nil {
+		return nil, err
+	}
+	return numaSpec, nil
+}
+
 // GetResourceSpec parses ResourceSpec from annotations
 func GetResourceSpec(annotations map[string]string) (*ResourceSpec, error) {
 	resourceSpec := &ResourceSpec{}