diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml new file mode 100644 index 000000000..8bab93b88 --- /dev/null +++ b/.github/workflows/benchmark.yaml @@ -0,0 +1,54 @@ +name: Benchmark + +on: + pull_request: + paths: + - pkg/**/* + - cmd/**/* + - test/**/* + - hack/**/* + - kustomize/**/* + - go.mod + - .github/workflows/benchmark.yaml + - '!hack/releases-helm-chart.sh' + push: + paths: + - pkg/**/* + - cmd/**/* + - test/**/* + - hack/**/* + - kustomize/**/* + - go.mod + - .github/workflows/benchmark.yaml + - '!hack/releases-helm-chart.sh' + +env: + CGO_ENABLED: "0" + GO_VERSION: "1.23.0" + +jobs: + benchmark: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + + - name: Test Benchmark + shell: bash + run: | + ./hack/e2e-test.sh e2e/kwokctl/benchmark + + - name: Test Benchmark Hack + shell: bash + run: | + ./hack/e2e-test.sh e2e/kwokctl/benchmark-hack + + - name: Upload logs + uses: actions/upload-artifact@v4 + if: failure() + with: + name: kwok-logs-benchmark + path: ${{ github.workspace }}/logs diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 57952ad68..eb8ac8d7a 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -270,12 +270,6 @@ jobs: fi ./hack/e2e-test.sh kwokctl/kwokctl_${{ matrix.kwokctl-runtime }} - - name: Test Benchmark - if: ${{ matrix.os == 'ubuntu-latest' && matrix.kwokctl-runtime == 'binary' }} - shell: bash - run: | - ./hack/e2e-test.sh e2e/kwokctl/benchmark - - name: Test Auto Detect if: ${{ matrix.kwokctl-runtime == 'binary' }} shell: bash diff --git a/test/e2e/benchmark.go b/test/e2e/benchmark.go index 968dab28f..6482c4650 100644 --- a/test/e2e/benchmark.go +++ b/test/e2e/benchmark.go @@ -120,10 +120,10 @@ func scaleCreateNode(ctx context.Context, t *testing.T, kwokctlPath string, name func CaseBenchmark(kwokctlPath, clusterName string) *features.FeatureBuilder { return features.New("Benchmark"). Assess("Create nodes", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { - ctx0, cancel := context.WithTimeout(ctx, 120*time.Second) + ctx0, cancel := context.WithTimeout(ctx, 180*time.Second) defer cancel() - err := scaleCreateNode(ctx0, t, kwokctlPath, clusterName, 2000) + err := scaleCreateNode(ctx0, t, kwokctlPath, clusterName, 5000) if err != nil { t.Fatal(err) } @@ -133,7 +133,7 @@ func CaseBenchmark(kwokctlPath, clusterName string) *features.FeatureBuilder { ctx0, cancel := context.WithTimeout(ctx, 240*time.Second) defer cancel() - err := scaleCreatePod(ctx0, t, kwokctlPath, clusterName, 5000) + err := scaleCreatePod(ctx0, t, kwokctlPath, clusterName, 10000) if err != nil { t.Fatal(err) } diff --git a/test/e2e/benchmark_hack.go b/test/e2e/benchmark_hack.go new file mode 100644 index 000000000..f59c54906 --- /dev/null +++ b/test/e2e/benchmark_hack.go @@ -0,0 +1,196 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "fmt" + "io" + "os/exec" + "testing" + "time" + + "sigs.k8s.io/e2e-framework/pkg/envconf" + "sigs.k8s.io/e2e-framework/pkg/features" +) + +func readerPodYaml(size int) io.Reader { + r, w := io.Pipe() + go func() { + defer w.Close() + for i := 0; i < size; i++ { + _, _ = fmt.Fprintf(w, podYaml, i, i) + } + }() + return r +} + +var podYaml = ` +apiVersion: v1 +kind: Pod +metadata: + name: pod-%d + namespace: default + uid: 00000000-0000-0000-0001-%012d +spec: + containers: + - image: busybox + name: container-0 + nodeName: node-0 +--- +` + +func scaleCreatePodWithHack(ctx context.Context, t *testing.T, kwokctlPath string, name string, size int) error { + scaleCmd := exec.CommandContext(ctx, kwokctlPath, "--name", name, "hack", "put", "--path", "-") + scaleCmd.Stdin = readerPodYaml(size) + if err := scaleCmd.Start(); err != nil { + return fmt.Errorf("failed to start scale command: %w", err) + } + + if err := waitResource(ctx, t, kwokctlPath, name, "Pod", "Running", size, 5, 10); err != nil { + return fmt.Errorf("failed to wait for resource: %w", err) + } + return nil +} + +func readerPodDeleteYaml(size int) io.Reader { + r, w := io.Pipe() + go func() { + now := time.Now().UTC().Format(time.RFC3339) + defer w.Close() + for i := 0; i < size; i++ { + _, _ = fmt.Fprintf(w, podDeleteYaml, i, i, now) + } + }() + return r +} + +var podDeleteYaml = ` +apiVersion: v1 +kind: Pod +metadata: + name: pod-%d + namespace: default + uid: 00000000-0000-0000-0001-%012d + deletionTimestamp: %s +spec: + containers: + - image: busybox + name: container-0 + nodeName: node-0 +--- +` + +func scaleDeletePodWithHack(ctx context.Context, t *testing.T, kwokctlPath string, name string, size int) error { + scaleCmd := exec.CommandContext(ctx, kwokctlPath, "--name", name, "hack", "put", "--path", "-") + scaleCmd.Stdin = readerPodDeleteYaml(size) + if err := scaleCmd.Start(); err != nil { + return fmt.Errorf("failed to start scale command: %w", err) + } + + if err := waitResource(ctx, t, kwokctlPath, name, "Pod", "Running", 0, 5, 10); err != nil { + return fmt.Errorf("failed to wait for resource: %w", err) + } + return nil +} + +func readerNodeYaml(size int) io.Reader { + r, w := io.Pipe() + go func() { + defer w.Close() + for i := 0; i < size; i++ { + _, _ = fmt.Fprintf(w, nodeYaml, i, i) + } + }() + return r +} + +var nodeYaml = ` +apiVersion: v1 +kind: Node +metadata: + annotations: + kwok.x-k8s.io/node: fake + node.alpha.kubernetes.io/ttl: "0" + labels: + beta.kubernetes.io/arch: amd64 + beta.kubernetes.io/os: linux + kubernetes.io/arch: amd64 + kubernetes.io/os: linux + kubernetes.io/role: agent + node-role.kubernetes.io/agent: "" + type: kwok + name: node-%d + uid: 00000000-0000-0000-0000-%012d +status: + allocatable: + cpu: "32" + memory: 256Gi + pods: "110" + capacity: + cpu: "32" + memory: 256Gi + pods: "110" +--- +` + +func scaleCreateNodeWithHack(ctx context.Context, t *testing.T, kwokctlPath string, name string, size int) error { + scaleCmd := exec.CommandContext(ctx, kwokctlPath, "--name", name, "hack", "put", "--path", "-") + scaleCmd.Stdin = readerNodeYaml(size) + if err := scaleCmd.Start(); err != nil { + return fmt.Errorf("failed to start scale command: %w", err) + } + + if err := waitResource(ctx, t, kwokctlPath, name, "Node", "Ready", size, 10, 10); err != nil { + return fmt.Errorf("failed to wait for resource: %w", err) + } + return nil +} + +func CaseBenchmarkWithHack(kwokctlPath, clusterName string) *features.FeatureBuilder { + return features.New("Benchmark"). + Assess("Create nodes", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + ctx0, cancel := context.WithTimeout(ctx, 180*time.Second) + defer cancel() + + err := scaleCreateNodeWithHack(ctx0, t, kwokctlPath, clusterName, 5000) + if err != nil { + t.Fatal(err) + } + return ctx + }). + Assess("Create pods", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + ctx0, cancel := context.WithTimeout(ctx, 240*time.Second) + defer cancel() + + err := scaleCreatePodWithHack(ctx0, t, kwokctlPath, clusterName, 10000) + if err != nil { + t.Fatal(err) + } + return ctx + }). + Assess("Delete pods", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + ctx0, cancel := context.WithTimeout(ctx, 240*time.Second) + defer cancel() + + err := scaleDeletePodWithHack(ctx0, t, kwokctlPath, clusterName, 10000) + if err != nil { + t.Fatal(err) + } + return ctx + }) +} diff --git a/test/e2e/kwokctl/benchmark-hack/kubectl_test.go b/test/e2e/kwokctl/benchmark-hack/kubectl_test.go new file mode 100644 index 000000000..67482ef80 --- /dev/null +++ b/test/e2e/kwokctl/benchmark-hack/kubectl_test.go @@ -0,0 +1,29 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package benchmark_hack_test + +import ( + "testing" + + "sigs.k8s.io/kwok/test/e2e" +) + +func TestBenchmarkWithHack(t *testing.T) { + f0 := e2e.CaseBenchmarkWithHack(kwokctlPath, clusterName). + Feature() + testEnv.Test(t, f0) +} diff --git a/test/e2e/kwokctl/benchmark-hack/main_test.go b/test/e2e/kwokctl/benchmark-hack/main_test.go new file mode 100644 index 000000000..e1702cd67 --- /dev/null +++ b/test/e2e/kwokctl/benchmark-hack/main_test.go @@ -0,0 +1,72 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package benchmark_hack_test is a test benchmarking environment for kwok. +package benchmark_hack_test + +import ( + "os" + "runtime" + "testing" + + "sigs.k8s.io/e2e-framework/pkg/env" + "sigs.k8s.io/e2e-framework/pkg/envconf" + "sigs.k8s.io/e2e-framework/support/kwok" + + "sigs.k8s.io/kwok/pkg/consts" + "sigs.k8s.io/kwok/pkg/utils/path" + "sigs.k8s.io/kwok/test/e2e/helper" +) + +var ( + runtimeEnv = consts.RuntimeTypeBinary + testEnv env.Environment + pwd = os.Getenv("PWD") + rootDir = path.Join(pwd, "../../../..") + logsDir = path.Join(rootDir, "logs") + clusterName = envconf.RandomName("kwok-e2e-benchmark-hack", 24) + kwokPath = path.Join(rootDir, "bin", runtime.GOOS, runtime.GOARCH, "kwok"+helper.BinSuffix) + kwokctlPath = path.Join(rootDir, "bin", runtime.GOOS, runtime.GOARCH, "kwokctl"+helper.BinSuffix) + baseArgs = []string{ + "--kwok-controller-binary=" + kwokPath, + "--runtime=" + runtimeEnv, + "--wait=15m", + "--disable-kube-scheduler", + "--disable-qps-limits", + } +) + +func init() { + _ = os.Setenv("KWOK_WORKDIR", path.Join(rootDir, "workdir")) +} + +func TestMain(m *testing.M) { + testEnv = helper.Environment() + + k := kwok.NewProvider(). + WithName(clusterName). + WithPath(kwokctlPath) + testEnv.Setup( + helper.BuildKwokBinary(rootDir), + helper.BuildKwokctlBinary(rootDir), + helper.CreateCluster(k, baseArgs...), + ) + testEnv.Finish( + helper.ExportLogs(k, logsDir), + helper.DestroyCluster(k), + ) + os.Exit(testEnv.Run(m)) +} diff --git a/test/e2e/kwokctl/benchmark/main_test.go b/test/e2e/kwokctl/benchmark/main_test.go index 8dfab855e..481814684 100644 --- a/test/e2e/kwokctl/benchmark/main_test.go +++ b/test/e2e/kwokctl/benchmark/main_test.go @@ -45,6 +45,7 @@ var ( "--runtime=" + runtimeEnv, "--wait=15m", "--disable-kube-scheduler", + "--disable-qps-limits", } )