Skip to content

Commit 8dbb097

Browse files
HirazawaUiHirazawaUi
committedNov 28, 2024·
add e2e test for kubelet instance config
1 parent bac7146 commit 8dbb097

File tree

6 files changed

+506
-0
lines changed

6 files changed

+506
-0
lines changed
 

‎kinder/ci/tools/update-workflows/config.yaml

+12
Original file line numberDiff line numberDiff line change
@@ -250,3 +250,15 @@ jobGroups:
250250
- ./templates/workflows/control-plane-local-kubelet-mode-tasks.yaml
251251
jobs:
252252
- kubernetesVersion: latest
253+
254+
- name: instance-config
255+
testInfraJobSpec:
256+
targetFile: kubeadm-kinder-instance-config.yaml
257+
template: ./templates/testinfra/kubeadm-kinder-instance-config.yaml
258+
kinderWorkflowSpec:
259+
targetFile: instance-config-{{ .KubernetesVersion }}.yaml
260+
template: ./templates/workflows/instance-config.yaml
261+
additionalFiles:
262+
- ./templates/workflows/instance-config-tasks.yaml
263+
jobs:
264+
- kubernetesVersion: latest

‎kinder/ci/tools/update-workflows/templates/testinfra/kubeadm-kinder-instance-config.yaml

+42
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
- name: ci-kubernetes-e2e-kubeadm-kinder-instance-config-{{ dashVer .KubernetesVersion }}
2+
cluster: k8s-infra-prow-build
3+
interval: {{ .JobInterval }}
4+
decorate: true
5+
labels:
6+
preset-dind-enabled: "true"
7+
preset-kind-volume-mounts: "true"
8+
annotations:
9+
testgrid-dashboards: sig-cluster-lifecycle-kubeadm
10+
testgrid-tab-name: kubeadm-kinder-instance-config-{{ dashVer .KubernetesVersion }}
11+
testgrid-alert-email: sig-cluster-lifecycle-kubeadm-alerts@kubernetes.io
12+
description: "OWNER: sig-cluster-lifecycle (kinder); Uses kubeadm/kinder to create and upgrade a cluster using kubelet instance config"
13+
testgrid-num-columns-recent: "20"
14+
{{ .AlertAnnotations }}
15+
decoration_config:
16+
timeout: 60m
17+
extra_refs:
18+
- org: kubernetes
19+
repo: kubernetes
20+
base_ref: {{ branchFor .KubernetesVersion }}
21+
path_alias: k8s.io/kubernetes
22+
- org: kubernetes
23+
repo: kubeadm
24+
base_ref: main
25+
path_alias: k8s.io/kubeadm
26+
spec:
27+
containers:
28+
- image: gcr.io/k8s-staging-test-infra/kubekins-e2e:{{ .TestInfraImage }}-{{ imageVer .KubernetesVersion }}
29+
command:
30+
- runner.sh
31+
- "../kubeadm/kinder/ci/kinder-run.sh"
32+
args:
33+
- {{ .WorkflowFile }}
34+
securityContext:
35+
privileged: true
36+
resources:
37+
limits:
38+
memory: "9000Mi"
39+
cpu: 2000m
40+
requests:
41+
memory: "9000Mi"
42+
cpu: 2000m

‎kinder/ci/tools/update-workflows/templates/workflows/instance-config-tasks.yaml

+214
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,214 @@
1+
# IMPORTANT! this workflow is imported by instance-config-* workflows.
2+
version: 1
3+
summary: |
4+
This workflow implements a sequence of tasks to test that kubeadm
5+
works properly in scenarios using kubelet instance configuration.
6+
vars:
7+
# vars defines default values for variable used by tasks in this workflow;
8+
# those values might be overridden when importing this files.
9+
kubernetesVersion: latest
10+
upgradeVersion: latest
11+
controlPlaneNodes: 3
12+
workerNodes: 2
13+
baseImage: kindest/base:v20221102-76f15095 # has containerd
14+
image: kindest/node:test
15+
clusterName: kinder-instance-config
16+
kubeadmVerbosity: 6
17+
tasks:
18+
- name: pull-base-image
19+
description: |
20+
pulls kindest/base image with docker in docker and all the prerequisites necessary for running kind(er)
21+
cmd: docker
22+
args:
23+
- pull
24+
- "{{ .vars.baseImage }}"
25+
- name: add-kubernetes-versions
26+
description: |
27+
creates a node-image-variant by adding a Kubernetes version
28+
cmd: kinder
29+
args:
30+
- build
31+
- node-image-variant
32+
- --base-image={{ .vars.baseImage }}
33+
- --image={{ .vars.image }}
34+
- --with-init-artifacts={{ .vars.kubernetesVersion }}
35+
- --with-upgrade-artifacts={{ .vars.upgradeVersion }}
36+
- --loglevel=debug
37+
timeout: 15m
38+
- name: create-cluster
39+
description: |
40+
create a set of nodes ready for hosting the Kubernetes cluster
41+
cmd: kinder
42+
args:
43+
- create
44+
- cluster
45+
- --name={{ .vars.clusterName }}
46+
- --image={{ .vars.image }}
47+
- --control-plane-nodes={{ .vars.controlPlaneNodes }}
48+
- --worker-nodes={{ .vars.workerNodes }}
49+
- --kubeadm-feature-gate="NodeLocalCRISocket=true"
50+
- --loglevel=debug
51+
timeout: 5m
52+
- name: init
53+
description: |
54+
Initializes the Kubernetes cluster with version "initVersion"
55+
by starting the boostrap control-plane nodes
56+
cmd: kinder
57+
args:
58+
- do
59+
- kubeadm-init
60+
- --name={{ .vars.clusterName }}
61+
- --loglevel=debug
62+
- --kubeadm-verbosity={{ .vars.kubeadmVerbosity }}
63+
- --kubeadm-feature-gate="NodeLocalCRISocket=true"
64+
- --copy-certs=auto
65+
timeout: 5m
66+
- name: join
67+
description: |
68+
Join the other nodes to the Kubernetes cluster
69+
cmd: kinder
70+
args:
71+
- do
72+
- kubeadm-join
73+
- --name={{ .vars.clusterName }}
74+
- --loglevel=debug
75+
- --kubeadm-verbosity={{ .vars.kubeadmVerbosity }}
76+
- --kubeadm-feature-gate="NodeLocalCRISocket=true"
77+
- --copy-certs=auto
78+
timeout: 10m
79+
- name: check-config
80+
description: |
81+
Check if the kubelet instance configuration exists on each node.
82+
cmd: /bin/bash
83+
args:
84+
- -c
85+
- |
86+
set -x
87+
88+
for node in control-plane-{1..3} worker-{1..2}; do
89+
CMD="docker exec {{ .vars.clusterName }}-$node"
90+
91+
${CMD} bash -c '
92+
if [[ -f "/var/lib/kubelet/instance-config.yaml" ]] && grep -q "containerRuntimeEndpoint:" /var/lib/kubelet/instance-config.yaml; then
93+
echo "File exists and contains 'containerRuntimeEndpoint:' field."
94+
else
95+
echo "File does not exist or does not contain 'containerRuntimeEndpoint:' field."
96+
exit 1
97+
fi
98+
'
99+
done
100+
101+
exit 0
102+
timeout: 5m
103+
- name: remove-config
104+
description: |
105+
Remove the kubelet instance configuration from each node.
106+
cmd: /bin/bash
107+
args:
108+
- -c
109+
- |
110+
set -x
111+
112+
for node in control-plane-{1..3} worker-{1..2}; do
113+
CMD="docker exec {{ .vars.clusterName }}-$node"
114+
115+
${CMD} bash -c 'rm -f /var/lib/kubelet/instance-config.yaml'
116+
done
117+
118+
exit 0
119+
timeout: 5m
120+
- name: upgrade
121+
description: |
122+
upgrades the cluster to Kubernetes "upgradeVersion"
123+
cmd: kinder
124+
args:
125+
- do
126+
- kubeadm-upgrade
127+
- --upgrade-version={{ .vars.upgradeVersion }}
128+
- --name={{ .vars.clusterName }}
129+
- --loglevel=debug
130+
- --kubeadm-verbosity={{ .vars.kubeadmVerbosity }}
131+
- --kubeadm-feature-gate="NodeLocalCRISocket=true"
132+
timeout: 15m
133+
- name: check-config
134+
description: |
135+
Check if the kubelet instance configuration exists on each node.
136+
cmd: /bin/bash
137+
args:
138+
- -c
139+
- |
140+
set -x
141+
142+
for node in control-plane-{1..3} worker-{1..2}; do
143+
CMD="docker exec {{ .vars.clusterName }}-$node"
144+
145+
${CMD} bash -c '
146+
if [[ -f "/var/lib/kubelet/instance-config.yaml" ]] && grep -q "containerRuntimeEndpoint:" /var/lib/kubelet/instance-config.yaml; then
147+
echo "File exists and contains 'containerRuntimeEndpoint:' field."
148+
else
149+
echo "File does not exist or does not contain 'containerRuntimeEndpoint:' field."
150+
exit 1
151+
fi
152+
'
153+
done
154+
155+
exit 0
156+
timeout: 5m
157+
- name: cluster-info
158+
description: |
159+
Runs cluster-info
160+
cmd: kinder
161+
args:
162+
- do
163+
- cluster-info
164+
- --name={{ .vars.clusterName }}
165+
- --loglevel=debug
166+
- name: e2e-kubeadm
167+
description: |
168+
Runs kubeadm e2e tests
169+
cmd: kinder
170+
args:
171+
- test
172+
- e2e-kubeadm
173+
- --test-flags=--report-dir={{ .env.ARTIFACTS }} --report-prefix=e2e-kubeadm
174+
- --name={{ .vars.clusterName }}
175+
- --loglevel=debug
176+
timeout: 10m
177+
- name: get-logs
178+
description: |
179+
Collects all the test logs
180+
cmd: kinder
181+
args:
182+
- export
183+
- logs
184+
- --loglevel=debug
185+
- --name={{ .vars.clusterName }}
186+
- "{{ .env.ARTIFACTS }}"
187+
force: true
188+
timeout: 5m
189+
# kind export log is know to be flaky, so we are temporary ignoring errors in order
190+
# to make the test pass in case everything else passed
191+
# see https://github.com/kubernetes-sigs/kind/issues/456
192+
ignoreError: true
193+
- name: reset
194+
description: |
195+
Exec kubeadm reset
196+
cmd: kinder
197+
args:
198+
- do
199+
- kubeadm-reset
200+
- --name={{ .vars.clusterName }}
201+
- --loglevel=debug
202+
- --kubeadm-verbosity={{ .vars.kubeadmVerbosity }}
203+
- --kubeadm-feature-gate="NodeLocalCRISocket=true"
204+
force: true
205+
- name: delete
206+
description: |
207+
Deletes the cluster
208+
cmd: kinder
209+
args:
210+
- delete
211+
- cluster
212+
- --name={{ .vars.clusterName }}
213+
- --loglevel=debug
214+
force: true

‎kinder/ci/tools/update-workflows/templates/workflows/instance-config.yaml

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
version: 1
2+
summary: |
3+
This workflow tests the proper functioning of the {{ .KubernetesVersion }} version of both kubeadm and Kubernetes using
4+
kubelet instance configuration.
5+
test grid > https://testgrid.k8s.io/sig-cluster-lifecycle-kubeadm#kubeadm-kinder-instance-config{{ dashVer .KubernetesVersion }}
6+
config > https://git.k8s.io/test-infra/config/jobs/kubernetes/sig-cluster-lifecycle/{{ .TargetFile }}
7+
vars:
8+
kubernetesVersion: "\{\{ resolve `ci/{{ ciLabelFor .KubernetesVersion }}` \}\}"
9+
upgradeVersion: "\{\{ resolve `ci/{{ ciLabelFor .KubernetesVersion }}` \}\}"
10+
tasks:
11+
- import: instance-config-tasks.yaml

‎kinder/ci/workflows/instance-config-latest.yaml

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
# AUTOGENERATED by https://git.k8s.io/kubeadm/kinder/ci/tools/update-workflows
2+
version: 1
3+
summary: |
4+
This workflow tests the proper functioning of the latest version of both kubeadm and Kubernetes using
5+
kubelet instance configuration.
6+
test grid > https://testgrid.k8s.io/sig-cluster-lifecycle-kubeadm#kubeadm-kinder-instance-configlatest
7+
config > https://git.k8s.io/test-infra/config/jobs/kubernetes/sig-cluster-lifecycle/kubeadm-kinder-instance-config.yaml
8+
vars:
9+
kubernetesVersion: "{{ resolve `ci/latest` }}"
10+
upgradeVersion: "{{ resolve `ci/latest` }}"
11+
tasks:
12+
- import: instance-config-tasks.yaml

0 commit comments

Comments
 (0)