forked from kumahq/kuma
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathk3d.mk
174 lines (156 loc) · 7.75 KB
/
k3d.mk
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
CI_K3S_VERSION ?= $(K8S_MIN_VERSION)
METALLB_VERSION ?= v0.13.9
KUMA_MODE ?= standalone
KUMA_NAMESPACE ?= kuma-system
# Comment about PORT_PREFIX generation
#
# First step: $(KIND_CLUSTER_NAME:kuma%=300%) will replace a string "kuma" from
# the $(KIND_CLUSTER_NAME) variable with the string "300" (default/initial
# prefix):
#
# Initial value Step#1
# KIND_CLUSTER_NAME(kuma) -> 300
# KIND_CLUSTER_NAME(kuma-1) -> 300-1
# KIND_CLUSTER_NAME(kuma-2) -> 300-2
# KIND_CLUSTER_NAME(kuma-3) -> 300-3
# [...etc]
#
# The next step - $(patsubst 300-%,300+%-1,...) will replace string
# "300-[1,2,3...]" with string "300+[1,2,3...]-1" ("-1" is necessary to preserve
# the current overflow, so when the KIND_CLUSTER_NAME is equal "kuma", OR
# "kuma-1" when value of the port will be equal "300"):
#
# Initial value Step#1 Step#2
# KIND_CLUSTER_NAME(kuma) -> 300 -> 300
# KIND_CLUSTER_NAME(kuma-1) -> 300-1 -> 300+1-1
# KIND_CLUSTER_NAME(kuma-2) -> 300-2 -> 300+2-1
# KIND_CLUSTER_NAME(kuma-3) -> 300-3 -> 300+3-1
# [...etc]
#
# The last step $$((...)) will call the shell to use the expression we generated
# earlier and calculate it's arithmetic value:
#
# Initial value Step#1 Step#2 Step#3 Result
# KIND_CLUSTER_NAME(kuma) -> 300 -> 300 -> 300 -> PORT_PREFIX(300)
# KIND_CLUSTER_NAME(kuma-1) -> 300-1 -> 300+1-1 -> 300 -> PORT_PREFIX(300)
# KIND_CLUSTER_NAME(kuma-2) -> 300-2 -> 300+2-1 -> 301 -> PORT_PREFIX(301)
# KIND_CLUSTER_NAME(kuma-3) -> 300-3 -> 300+3-1 -> 302 -> PORT_PREFIX(302)
# [...etc]
PORT_PREFIX := $$(($(patsubst 300-%,300+%-1,$(KIND_CLUSTER_NAME:kuma%=300%))))
K3D_NETWORK_CNI ?= flannel
K3D_CLUSTER_CREATE_OPTS ?= -i rancher/k3s:$(CI_K3S_VERSION) \
--k3s-arg '--disable=traefik@server:0' \
--k3s-arg '--disable=metrics-server@server:0' \
--k3s-arg '--disable=servicelb@server:0' \
--volume '$(subst @,\@,$(TOP))/test/framework/deployments:/tmp/deployments@server:0' \
--network kind \
--port "$(PORT_PREFIX)80-$(PORT_PREFIX)99:30080-30099@server:0" \
--timeout 120s
ifeq ($(K3D_NETWORK_CNI),calico)
K3D_CLUSTER_CREATE_OPTS += --volume "$(TOP)/test/k3d/calico.yaml.kubelint-excluded:/var/lib/rancher/k3s/server/manifests/calico.yaml" \
--k3s-arg '--flannel-backend=none@server:*'
endif
ifdef CI
ifeq ($(GOOS),linux)
ifneq (,$(findstring legacy,$(CIRCLE_JOB)))
K3D_CLUSTER_CREATE_OPTS += --volume "/sys/fs/bpf:/sys/fs/bpf:shared"
endif
endif
endif
ifeq ($(GOOS),linux)
ifndef CI
K3D_CLUSTER_CREATE_OPTS += --volume "/sys/fs/bpf:/sys/fs/bpf:shared"
endif
endif
.PHONY: k3d/network/create
k3d/network/create:
@touch $(BUILD_DIR)/k3d_network.lock && \
if [ `which flock` ]; then flock -x $(BUILD_DIR)/k3d_network.lock -c 'docker network create -d=bridge -o com.docker.network.bridge.enable_ip_masquerade=true --ipv6 --subnet "fd00:fd12:3456::0/64" kind || true'; \
else docker network create -d=bridge -o com.docker.network.bridge.enable_ip_masquerade=true --ipv6 --subnet "fd00:fd12:3456::0/64" kind || true; fi && \
rm -f $(BUILD_DIR)/k3d_network.lock
.PHONY: k3d/start
k3d/start: ${KIND_KUBECONFIG_DIR} k3d/network/create
@echo "PORT_PREFIX=$(PORT_PREFIX)"
@KUBECONFIG=$(KIND_KUBECONFIG) \
$(K3D_BIN) cluster create "$(KIND_CLUSTER_NAME)" $(K3D_CLUSTER_CREATE_OPTS)
$(MAKE) k3d/wait
@echo
@echo '>>> You need to manually run the following command in your shell: >>>'
@echo
@echo export KUBECONFIG="$(KIND_KUBECONFIG)"
@echo
@echo '<<< ------------------------------------------------------------- <<<'
@echo
$(MAKE) k3d/configure/ebpf
$(MAKE) k3d/configure/metallb
.PHONY: k3d/configure/ebpf
k3d/configure/ebpf:
ifeq ($(GOOS),darwin)
docker exec k3d-$(KIND_CLUSTER_NAME)-server-0 mount bpffs /sys/fs/bpf -t bpf && \
docker exec k3d-$(KIND_CLUSTER_NAME)-server-0 mount --make-shared /sys/fs/bpf
endif
.PHONY: k3d/configure/metallb
k3d/configure/metallb:
@KUBECONFIG=$(KIND_KUBECONFIG) $(KUBECTL) apply -f https://raw.githubusercontent.com/metallb/metallb/$(METALLB_VERSION)/config/manifests/metallb-native.yaml
@KUBECONFIG=$(KIND_KUBECONFIG) $(KUBECTL) wait --timeout=90s --for=condition=Ready -n metallb-system --all pods
@# Construct a valid address space from the docker network and the template IPAddressPool
@IFS=. read -ra NETWORK_ADDR_SPACE <<< "$$(docker network inspect kind --format '{{ (index .IPAM.Config 0).Subnet }}')"; \
IFS=/ read -r _byte prefix <<< "$${NETWORK_ADDR_SPACE[3]}"; \
if [[ "$${prefix}" -gt 16 ]]; then echo "Unexpected docker network, expecting a prefix of at most 16 bits"; exit 1; fi; \
IFS=. read -ra BASE_ADDR_SPACE <<< "$$(yq 'select(.kind == "IPAddressPool") | .spec.addresses[0]' $(KUMA_DIR)/mk/metallb-k3d-$(KIND_CLUSTER_NAME).yaml)"; \
ADDR_SPACE="$${NETWORK_ADDR_SPACE[0]}.$${NETWORK_ADDR_SPACE[1]}.$${BASE_ADDR_SPACE[2]}.$${BASE_ADDR_SPACE[3]}" \
yq '(select(.kind == "IPAddressPool") | .spec.addresses[0]) = env(ADDR_SPACE)' $(KUMA_DIR)/mk/metallb-k3d-$(KIND_CLUSTER_NAME).yaml | \
KUBECONFIG=$(KIND_KUBECONFIG) $(KUBECTL) apply -f -
.PHONY: k3d/wait
k3d/wait:
until \
KUBECONFIG=$(KIND_KUBECONFIG) $(KUBECTL) wait -n kube-system --timeout=5s --for condition=Ready --all pods; \
do echo "Waiting for the cluster to come up" && sleep 1; done
.PHONY: k3d/stop
k3d/stop:
@KUBECONFIG=$(KIND_KUBECONFIG) $(K3D_BIN) cluster delete "$(KIND_CLUSTER_NAME)"
.PHONY: k3d/stop/all
k3d/stop/all:
@KUBECONFIG=$(KIND_KUBECONFIG) $(K3D_BIN) cluster delete --all
.PHONY: k3d/load/images
k3d/load/images:
# https://github.com/k3d-io/k3d/issues/900 can cause failures that simple retry will fix
for i in 1 2 3 4 5; do $(K3D_BIN) image import --mode=direct $(KUMA_IMAGES) --cluster=$(KIND_CLUSTER_NAME) --verbose && s=0 && break || s=$$? && echo "Image import failed. Retrying..."; done; (exit $$s)
.PHONY: k3d/load
k3d/load:
$(MAKE) images
$(MAKE) docker/tag
$(MAKE) k3d/load/images
.PHONY: k3d/deploy/kuma
k3d/deploy/kuma: build/kumactl k3d/load
@KUBECONFIG=$(KIND_KUBECONFIG) $(BUILD_ARTIFACTS_DIR)/kumactl/kumactl install --mode $(KUMA_MODE) control-plane $(KUMACTL_INSTALL_CONTROL_PLANE_IMAGES) | KUBECONFIG=$(KIND_KUBECONFIG) $(KUBECTL) apply -f -
@KUBECONFIG=$(KIND_KUBECONFIG) $(KUBECTL) wait --timeout=60s --for=condition=Available -n $(KUMA_NAMESPACE) deployment/kuma-control-plane
@KUBECONFIG=$(KIND_KUBECONFIG) $(KUBECTL) wait --timeout=60s --for=condition=Ready -n $(KUMA_NAMESPACE) pods -l app=kuma-control-plane
until \
KUBECONFIG=$(KIND_KUBECONFIG) $(KUBECTL) get mesh default ; \
do echo "Waiting for default mesh to be present" && sleep 1; done
.PHONY: k3d/deploy/helm
k3d/deploy/helm: k3d/load
KUBECONFIG=$(KIND_KUBECONFIG) $(KUBECTL) delete namespace $(KUMA_NAMESPACE) --wait | true
KUBECONFIG=$(KIND_KUBECONFIG) $(KUBECTL) create namespace $(KUMA_NAMESPACE)
KUBECONFIG=$(KIND_KUBECONFIG) helm upgrade --install --namespace $(KUMA_NAMESPACE) \
--set global.image.registry="$(DOCKER_REGISTRY)" \
--set global.image.tag="$(BUILD_INFO_VERSION)" \
--set cni.enabled=true \
--set cni.chained=true \
--set cni.netDir=/var/lib/rancher/k3s/agent/etc/cni/net.d/ \
--set cni.binDir=/bin/ \
--set cni.confName=10-flannel.conflist \
kuma ./deployments/charts/kuma
@KUBECONFIG=$(KIND_KUBECONFIG) $(KUBECTL) wait --timeout=60s --for=condition=Available -n $(KUMA_NAMESPACE) deployment/kuma-control-plane
@KUBECONFIG=$(KIND_KUBECONFIG) $(KUBECTL) wait --timeout=60s --for=condition=Ready -n $(KUMA_NAMESPACE) pods -l app=kuma-control-plane
.PHONY: k3d/deploy/demo
k3d/deploy/demo: build/kumactl
@$(BUILD_ARTIFACTS_DIR)/kumactl/kumactl install demo | KUBECONFIG=$(KIND_KUBECONFIG) $(KUBECTL) apply -f -
@KUBECONFIG=$(KIND_KUBECONFIG) $(KUBECTL) wait --timeout=60s --for=condition=Ready -n kuma-demo --all pods
.PHONY: k3d/restart
k3d/restart:
$(MAKE) k3d/stop
$(MAKE) k3d/start
$(MAKE) k3d/deploy/kuma
$(MAKE) k3d/deploy/demo