forked from metal3-io/baremetal-operator
-
Notifications
You must be signed in to change notification settings - Fork 1
/
Copy pathci-e2e.sh
executable file
·247 lines (206 loc) · 9.17 KB
/
ci-e2e.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
#!/usr/bin/env bash
# -----------------------------------------------------------------------------
# Description: This script sets up the environment and runs E2E tests for the
# BMO project. It uses either vbmc or sushy-tools based on
# the BMC_PROTOCOL environment variable.
# Supported protocols are: ipmi, redfish and redfish-virtualmedia.
# VBMC is used for ipmi and sushy-tools for both redfish protocols.
# By default, redfish-virtualmedia will be used.
# Usage: export BMC_PROTOCOL="redfish" # Or "ipmi" or "redfish-virtualmedia"
# ./ci-e2e.sh
# -----------------------------------------------------------------------------
set -eux
REPO_ROOT=$(realpath "$(dirname "${BASH_SOURCE[0]}")/..")
cd "${REPO_ROOT}" || exit 1
BMC_PROTOCOL="${BMC_PROTOCOL:-"redfish-virtualmedia"}"
if [[ "${BMC_PROTOCOL}" == "redfish" ]] || [[ "${BMC_PROTOCOL}" == "redfish-virtualmedia" ]]; then
BMO_E2E_EMULATOR="sushy-tools"
elif [[ "${BMC_PROTOCOL}" == "ipmi" ]]; then
BMO_E2E_EMULATOR="vbmc"
else
echo "FATAL: Invalid BMC protocol specified: ${BMC_PROTOCOL}"
exit 1
fi
echo "BMC_PROTOCOL=${BMC_PROTOCOL}"
echo "BMO_E2E_EMULATOR=${BMO_E2E_EMULATOR}"
export E2E_CONF_FILE="${REPO_ROOT}/test/e2e/config/ironic.yaml"
export E2E_BMCS_CONF_FILE="${REPO_ROOT}/test/e2e/config/bmcs-${BMC_PROTOCOL}.yaml"
LOAD_LOCAL_IRONIC="${LOAD_LOCAL_IRONIC:-false}"
case "${GINKGO_FOCUS:-}" in
*upgrade*)
export DEPLOY_IRONIC="false"
export DEPLOY_BMO="false"
export DEPLOY_CERT_MANAGER="false"
export GINKGO_NODES=1
;;
*)
export GINKGO_SKIP="${GINKGO_SKIP:-upgrade}"
;;
esac
export USE_EXISTING_CLUSTER="true"
# Ensure requirements are installed
"${REPO_ROOT}/hack/e2e/ensure_go.sh"
export PATH="/usr/local/go/bin:${PATH}"
"${REPO_ROOT}/hack/e2e/ensure_minikube.sh"
"${REPO_ROOT}/hack/e2e/ensure_htpasswd.sh"
# CAPI test framework uses kubectl in the background
"${REPO_ROOT}/hack/e2e/ensure_kubectl.sh"
"${REPO_ROOT}/hack/e2e/ensure_yq.sh"
# Build the container image with e2e tag (used in tests)
IMG=quay.io/metal3-io/baremetal-operator:e2e make docker
# Set up minikube
minikube start --driver=kvm2
virsh -c qemu:///system net-define "${REPO_ROOT}/hack/e2e/net.xml"
virsh -c qemu:///system net-start baremetal-e2e
# Attach baremetal-e2e interface to minikube with specific mac.
# This will give minikube a known reserved IP address that we can use for Ironic
virsh -c qemu:///system attach-interface --domain minikube --mac="52:54:00:6c:3c:01" \
--model virtio --source baremetal-e2e --type network --config
# Restart minikube to apply the changes
minikube stop
## Following loop is to avoid minikube restart issue
## https://github.com/kubernetes/minikube/issues/14456
while ! minikube start; do sleep 30; done
# Load the BMO e2e image into it
# minikube image load quay.io/metal3-io/baremetal-operator:e2e
# Temporary workaround for https://github.com/kubernetes/minikube/issues/18021
docker image save -o /tmp/bmo-e2e.tar quay.io/metal3-io/baremetal-operator:e2e
minikube image load /tmp/bmo-e2e.tar
rm /tmp/bmo-e2e.tar
if [[ "${LOAD_LOCAL_IRONIC}" == "true" ]]; then
echo "Saving local ironic image!"
docker image save -o \
"/tmp/ironic-e2e.tar" "quay.io/metal3-io/ironic:local"
minikube image load /tmp/ironic-e2e.tar
rm /tmp/ironic-e2e.tar
fi
# This IP is defined by the network we created above.
IP_ADDRESS="192.168.222.1"
# This IP is also defined by the network above, and is used consistently in all of
# our e2e overlays
export IRONIC_PROVISIONING_IP="192.168.222.199"
# Build vbmctl
make build-vbmctl
# Create VMs to act as BMHs in the tests.
./bin/vbmctl --yaml-source-file "${E2E_BMCS_CONF_FILE}"
if [[ "${BMO_E2E_EMULATOR}" == "vbmc" ]]; then
# Start VBMC
docker run --name vbmc --network host -d \
-v /var/run/libvirt/libvirt-sock:/var/run/libvirt/libvirt-sock \
-v /var/run/libvirt/libvirt-sock-ro:/var/run/libvirt/libvirt-sock-ro \
quay.io/metal3-io/vbmc
readarray -t BMCS < <(yq e -o=j -I=0 '.[]' "${E2E_BMCS_CONF_FILE}")
for bmc in "${BMCS[@]}"; do
address=$(echo "${bmc}" | jq -r '.address')
hostName=$(echo "${bmc}" | jq -r '.hostName')
vbmc_port="${address##*:}"
"${REPO_ROOT}/tools/bmh_test/vm2vbmc.sh" "${hostName}" "${vbmc_port}"
done
elif [[ "${BMO_E2E_EMULATOR}" == "sushy-tools" ]]; then
# Sushy-tools variables
SUSHY_EMULATOR_FILE="${REPO_ROOT}"/test/e2e/sushy-tools/sushy-emulator.conf
# Start sushy-tools
docker run --name sushy-tools -d --network host \
-v "${SUSHY_EMULATOR_FILE}":/etc/sushy/sushy-emulator.conf:Z \
-v /var/run/libvirt:/var/run/libvirt:Z \
-e SUSHY_EMULATOR_CONFIG=/etc/sushy/sushy-emulator.conf \
quay.io/metal3-io/sushy-tools:latest sushy-emulator
else
echo "FATAL: Invalid e2e emulator specified: ${BMO_E2E_EMULATOR}"
exit 1
fi
# Image server variables
CIRROS_VERSION="0.6.2"
IMAGE_FILE="cirros-${CIRROS_VERSION}-x86_64-disk.img"
export IMAGE_CHECKSUM="c8fc807773e5354afe61636071771906"
export IMAGE_URL="http://${IP_ADDRESS}/${IMAGE_FILE}"
IMAGE_DIR="${REPO_ROOT}/test/e2e/images"
mkdir -p "${IMAGE_DIR}"
## Download disk images
wget --quiet -P "${IMAGE_DIR}/" https://artifactory.nordix.org/artifactory/metal3/images/iso/"${IMAGE_FILE}"
wget --quiet -P "${IMAGE_DIR}/" https://fastly-cdn.system-rescue.org/releases/11.00/systemrescue-11.00-amd64.iso
wget --quiet -P "${IMAGE_DIR}/" https://artifactory.nordix.org/artifactory/metal3/images/iso/minimal_linux_live-v2.iso
## Start the image server
docker run --name image-server-e2e -d \
-p 80:8080 \
-v "${IMAGE_DIR}:/usr/share/nginx/html" nginxinc/nginx-unprivileged
# Generate ssh key pair for verifying provisioned BMHs
ssh-keygen -t ed25519 -f "${IMAGE_DIR}/ssh_testkey" -q -N ""
# Build an ISO image with baked ssh key
# See https://www.system-rescue.org/scripts/sysrescue-customize/
# We use the systemrescue ISO and their script for customizing it.
pushd "${IMAGE_DIR}"
wget -O sysrescue-customize https://gitlab.com/systemrescue/systemrescue-sources/-/raw/main/airootfs/usr/share/sysrescue/bin/sysrescue-customize?inline=false
chmod +x sysrescue-customize
pub_ssh_key=$(cut -d " " -f "1,2" "ssh_testkey.pub")
mkdir -p recipe/iso_add/sysrescue.d
# Reference: https://www.system-rescue.org/manual/Configuring_SystemRescue/
cat << EOF > recipe/iso_add/sysrescue.d/90-config.yaml
---
global:
nofirewall: true
sysconfig:
authorized_keys:
"[email protected]": "${pub_ssh_key}"
EOF
./sysrescue-customize --auto --recipe-dir recipe --source systemrescue-11.00-amd64.iso --dest=sysrescue-out.iso
export ISO_IMAGE_URL="http://${IP_ADDRESS}/sysrescue-out.iso"
popd
# Generate credentials
BMO_OVERLAYS=(
"${REPO_ROOT}/config/overlays/e2e"
"${REPO_ROOT}/config/overlays/e2e-release-0.6"
"${REPO_ROOT}/config/overlays/e2e-release-0.8"
"${REPO_ROOT}/config/overlays/e2e-release-0.9"
)
IRONIC_OVERLAYS=(
"${REPO_ROOT}/ironic-deployment/overlays/e2e"
"${REPO_ROOT}/ironic-deployment/overlays/e2e-with-inspector"
"${REPO_ROOT}/ironic-deployment/overlays/e2e-release-24.0-with-inspector"
"${REPO_ROOT}/ironic-deployment/overlays/e2e-release-24.1"
"${REPO_ROOT}/ironic-deployment/overlays/e2e-release-25.0"
"${REPO_ROOT}/ironic-deployment/overlays/e2e-release-26.0"
)
IRONIC_USERNAME="$(uuidgen)"
IRONIC_PASSWORD="$(uuidgen)"
IRONIC_INSPECTOR_USERNAME="$(uuidgen)"
IRONIC_INSPECTOR_PASSWORD="$(uuidgen)"
# These must be exported so that envsubst can pick them up below
export IRONIC_USERNAME
export IRONIC_PASSWORD
export IRONIC_INSPECTOR_USERNAME
export IRONIC_INSPECTOR_PASSWORD
for overlay in "${BMO_OVERLAYS[@]}"; do
echo "${IRONIC_USERNAME}" > "${overlay}/ironic-username"
echo "${IRONIC_PASSWORD}" > "${overlay}/ironic-password"
if [[ "${overlay}" =~ release-0\.[1-5]$ ]]; then
echo "${IRONIC_INSPECTOR_USERNAME}" > "${overlay}/ironic-inspector-username"
echo "${IRONIC_INSPECTOR_PASSWORD}" > "${overlay}/ironic-inspector-password"
fi
done
for overlay in "${IRONIC_OVERLAYS[@]}"; do
echo "IRONIC_HTPASSWD=$(htpasswd -n -b -B "${IRONIC_USERNAME}" "${IRONIC_PASSWORD}")" > \
"${overlay}/ironic-htpasswd"
envsubst < "${REPO_ROOT}/ironic-deployment/components/basic-auth/ironic-auth-config-tpl" > \
"${overlay}/ironic-auth-config"
if [[ "${overlay}" =~ -with-inspector ]]; then
IRONIC_INSPECTOR_AUTH_CONFIG_TPL="/tmp/ironic-inspector-auth-config-tpl"
curl -o "${IRONIC_INSPECTOR_AUTH_CONFIG_TPL}" https://raw.githubusercontent.com/metal3-io/baremetal-operator/release-0.5/ironic-deployment/components/basic-auth/ironic-inspector-auth-config-tpl
envsubst < "${IRONIC_INSPECTOR_AUTH_CONFIG_TPL}" > \
"${overlay}/ironic-inspector-auth-config"
echo "INSPECTOR_HTPASSWD=$(htpasswd -n -b -B "${IRONIC_INSPECTOR_USERNAME}" \
"${IRONIC_INSPECTOR_PASSWORD}")" > "${overlay}/ironic-inspector-htpasswd"
fi
done
# We need to gather artifacts/logs before exiting also if there are errors
set +e
# Run the e2e tests
make test-e2e
test_status="$?"
LOGS_DIR="${REPO_ROOT}/test/e2e/_artifacts/logs"
mkdir -p "${LOGS_DIR}/qemu"
sudo sh -c "cp -r /var/log/libvirt/qemu/* ${LOGS_DIR}/qemu/"
sudo chown -R "${USER}:${USER}" "${LOGS_DIR}/qemu"
# Collect all artifacts
tar --directory test/e2e/ -czf "artifacts-e2e-${BMO_E2E_EMULATOR}-${BMC_PROTOCOL}.tar.gz" _artifacts
exit "${test_status}"