-
Notifications
You must be signed in to change notification settings - Fork 213
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
refactor: updated STLS bootstrap integration with linux CSE #4386
base: master
Are you sure you want to change the base?
Changes from all commits
75e0a07
bc9d6fa
d78b2c1
6fa54a2
5f24b0f
5c9ca22
2cfa54b
957efa5
1f6c6d8
607a090
a7d0c67
d5f1c31
782f779
319c54e
aaead67
80f990b
164ec41
475b644
e10dea1
d114cd4
18d9edb
6359791
97efab9
58cbac0
0b91a35
c568c8a
ce478fc
627b8dd
b3a0cc5
7b19852
6f0b360
5201760
ed1c29a
2ab182e
faf33f3
8ce2375
3e5e7e1
f18ed87
09f5ef6
e9d75ce
a0f361e
2dc3d11
3f39345
5432003
68ed294
04f9e74
3908614
074126d
c01d45e
89cc014
3cf8573
45209d4
ff3ed23
d81a2f4
d4ce1c7
ab7c029
03c235c
f7888ca
7719d75
1f1bf72
51dc3fc
05320cf
279add8
b10b0f0
337ba54
60d6453
f9da79a
f7b3891
a7e5064
030bacd
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -293,6 +293,27 @@ configureCNIIPTables() { | |
fi | ||
} | ||
|
||
configureKubeletSecureTLSBootstrap() { | ||
# default AAD resource here so we can minimze bootstrap contract surface | ||
AAD_RESOURCE="6dae42f8-4368-4678-94ff-3960e28e3630" | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. If user doesn't set CUSTOM_SECURE_TLS_BOOTSTRAP_AAD_RESOURCE, AAD_RESOURCE is defaulted to 6dae42f8-4368-4678-94ff-3960e28e3630. Does it mean this AAD_RESOURCE is always available in every user's subscription? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. yes, as far as I know - this is the app ID of AKS's AAD server, I've spoken with Baichao and Weinong about this: There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. though will be a good idea to confirm this is correct gov environments as well |
||
if [ -n "$CUSTOM_SECURE_TLS_BOOTSTRAP_AAD_RESOURCE" ]; then | ||
AAD_RESOURCE="$CUSTOM_SECURE_TLS_BOOTSTRAP_AAD_RESOURCE" | ||
fi | ||
TIMEOUT_START_SECONDS=270 # 90s (default) + 180s for secure bootstrapping = 4.5 minutes | ||
|
||
SECURE_TLS_BOOTSTRAP_KUBELET_DROP_IN=/etc/systemd/system/kubelet.service.d/10-securetlsbootstrap.conf | ||
mkdir -p "$(dirname "${SECURE_TLS_BOOTSTRAP_KUBELET_DROP_IN}")" | ||
touch "${SECURE_TLS_BOOTSTRAP_KUBELET_DROP_IN}" | ||
chmod 0600 "${SECURE_TLS_BOOTSTRAP_KUBELET_DROP_IN}" | ||
cat > "${SECURE_TLS_BOOTSTRAP_KUBELET_DROP_IN}" <<EOF | ||
[Service] | ||
TimeoutStartSec=${TIMEOUT_START_SECONDS} | ||
ExecStartPre=-/opt/azure/tlsbootstrap/secure-tls-bootstrap.sh | ||
Environment="SECURE_TLS_BOOTSTRAP_AAD_RESOURCE=${AAD_RESOURCE}" | ||
Environment="API_SERVER_NAME=${API_SERVER_NAME}" | ||
EOF | ||
} | ||
|
||
disableSystemdResolved() { | ||
ls -ltr /etc/resolv.conf | ||
cat /etc/resolv.conf | ||
|
@@ -402,60 +423,15 @@ ensureKubelet() { | |
if [ -n "${AZURE_ENVIRONMENT_FILEPATH}" ]; then | ||
echo "AZURE_ENVIRONMENT_FILEPATH=${AZURE_ENVIRONMENT_FILEPATH}" >> "${KUBELET_DEFAULT_FILE}" | ||
fi | ||
|
||
KUBE_CA_FILE="/etc/kubernetes/certs/ca.crt" | ||
mkdir -p "$(dirname "${KUBE_CA_FILE}")" | ||
echo "${KUBE_CA_CRT}" | base64 -d > "${KUBE_CA_FILE}" | ||
chmod 0600 "${KUBE_CA_FILE}" | ||
|
||
if [ "${ENABLE_SECURE_TLS_BOOTSTRAPPING}" == "true" ] || [ "${ENABLE_TLS_BOOTSTRAPPING}" == "true" ]; then | ||
KUBELET_TLS_DROP_IN="/etc/systemd/system/kubelet.service.d/10-tlsbootstrap.conf" | ||
mkdir -p "$(dirname "${KUBELET_TLS_DROP_IN}")" | ||
touch "${KUBELET_TLS_DROP_IN}" | ||
chmod 0600 "${KUBELET_TLS_DROP_IN}" | ||
tee "${KUBELET_TLS_DROP_IN}" > /dev/null <<EOF | ||
[Service] | ||
Environment="KUBELET_TLS_BOOTSTRAP_FLAGS=--kubeconfig /var/lib/kubelet/kubeconfig --bootstrap-kubeconfig /var/lib/kubelet/bootstrap-kubeconfig" | ||
EOF | ||
fi | ||
|
||
if [ "${ENABLE_SECURE_TLS_BOOTSTRAPPING}" == "true" ]; then | ||
AAD_RESOURCE="6dae42f8-4368-4678-94ff-3960e28e3630" | ||
if [ -n "$CUSTOM_SECURE_TLS_BOOTSTRAP_AAD_SERVER_APP_ID" ]; then | ||
AAD_RESOURCE=$CUSTOM_SECURE_TLS_BOOTSTRAP_AAD_SERVER_APP_ID | ||
fi | ||
SECURE_BOOTSTRAP_KUBECONFIG_FILE=/var/lib/kubelet/bootstrap-kubeconfig | ||
mkdir -p "$(dirname "${SECURE_BOOTSTRAP_KUBECONFIG_FILE}")" | ||
touch "${SECURE_BOOTSTRAP_KUBECONFIG_FILE}" | ||
chmod 0644 "${SECURE_BOOTSTRAP_KUBECONFIG_FILE}" | ||
tee "${SECURE_BOOTSTRAP_KUBECONFIG_FILE}" > /dev/null <<EOF | ||
apiVersion: v1 | ||
kind: Config | ||
clusters: | ||
- name: localcluster | ||
cluster: | ||
certificate-authority: /etc/kubernetes/certs/ca.crt | ||
server: https://${API_SERVER_NAME}:443 | ||
users: | ||
- name: kubelet-bootstrap | ||
user: | ||
exec: | ||
apiVersion: client.authentication.k8s.io/v1 | ||
command: /opt/azure/tlsbootstrap/tls-bootstrap-client | ||
args: | ||
- bootstrap | ||
- --next-proto=aks-tls-bootstrap | ||
- --aad-resource=${AAD_RESOURCE} | ||
interactiveMode: Never | ||
provideClusterInfo: true | ||
contexts: | ||
- context: | ||
cluster: localcluster | ||
user: kubelet-bootstrap | ||
name: bootstrap-context | ||
current-context: bootstrap-context | ||
EOF | ||
elif [ "${ENABLE_TLS_BOOTSTRAPPING}" == "true" ]; then | ||
if [ "${ENABLE_TLS_BOOTSTRAPPING}" == "true" ]; then | ||
# used in vanilla TLS bootstrapping cases and when secure TLS bootstrapping has failed to generate a kubeconfig | ||
# by the time we need to start kubelet | ||
BOOTSTRAP_KUBECONFIG_FILE=/var/lib/kubelet/bootstrap-kubeconfig | ||
mkdir -p "$(dirname "${BOOTSTRAP_KUBECONFIG_FILE}")" | ||
touch "${BOOTSTRAP_KUBECONFIG_FILE}" | ||
|
@@ -479,7 +455,13 @@ contexts: | |
name: bootstrap-context | ||
current-context: bootstrap-context | ||
EOF | ||
else | ||
fi | ||
|
||
if [ "${ENABLE_SECURE_TLS_BOOTSTRAPPING}" == "true" ]; then | ||
configureKubeletSecureTLSBootstrap | ||
fi | ||
|
||
if [ "${ENABLE_SECURE_TLS_BOOTSTRAPPING}" == "false" ] && [ "${ENABLE_TLS_BOOTSTRAPPING}" == "false" ]; then | ||
cameronmeissner marked this conversation as resolved.
Show resolved
Hide resolved
|
||
KUBECONFIG_FILE=/var/lib/kubelet/kubeconfig | ||
mkdir -p "$(dirname "${KUBECONFIG_FILE}")" | ||
touch "${KUBECONFIG_FILE}" | ||
|
@@ -528,6 +510,16 @@ EOF | |
logs_to_events "AKS.CSE.ensureKubelet.installCredentalProvider" installCredentalProvider | ||
fi | ||
|
||
if [ "${ENABLE_SECURE_TLS_BOOTSTRAPPING}" == "true" ]; then | ||
# we continue with CSE without ensuring the kubelet startup completed | ||
# due to user-assigned managed identities not being assigned by RP to agent nodes | ||
# until after the initial PUT has been completed, thus we need to ensure to | ||
# complete CSE as quickly as possible so secure bootstrapping has a chance to run | ||
# in cases where the node is using a kubelet identity | ||
systemctlEnableAndStartNoBlock kubelet || exit $ERR_KUBELET_START_FAIL | ||
return 0 | ||
fi | ||
|
||
systemctlEnableAndStart kubelet || exit $ERR_KUBELET_START_FAIL | ||
} | ||
|
||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,93 @@ | ||
#!/bin/bash | ||
|
||
set -euxo pipefail | ||
|
||
EVENTS_LOGGING_DIR="/var/log/azure/Microsoft.Azure.Extensions.CustomScript/events" | ||
NEXT_PROTO_VALUE="aks-tls-bootstrap" | ||
|
||
RETRY_PERIOD_SECONDS=180 # 3 minutes | ||
RETRY_WAIT_SECONDS=3 | ||
|
||
AAD_RESOURCE="${SECURE_TLS_BOOTSTRAP_AAD_RESOURCE:-""}" | ||
API_SERVER_NAME="${API_SERVER_NAME:-""}" | ||
|
||
CLIENT_BINARY_PATH="${SECURE_TLS_BOOTSTRAP_CLIENT_BINARY_PATH:-/opt/azure/tlsbootstrap/tls-bootstrap-client}" | ||
KUBECONFIG_PATH="${SECURE_TLS_BOOTSTRAP_KUBECONFIG_PATH:-/var/lib/kubelet/kubeconfig}" | ||
CLIENT_CERT_PATH="${SECURE_TLS_BOOTSTRAP_CLIENT_CERT_PATH:-/etc/kubernetes/certs/client.crt}" | ||
CLIENT_KEY_PATH="${SECURE_TLS_BOOTSTRAP_CLIENT_KEY_PATH:-/etc/kubernetes/certs/client.key}" | ||
AZURE_CONFIG_PATH="${SECURE_TLS_BOOTSTRAP_ZURE_CONFIG_PATH:-/etc/kubernetes/azure.json}" | ||
CLUSTER_CA_FILE_PATH="${SECURE_TLS_BOOTSTRAP_CLUSTER_CA_FILE_PATH:-/etc/kubernetes/certs/ca.crt}" | ||
LOG_FILE_PATH="${SECURE_TLS_BOOTSTRAP_LOG_FILE_PATH:-/var/log/azure/aks/secure-tls-bootstrap.log}" | ||
|
||
logs_to_events() { | ||
local task=$1; shift | ||
local eventsFileName=$(date +%s%3N) | ||
|
||
local startTime=$(date +"%F %T.%3N") | ||
${@} | ||
ret=$? | ||
local endTime=$(date +"%F %T.%3N") | ||
|
||
msg_string=$(jq -n --arg Status "Succeeded" '{Status: $Status}') | ||
if [ "$ret" != "0" ]; then | ||
msg_string=$(jq -n --arg Status "Failed" --arg LogTail "$(tail -n 10 $LOG_FILE_PATH)" '{Status: $Status, LogTail: $LogTail}') | ||
fi | ||
|
||
json_string=$( jq -n \ | ||
--arg Timestamp "${startTime}" \ | ||
--arg OperationId "${endTime}" \ | ||
--arg Version "1.23" \ | ||
--arg TaskName "${task}" \ | ||
--arg EventLevel "Informational" \ | ||
--arg Message "${msg_string}" \ | ||
--arg EventPid "0" \ | ||
--arg EventTid "0" \ | ||
'{Timestamp: $Timestamp, OperationId: $OperationId, Version: $Version, TaskName: $TaskName, EventLevel: $EventLevel, Message: $Message, EventPid: $EventPid, EventTid: $EventTid}' | ||
) | ||
echo ${json_string} > "${EVENTS_LOGGING_DIR}/${eventsFileName}.json" | ||
|
||
if [ "$ret" != "0" ]; then | ||
return $ret | ||
fi | ||
} | ||
|
||
bootstrap() { | ||
if [ -z "$API_SERVER_NAME" ]; then | ||
echo "ERROR: missing apiserver FQDN, cannot continue bootstrapping" | ||
return 1 | ||
fi | ||
if [ ! -f "$CLIENT_BINARY_PATH" ]; then | ||
echo "ERROR: bootstrap client binary does not exist at path $CLIENT_BINARY_PATH" | ||
return 1 | ||
fi | ||
|
||
chmod +x $CLIENT_BINARY_PATH | ||
|
||
deadline=$(($(date +%s) + RETRY_PERIOD_SECONDS)) | ||
while true; do | ||
now=$(date +%s) | ||
if [ $((now - deadline)) -ge 0 ]; then | ||
echo "ERROR: bootstrapping deadline exceeded" | ||
return 1 | ||
fi | ||
|
||
$CLIENT_BINARY_PATH bootstrap \ | ||
--aad-resource="$AAD_RESOURCE" \ | ||
--apiserver-fqdn="$API_SERVER_NAME" \ | ||
--cluster-ca-file="$CLUSTER_CA_FILE_PATH" \ | ||
--azure-config="$AZURE_CONFIG_PATH" \ | ||
--cert-file="$CLIENT_CERT_PATH" \ | ||
--key-file="$CLIENT_KEY_PATH" \ | ||
--next-proto="$NEXT_PROTO_VALUE" \ | ||
--kubeconfig="$KUBECONFIG_PATH" \ | ||
--log-file="$LOG_FILE_PATH" | ||
|
||
[ $? -eq 0 ] && echo "secure TLS bootstrapping succeeded, generated kubeconfig is at ${KUBECONFIG_PATH}" && break | ||
|
||
sleep $RETRY_WAIT_SECONDS | ||
done | ||
} | ||
|
||
logs_to_events "AKS.performSecureTLSBootstrapping" bootstrap || exit $? | ||
|
||
#EOF |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
cse_cmd.sh var naming change, want to loop in @Bryce-Soghigian @tallaxes here for viz, however, I dont think this variable is being used yet that the renaming would break them?
still tagging them.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
thanks, keep forgetting
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Ack, and appreciated. Not used yet, so should be fine - but only as long as not setting it at all is tolerated by the rest of the logic.