Skip to content

Commit

Permalink
fix: use fixed retry timeout in cse nslookup step
Browse files Browse the repository at this point in the history
  • Loading branch information
Li Ma committed Feb 22, 2024
1 parent a021138 commit 4cae0bd
Show file tree
Hide file tree
Showing 216 changed files with 1,458 additions and 361 deletions.
16 changes: 16 additions & 0 deletions parts/linux/cloud-init/artifacts/cse_helpers.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,22 @@ retrycmd_if_failure() {
done
echo Executed \"$@\" $i times;
}
retrycmd_nslookup() {
wait_sleep=$1; timeout=$2; total_timeout=$3; record=$4
start_time=$(date +%s)
while true; do
nslookup -timeout=$timeout -retry=0 $record && break || \
current_time=$(date +%s)
# Check if the total_timeout has been reached
if [ $((current_time - start_time)) -ge $total_timeout ]; then
echo "Total timeout $total_timeout reached, nslookup -timeout=$timeout -retry=0 $record failed"
return 1
fi
sleep $wait_sleep
done
current_time=$(date +%s)
echo "Executed nslookup -timeout=$timeout -retry=0 $record for $((current_time - start_time)) seconds";
}
retrycmd_if_failure_no_stats() {
retries=$1; wait_sleep=$2; timeout=$3; shift && shift && shift
for i in $(seq 1 $retries); do
Expand Down
6 changes: 3 additions & 3 deletions parts/linux/cloud-init/artifacts/cse_main.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -324,12 +324,12 @@ if [[ $API_SERVER_NAME == *.privatelink.* ]]; then
API_SERVER_CONN_RETRIES=100
fi
if ! [[ ${API_SERVER_NAME} =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
API_SERVER_DNS_RETRIES=100
API_SERVER_DNS_RETRY_TIMEOUT=300
if [[ $API_SERVER_NAME == *.privatelink.* ]]; then
API_SERVER_DNS_RETRIES=200
API_SERVER_DNS_RETRY_TIMEOUT=600
fi
if [[ "${ENABLE_HOSTS_CONFIG_AGENT}" != "true" ]]; then
RES=$(logs_to_events "AKS.CSE.apiserverNslookup" "retrycmd_if_failure ${API_SERVER_DNS_RETRIES} 1 20 nslookup -timeout=5 -retry=0 ${API_SERVER_NAME}")
RES=$(logs_to_events "AKS.CSE.apiserverNslookup" "retrycmd_nslookup 1 15 ${API_SERVER_DNS_RETRY_TIMEOUT} ${API_SERVER_NAME}")
STS=$?
else
STS=0
Expand Down
4 changes: 2 additions & 2 deletions pkg/agent/testdata/AKSUbuntu1604+Containerd/CustomData
View file

Large diffs are not rendered by default.

6 changes: 3 additions & 3 deletions pkg/agent/testdata/AKSUbuntu1604+Containerd/line33.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -280,12 +280,12 @@ if [[ $API_SERVER_NAME == *.privatelink.* ]]; then
API_SERVER_CONN_RETRIES=100
fi
if ! [[ ${API_SERVER_NAME} =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
API_SERVER_DNS_RETRIES=100
API_SERVER_DNS_RETRY_TIMEOUT=300
if [[ $API_SERVER_NAME == *.privatelink.* ]]; then
API_SERVER_DNS_RETRIES=200
API_SERVER_DNS_RETRY_TIMEOUT=600
fi
if [[ "${ENABLE_HOSTS_CONFIG_AGENT}" != "true" ]]; then
RES=$(logs_to_events "AKS.CSE.apiserverNslookup" "retrycmd_if_failure ${API_SERVER_DNS_RETRIES} 1 20 nslookup -timeout=5 -retry=0 ${API_SERVER_NAME}")
RES=$(logs_to_events "AKS.CSE.apiserverNslookup" "retrycmd_nslookup 1 15 ${API_SERVER_DNS_RETRY_TIMEOUT} ${API_SERVER_NAME}")
STS=$?
else
STS=0
Expand Down
15 changes: 15 additions & 0 deletions pkg/agent/testdata/AKSUbuntu1604+Containerd/line9.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,21 @@ retrycmd_if_failure() {
done
echo Executed \"$@\" $i times;
}
retrycmd_nslookup() {
wait_sleep=$1; timeout=$2; total_timeout=$3; record=$4
start_time=$(date +%s)
while true; do
nslookup -timeout=$timeout -retry=0 $record && break || \
current_time=$(date +%s)
if [ $((current_time - start_time)) -ge $total_timeout ]; then
echo "Total timeout $total_timeout reached, nslookup -timeout=$timeout -retry=0 $record failed"
return 1
fi
sleep $wait_sleep
done
current_time=$(date +%s)
echo "Executed nslookup -timeout=$timeout -retry=0 $record for $((current_time - start_time)) seconds";
}
retrycmd_if_failure_no_stats() {
retries=$1; wait_sleep=$2; timeout=$3; shift && shift && shift
for i in $(seq 1 $retries); do
Expand Down
4 changes: 2 additions & 2 deletions pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+CustomLinuxOSConfig/CustomData
View file

Large diffs are not rendered by default.

6 changes: 3 additions & 3 deletions pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+CustomLinuxOSConfig/line33.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -280,12 +280,12 @@ if [[ $API_SERVER_NAME == *.privatelink.* ]]; then
API_SERVER_CONN_RETRIES=100
fi
if ! [[ ${API_SERVER_NAME} =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
API_SERVER_DNS_RETRIES=100
API_SERVER_DNS_RETRY_TIMEOUT=300
if [[ $API_SERVER_NAME == *.privatelink.* ]]; then
API_SERVER_DNS_RETRIES=200
API_SERVER_DNS_RETRY_TIMEOUT=600
fi
if [[ "${ENABLE_HOSTS_CONFIG_AGENT}" != "true" ]]; then
RES=$(logs_to_events "AKS.CSE.apiserverNslookup" "retrycmd_if_failure ${API_SERVER_DNS_RETRIES} 1 20 nslookup -timeout=5 -retry=0 ${API_SERVER_NAME}")
RES=$(logs_to_events "AKS.CSE.apiserverNslookup" "retrycmd_nslookup 1 15 ${API_SERVER_DNS_RETRY_TIMEOUT} ${API_SERVER_NAME}")
STS=$?
else
STS=0
Expand Down
15 changes: 15 additions & 0 deletions pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+CustomLinuxOSConfig/line9.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,21 @@ retrycmd_if_failure() {
done
echo Executed \"$@\" $i times;
}
retrycmd_nslookup() {
wait_sleep=$1; timeout=$2; total_timeout=$3; record=$4
start_time=$(date +%s)
while true; do
nslookup -timeout=$timeout -retry=0 $record && break || \
current_time=$(date +%s)
if [ $((current_time - start_time)) -ge $total_timeout ]; then
echo "Total timeout $total_timeout reached, nslookup -timeout=$timeout -retry=0 $record failed"
return 1
fi
sleep $wait_sleep
done
current_time=$(date +%s)
echo "Executed nslookup -timeout=$timeout -retry=0 $record for $((current_time - start_time)) seconds";
}
retrycmd_if_failure_no_stats() {
retries=$1; wait_sleep=$2; timeout=$3; shift && shift && shift
for i in $(seq 1 $retries); do
Expand Down
4 changes: 2 additions & 2 deletions pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+DynamicKubeletConfig/CustomData
View file

Large diffs are not rendered by default.

6 changes: 3 additions & 3 deletions pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+DynamicKubeletConfig/line33.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -280,12 +280,12 @@ if [[ $API_SERVER_NAME == *.privatelink.* ]]; then
API_SERVER_CONN_RETRIES=100
fi
if ! [[ ${API_SERVER_NAME} =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
API_SERVER_DNS_RETRIES=100
API_SERVER_DNS_RETRY_TIMEOUT=300
if [[ $API_SERVER_NAME == *.privatelink.* ]]; then
API_SERVER_DNS_RETRIES=200
API_SERVER_DNS_RETRY_TIMEOUT=600
fi
if [[ "${ENABLE_HOSTS_CONFIG_AGENT}" != "true" ]]; then
RES=$(logs_to_events "AKS.CSE.apiserverNslookup" "retrycmd_if_failure ${API_SERVER_DNS_RETRIES} 1 20 nslookup -timeout=5 -retry=0 ${API_SERVER_NAME}")
RES=$(logs_to_events "AKS.CSE.apiserverNslookup" "retrycmd_nslookup 1 15 ${API_SERVER_DNS_RETRY_TIMEOUT} ${API_SERVER_NAME}")
STS=$?
else
STS=0
Expand Down
15 changes: 15 additions & 0 deletions pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+DynamicKubeletConfig/line9.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,21 @@ retrycmd_if_failure() {
done
echo Executed \"$@\" $i times;
}
retrycmd_nslookup() {
wait_sleep=$1; timeout=$2; total_timeout=$3; record=$4
start_time=$(date +%s)
while true; do
nslookup -timeout=$timeout -retry=0 $record && break || \
current_time=$(date +%s)
if [ $((current_time - start_time)) -ge $total_timeout ]; then
echo "Total timeout $total_timeout reached, nslookup -timeout=$timeout -retry=0 $record failed"
return 1
fi
sleep $wait_sleep
done
current_time=$(date +%s)
echo "Executed nslookup -timeout=$timeout -retry=0 $record for $((current_time - start_time)) seconds";
}
retrycmd_if_failure_no_stats() {
retries=$1; wait_sleep=$2; timeout=$3; shift && shift && shift
for i in $(seq 1 $retries); do
Expand Down
Loading

0 comments on commit 4cae0bd

Please sign in to comment.