diff --git a/.github/workflows/nfr.yml b/.github/workflows/nfr.yml index 6dcc993e41..37c768636e 100644 --- a/.github/workflows/nfr.yml +++ b/.github/workflows/nfr.yml @@ -123,9 +123,9 @@ jobs: working-directory: ./tests run: | if ${{ needs.vars.outputs.test_label != 'all' }}; then - sed -i '/^GINKGO_LABEL=/s/=.*/="${{ needs.vars.outputs.test_label }}"/' "scripts/vars.env" && make nfr-test; + sed -i '/^GINKGO_LABEL=/s/=.*/="${{ needs.vars.outputs.test_label }}"/' "scripts/vars.env" && make nfr-test CI=true; else - make nfr-test; + make nfr-test CI=true; fi - name: Upload Artifacts diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cbed6607f0..30527b893f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -81,6 +81,12 @@ repos: - --template-files=_templates.gotmpl - --template-files=README.md.gotmpl + - repo: https://github.com/scop/pre-commit-shfmt + rev: v3.8.0-1 + hooks: + - id: shfmt-src + args: [-w, -s, -i, "4"] + ci: skip: [golangci-lint-full, prettier, markdownlint-cli2, yamllint] autofix_prs: false diff --git a/scripts/generate-manifests.sh b/scripts/generate-manifests.sh index 705b5e48e7..d387bf71fa 100755 --- a/scripts/generate-manifests.sh +++ b/scripts/generate-manifests.sh @@ -5,27 +5,27 @@ charts=$(find examples/helm -maxdepth 1 -mindepth 1 -type d -exec basename {} \;) generate_manifests() { - chart=$1 - manifest=deploy/$chart/deploy.yaml - mkdir -p deploy/$chart - - helm_parameters="--namespace nginx-gateway --set nameOverride=nginx-gateway --skip-crds" - if [ "$chart" == "openshift" ]; then - chart="default" - helm_parameters="$helm_parameters --api-versions security.openshift.io/v1/SecurityContextConstraints" - fi - - helm template nginx-gateway $helm_parameters --values examples/helm/$chart/values.yaml charts/nginx-gateway-fabric >$manifest 2>/dev/null - sed -i.bak '/app.kubernetes.io\/managed-by: Helm/d' $manifest - sed -i.bak '/helm.sh/d' $manifest - cp $manifest config/base - kubectl kustomize config/base >$manifest - rm -f config/base/deploy.yaml - rm -f $manifest.bak + chart=$1 + manifest=deploy/${chart}/deploy.yaml + mkdir -p deploy/${chart} + + helm_parameters="--namespace nginx-gateway --set nameOverride=nginx-gateway --skip-crds" + if [ "${chart}" == "openshift" ]; then + chart="default" + helm_parameters="${helm_parameters} --api-versions security.openshift.io/v1/SecurityContextConstraints" + fi + + helm template nginx-gateway ${helm_parameters} --values examples/helm/${chart}/values.yaml charts/nginx-gateway-fabric >${manifest} 2>/dev/null + sed -i.bak '/app.kubernetes.io\/managed-by: Helm/d' ${manifest} + sed -i.bak '/helm.sh/d' ${manifest} + cp ${manifest} config/base + kubectl kustomize config/base >${manifest} + rm -f config/base/deploy.yaml + rm -f ${manifest}.bak } -for chart in $charts; do - generate_manifests $chart +for chart in ${charts}; do + generate_manifests ${chart} done # For OpenShift, we don't need a Helm example so we generate the manifests from the default values.yaml @@ -33,7 +33,7 @@ generate_manifests openshift # FIXME(lucacome): Implement a better way to generate the static deployment file # https://github.com/nginxinc/nginx-gateway-fabric/issues/2326 -helm template nginx-gateway charts/nginx-gateway-fabric --set nameOverride=nginx-gateway --set metrics.enable=false --set nginxGateway.productTelemetry.enable=false -n nginx-gateway -s templates/deployment.yaml > config/tests/static-deployment.yaml +helm template nginx-gateway charts/nginx-gateway-fabric --set nameOverride=nginx-gateway --set metrics.enable=false --set nginxGateway.productTelemetry.enable=false -n nginx-gateway -s templates/deployment.yaml >config/tests/static-deployment.yaml sed -i.bak '/app.kubernetes.io\/managed-by: Helm/d' config/tests/static-deployment.yaml sed -i.bak '/helm.sh/d' config/tests/static-deployment.yaml rm -f config/tests/static-deployment.yaml.bak diff --git a/tests/reconfig/scripts/create-resources-gw-last.sh b/tests/reconfig/scripts/create-resources-gw-last.sh index 8fe41f45d3..a1be0fc9e6 100755 --- a/tests/reconfig/scripts/create-resources-gw-last.sh +++ b/tests/reconfig/scripts/create-resources-gw-last.sh @@ -3,7 +3,7 @@ num_namespaces=$1 # Create namespaces -for ((i=1; i<=$num_namespaces; i++)); do +for ((i = 1; i <= num_namespaces; i++)); do namespace_name="namespace$i" kubectl create namespace "$namespace_name" done @@ -13,13 +13,13 @@ kubectl create -f certificate-ns-and-cafe-secret.yaml kubectl create -f reference-grant.yaml # Create backend service and apps -for ((i=1; i<=$num_namespaces; i++)); do +for ((i = 1; i <= num_namespaces; i++)); do namespace_name="namespace$i" sed -e "s/coffee/coffee${namespace_name}/g" -e "s/tea/tea${namespace_name}/g" cafe.yaml | kubectl apply -n "$namespace_name" -f - done # Create routes -for ((i=1; i<=$num_namespaces; i++)); do +for ((i = 1; i <= num_namespaces; i++)); do namespace_name="namespace$i" sed -e "s/coffee/coffee${namespace_name}/g" -e "s/tea/tea${namespace_name}/g" cafe-routes.yaml | kubectl apply -n "$namespace_name" -f - done diff --git a/tests/reconfig/scripts/create-resources-routes-last.sh b/tests/reconfig/scripts/create-resources-routes-last.sh index c502c22d96..be41d9a706 100755 --- a/tests/reconfig/scripts/create-resources-routes-last.sh +++ b/tests/reconfig/scripts/create-resources-routes-last.sh @@ -3,13 +3,13 @@ num_namespaces=$1 # Create namespaces -for ((i=1; i<=$num_namespaces; i++)); do +for ((i = 1; i <= num_namespaces; i++)); do namespace_name="namespace$i" kubectl create namespace "$namespace_name" done # Create backend service and apps -for ((i=1; i<=$num_namespaces; i++)); do +for ((i = 1; i <= num_namespaces; i++)); do namespace_name="namespace$i" sed -e "s/coffee/coffee${namespace_name}/g" -e "s/tea/tea${namespace_name}/g" cafe.yaml | kubectl apply -n "$namespace_name" -f - done @@ -23,7 +23,7 @@ kubectl create -f reference-grant.yaml kubectl create -f gateway.yaml # Create routes -for ((i=1; i<=$num_namespaces; i++)); do +for ((i = 1; i <= num_namespaces; i++)); do namespace_name="namespace$i" sed -e "s/coffee/coffee${namespace_name}/g" -e "s/tea/tea${namespace_name}/g" cafe-routes.yaml | kubectl apply -n "$namespace_name" -f - done diff --git a/tests/reconfig/scripts/delete-multiple.sh b/tests/reconfig/scripts/delete-multiple.sh index 8f479833b4..2f9752e8c9 100755 --- a/tests/reconfig/scripts/delete-multiple.sh +++ b/tests/reconfig/scripts/delete-multiple.sh @@ -4,11 +4,11 @@ num_namespaces=$1 # Delete namespaces namespaces="" -for ((i=1; i<=$num_namespaces; i++)); do - namespaces+="namespace$i " +for ((i = 1; i <= num_namespaces; i++)); do + namespaces+="namespace${i} " done -kubectl delete namespace $namespaces +kubectl delete namespace "${namespaces}" # Delete single instance resources kubectl delete -f gateway.yaml diff --git a/tests/scripts/add-local-ip-auth-networks.sh b/tests/scripts/add-local-ip-auth-networks.sh index 98466d1811..2ba06012f5 100755 --- a/tests/scripts/add-local-ip-auth-networks.sh +++ b/tests/scripts/add-local-ip-auth-networks.sh @@ -1,8 +1,10 @@ #!/usr/bin/env bash +set -eo pipefail + source scripts/vars.env -CURRENT_AUTH_NETWORK=$(gcloud container clusters describe ${GKE_CLUSTER_NAME} --zone=${GKE_CLUSTER_ZONE} \ - --format="value(masterAuthorizedNetworksConfig.cidrBlocks[0])" | sed 's/cidrBlock=//') +CURRENT_AUTH_NETWORK=$(gcloud container clusters describe "${GKE_CLUSTER_NAME}" --zone="${GKE_CLUSTER_ZONE}" \ + --format="value(masterAuthorizedNetworksConfig.cidrBlocks[0])" | sed 's/cidrBlock=//') -gcloud container clusters update ${GKE_CLUSTER_NAME} --zone=${GKE_CLUSTER_ZONE} --enable-master-authorized-networks --master-authorized-networks=${SOURCE_IP_RANGE},${CURRENT_AUTH_NETWORK} +gcloud container clusters update "${GKE_CLUSTER_NAME}" --zone="${GKE_CLUSTER_ZONE}" --enable-master-authorized-networks --master-authorized-networks="${SOURCE_IP_RANGE}","${CURRENT_AUTH_NETWORK}" diff --git a/tests/scripts/check-pod-exit-code.sh b/tests/scripts/check-pod-exit-code.sh index a7455c7a56..f7a0f43419 100755 --- a/tests/scripts/check-pod-exit-code.sh +++ b/tests/scripts/check-pod-exit-code.sh @@ -1,6 +1,8 @@ #!/usr/bin/env bash +set -eo pipefail + CODE=$(kubectl get pod conformance -o jsonpath='{.status.containerStatuses[].state.terminated.exitCode}') -if [ $CODE -ne 0 ]; then +if [ "${CODE}" -ne 0 ]; then exit 2 fi diff --git a/tests/scripts/cleanup-router.sh b/tests/scripts/cleanup-router.sh index 7b95b642c8..fe7c215a65 100755 --- a/tests/scripts/cleanup-router.sh +++ b/tests/scripts/cleanup-router.sh @@ -1,6 +1,8 @@ #!/usr/bin/env bash +set -eo pipefail + source scripts/vars.env -gcloud compute routers nats delete ${RESOURCE_NAME} --quiet --router ${RESOURCE_NAME} --router-region ${GKE_CLUSTER_REGION} -gcloud compute routers delete ${RESOURCE_NAME} --quiet --region ${GKE_CLUSTER_REGION} +gcloud compute routers nats delete "${RESOURCE_NAME}" --quiet --router "${RESOURCE_NAME}" --router-region "${GKE_CLUSTER_REGION}" +gcloud compute routers delete "${RESOURCE_NAME}" --quiet --region "${GKE_CLUSTER_REGION}" diff --git a/tests/scripts/cleanup-vm.sh b/tests/scripts/cleanup-vm.sh index 6c7b4e6fda..b98589e463 100755 --- a/tests/scripts/cleanup-vm.sh +++ b/tests/scripts/cleanup-vm.sh @@ -1,17 +1,17 @@ #!/usr/bin/env bash +set -eo pipefail + source scripts/vars.env skip_gke_master_control_node_access="${1:-false}" # Remove VM IP from GKE master control node access, if required -if [ ${ADD_VM_IP_AUTH_NETWORKS} = "true" ] && [ ${skip_gke_master_control_node_access} != "true" ]; then - EXTERNAL_IP=$(gcloud compute instances describe ${RESOURCE_NAME} --zone ${GKE_CLUSTER_ZONE} --project=${GKE_PROJECT} --zone=${GKE_CLUSTER_ZONE} \ - --format='value(networkInterfaces[0].accessConfigs[0].natIP)') - CURRENT_AUTH_NETWORK=$(gcloud container clusters describe ${GKE_CLUSTER_NAME} --zone ${GKE_CLUSTER_ZONE} \ - --format="value(masterAuthorizedNetworksConfig.cidrBlocks[0])" | sed 's/cidrBlock=//') - gcloud container clusters update ${GKE_CLUSTER_NAME} --zone ${GKE_CLUSTER_ZONE} --enable-master-authorized-networks --master-authorized-networks=${CURRENT_AUTH_NETWORK} +if [ "${ADD_VM_IP_AUTH_NETWORKS}" = "true" ] && [ "${skip_gke_master_control_node_access}" != "true" ]; then + CURRENT_AUTH_NETWORK=$(gcloud container clusters describe "${GKE_CLUSTER_NAME}" --zone "${GKE_CLUSTER_ZONE}" \ + --format="value(masterAuthorizedNetworksConfig.cidrBlocks[0])" | sed 's/cidrBlock=//') + gcloud container clusters update "${GKE_CLUSTER_NAME}" --zone "${GKE_CLUSTER_ZONE}" --enable-master-authorized-networks --master-authorized-networks="${CURRENT_AUTH_NETWORK}" fi -gcloud compute instances delete ${RESOURCE_NAME} --quiet --project=${GKE_PROJECT} --zone=${GKE_CLUSTER_ZONE} -gcloud compute firewall-rules delete ${RESOURCE_NAME} --quiet --project=${GKE_PROJECT} +gcloud compute instances delete "${RESOURCE_NAME}" --quiet --project="${GKE_PROJECT}" --zone="${GKE_CLUSTER_ZONE}" +gcloud compute firewall-rules delete "${RESOURCE_NAME}" --quiet --project="${GKE_PROJECT}" diff --git a/tests/scripts/create-and-setup-gcp-vm.sh b/tests/scripts/create-and-setup-gcp-vm.sh index dd97a5e87d..0e66c5f99b 100755 --- a/tests/scripts/create-and-setup-gcp-vm.sh +++ b/tests/scripts/create-and-setup-gcp-vm.sh @@ -1,40 +1,42 @@ #!/usr/bin/env bash -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +set -o pipefail + +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) source scripts/vars.env -gcloud compute firewall-rules create ${RESOURCE_NAME} \ - --project=${GKE_PROJECT} \ +gcloud compute firewall-rules create "${RESOURCE_NAME}" \ + --project="${GKE_PROJECT}" \ --direction=INGRESS \ --priority=1000 \ --network=default \ --action=ALLOW \ --rules=tcp:22 \ - --source-ranges=${SOURCE_IP_RANGE} \ - --target-tags=${NETWORK_TAGS} + --source-ranges="${SOURCE_IP_RANGE}" \ + --target-tags="${NETWORK_TAGS}" -gcloud compute instances create ${RESOURCE_NAME} --project=${GKE_PROJECT} --zone=${GKE_CLUSTER_ZONE} --machine-type=n2-standard-2 \ +gcloud compute instances create "${RESOURCE_NAME}" --project="${GKE_PROJECT}" --zone="${GKE_CLUSTER_ZONE}" --machine-type=n2-standard-2 \ --network-interface=network-tier=PREMIUM,stack-type=IPV4_ONLY,subnet=default --maintenance-policy=MIGRATE \ - --provisioning-model=STANDARD --service-account=${GKE_SVC_ACCOUNT} \ + --provisioning-model=STANDARD --service-account="${GKE_SVC_ACCOUNT}" \ --scopes=https://www.googleapis.com/auth/devstorage.read_only,https://www.googleapis.com/auth/logging.write,https://www.googleapis.com/auth/monitoring.write,https://www.googleapis.com/auth/servicecontrol,https://www.googleapis.com/auth/service.management.readonly,https://www.googleapis.com/auth/trace.append,https://www.googleapis.com/auth/cloud-platform \ - --tags=${NETWORK_TAGS} --create-disk=auto-delete=yes,boot=yes,device-name=${RESOURCE_NAME},image-family=projects/${GKE_PROJECT}/global/images/ngf-debian,mode=rw,size=20 --no-shielded-secure-boot --shielded-vtpm --shielded-integrity-monitoring --labels=goog-ec-src=vm_add-gcloud --reservation-affinity=any + --tags="${NETWORK_TAGS}" --create-disk=auto-delete=yes,boot=yes,device-name="${RESOURCE_NAME}",image-family=projects/"${GKE_PROJECT}"/global/images/ngf-debian,mode=rw,size=20 --no-shielded-secure-boot --shielded-vtpm --shielded-integrity-monitoring --labels=goog-ec-src=vm_add-gcloud --reservation-affinity=any # Add VM IP to GKE master control node access, if required if [ "${ADD_VM_IP_AUTH_NETWORKS}" = "true" ]; then - EXTERNAL_IP=$(gcloud compute instances describe ${RESOURCE_NAME} --project=${GKE_PROJECT} --zone=${GKE_CLUSTER_ZONE} \ - --format='value(networkInterfaces[0].accessConfigs[0].natIP)') - CURRENT_AUTH_NETWORK=$(gcloud container clusters describe ${GKE_CLUSTER_NAME} --zone=${GKE_CLUSTER_ZONE} \ - --format="value(masterAuthorizedNetworksConfig.cidrBlocks[0])" | sed 's/cidrBlock=//') - gcloud container clusters update ${GKE_CLUSTER_NAME} --zone=${GKE_CLUSTER_ZONE} --enable-master-authorized-networks --master-authorized-networks=${EXTERNAL_IP}/32,${CURRENT_AUTH_NETWORK} + EXTERNAL_IP=$(gcloud compute instances describe "${RESOURCE_NAME}" --project="${GKE_PROJECT}" --zone="${GKE_CLUSTER_ZONE}" \ + --format='value(networkInterfaces[0].accessConfigs[0].natIP)') + CURRENT_AUTH_NETWORK=$(gcloud container clusters describe "${GKE_CLUSTER_NAME}" --zone="${GKE_CLUSTER_ZONE}" \ + --format="value(masterAuthorizedNetworksConfig.cidrBlocks[0])" | sed 's/cidrBlock=//') + gcloud container clusters update "${GKE_CLUSTER_NAME}" --zone="${GKE_CLUSTER_ZONE}" --enable-master-authorized-networks --master-authorized-networks="${EXTERNAL_IP}"/32,"${CURRENT_AUTH_NETWORK}" fi # Poll for SSH connectivity MAX_RETRIES=10 RETRY_INTERVAL=5 -for ((i=1; i<=MAX_RETRIES; i++)); do +for ((i = 1; i <= MAX_RETRIES; i++)); do echo "Attempt $i to connect to the VM..." - gcloud compute ssh username@${RESOURCE_NAME} --zone=${GKE_CLUSTER_ZONE} --project=${GKE_PROJECT} --quiet --command="echo 'VM is ready'" + gcloud compute ssh username@"${RESOURCE_NAME}" --zone="${GKE_CLUSTER_ZONE}" --project="${GKE_PROJECT}" --quiet --command="echo 'VM is ready'" if [ $? -eq 0 ]; then echo "SSH connection successful. VM is ready." break @@ -43,18 +45,18 @@ for ((i=1; i<=MAX_RETRIES; i++)); do sleep ${RETRY_INTERVAL} done -gcloud compute scp --zone ${GKE_CLUSTER_ZONE} --project=${GKE_PROJECT} ${SCRIPT_DIR}/vars.env username@${RESOURCE_NAME}:~ +gcloud compute scp --zone "${GKE_CLUSTER_ZONE}" --project="${GKE_PROJECT}" "${SCRIPT_DIR}"/vars.env username@"${RESOURCE_NAME}":~ if [ -n "${NGF_REPO}" ] && [ "${NGF_REPO}" != "nginxinc" ]; then - gcloud compute ssh --zone ${GKE_CLUSTER_ZONE} --project=${GKE_PROJECT} username@${RESOURCE_NAME} \ - --command="bash -i < /dev/null && pwd ) - ${SCRIPT_DIR}/add-local-ip-auth-networks.sh + SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) + "${SCRIPT_DIR}"/add-local-ip-auth-networks.sh fi diff --git a/tests/scripts/create-gke-router.sh b/tests/scripts/create-gke-router.sh index 421fc74f8a..599986638b 100755 --- a/tests/scripts/create-gke-router.sh +++ b/tests/scripts/create-gke-router.sh @@ -1,13 +1,15 @@ #!/usr/bin/env bash +set -eo pipefail + source scripts/vars.env -gcloud compute routers create ${RESOURCE_NAME} \ - --region ${GKE_CLUSTER_REGION} \ +gcloud compute routers create "${RESOURCE_NAME}" \ + --region "${GKE_CLUSTER_REGION}" \ --network default -gcloud compute routers nats create ${RESOURCE_NAME} \ - --router-region ${GKE_CLUSTER_REGION} \ - --router ${RESOURCE_NAME} \ +gcloud compute routers nats create "${RESOURCE_NAME}" \ + --router-region "${GKE_CLUSTER_REGION}" \ + --router "${RESOURCE_NAME}" \ --nat-all-subnet-ip-ranges \ --auto-allocate-nat-external-ips diff --git a/tests/scripts/delete-gke-cluster.sh b/tests/scripts/delete-gke-cluster.sh index 0f390bbd8f..2cebd452ad 100755 --- a/tests/scripts/delete-gke-cluster.sh +++ b/tests/scripts/delete-gke-cluster.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +set -eo pipefail + source scripts/vars.env -gcloud container clusters delete ${GKE_CLUSTER_NAME} --zone ${GKE_CLUSTER_ZONE} --project ${GKE_PROJECT} --quiet +gcloud container clusters delete "${GKE_CLUSTER_NAME}" --zone "${GKE_CLUSTER_ZONE}" --project "${GKE_PROJECT}" --quiet diff --git a/tests/scripts/remote-scripts/run-nfr-tests.sh b/tests/scripts/remote-scripts/run-nfr-tests.sh index c61077cbf9..d2db339478 100755 --- a/tests/scripts/remote-scripts/run-nfr-tests.sh +++ b/tests/scripts/remote-scripts/run-nfr-tests.sh @@ -2,16 +2,16 @@ set -e -source ~/vars.env +source "${HOME}"/vars.env -if [ "$START_LONGEVITY" == "true" ]; then +if [ "${START_LONGEVITY}" == "true" ]; then GINKGO_LABEL="longevity-setup" -elif [ "$STOP_LONGEVITY" == "true" ]; then +elif [ "${STOP_LONGEVITY}" == "true" ]; then GINKGO_LABEL="longevity-teardown" fi -cd nginx-gateway-fabric/tests && make .vm-nfr-test TAG=${TAG} PREFIX=${PREFIX} NGINX_PREFIX=${NGINX_PREFIX} NGINX_PLUS_PREFIX=${NGINX_PLUS_PREFIX} PLUS_ENABLED=${PLUS_ENABLED} GINKGO_LABEL=${GINKGO_LABEL} GINKGO_FLAGS=${GINKGO_FLAGS} PULL_POLICY=Always GW_SERVICE_TYPE=LoadBalancer GW_SVC_GKE_INTERNAL=true NGF_VERSION=${NGF_VERSION} +cd nginx-gateway-fabric/tests && make .vm-nfr-test TAG="${TAG}" PREFIX="${PREFIX}" NGINX_PREFIX="${NGINX_PREFIX}" NGINX_PLUS_PREFIX="${NGINX_PLUS_PREFIX}" PLUS_ENABLED="${PLUS_ENABLED}" GINKGO_LABEL=${GINKGO_LABEL} GINKGO_FLAGS="${GINKGO_FLAGS}" PULL_POLICY=Always GW_SERVICE_TYPE=LoadBalancer GW_SVC_GKE_INTERNAL=true NGF_VERSION="${NGF_VERSION}" -if [ "$START_LONGEVITY" == "true" ]; then +if [ "${START_LONGEVITY}" == "true" ]; then suite/scripts/longevity-wrk.sh fi diff --git a/tests/scripts/remote-scripts/run-tests.sh b/tests/scripts/remote-scripts/run-tests.sh index dad6dbc946..db681d2e32 100755 --- a/tests/scripts/remote-scripts/run-tests.sh +++ b/tests/scripts/remote-scripts/run-tests.sh @@ -2,6 +2,6 @@ set -e -source ~/vars.env +source "${HOME}"/vars.env -cd nginx-gateway-fabric/tests && make test TAG=${TAG} PREFIX=${PREFIX} NGINX_PREFIX=${NGINX_PREFIX} NGINX_PLUS_PREFIX=${NGINX_PLUS_PREFIX} PLUS_ENABLED=${PLUS_ENABLED} GINKGO_LABEL=${GINKGO_LABEL} GINKGO_FLAGS=${GINKGO_FLAGS} PULL_POLICY=Always GW_SERVICE_TYPE=LoadBalancer GW_SVC_GKE_INTERNAL=true NGF_VERSION=${NGF_VERSION} +cd nginx-gateway-fabric/tests && make test TAG="${TAG}" PREFIX="${PREFIX}" NGINX_PREFIX="${NGINX_PREFIX}" NGINX_PLUS_PREFIX="${NGINX_PLUS_PREFIX}" PLUS_ENABLED="${PLUS_ENABLED}" GINKGO_LABEL="${GINKGO_LABEL}" GINKGO_FLAGS="${GINKGO_FLAGS}" PULL_POLICY=Always GW_SERVICE_TYPE=LoadBalancer GW_SVC_GKE_INTERNAL=true NGF_VERSION="${NGF_VERSION}" diff --git a/tests/scripts/run-tests-gcp-vm.sh b/tests/scripts/run-tests-gcp-vm.sh index 9fa635b3a2..91c30b0ec1 100755 --- a/tests/scripts/run-tests-gcp-vm.sh +++ b/tests/scripts/run-tests-gcp-vm.sh @@ -1,6 +1,8 @@ #!/usr/bin/env bash -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +set -eo pipefail + +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) source scripts/vars.env @@ -9,48 +11,49 @@ if [ "${NFR}" = "true" ]; then SCRIPT=run-nfr-tests.sh fi -gcloud compute scp --zone ${GKE_CLUSTER_ZONE} --project=${GKE_PROJECT} ${SCRIPT_DIR}/vars.env username@${RESOURCE_NAME}:~ +gcloud compute scp --zone "${GKE_CLUSTER_ZONE}" --project="${GKE_PROJECT}" "${SCRIPT_DIR}"/vars.env username@"${RESOURCE_NAME}":~ -gcloud compute ssh --zone ${GKE_CLUSTER_ZONE} --project=${GKE_PROJECT} username@${RESOURCE_NAME} \ +gcloud compute ssh --zone "${GKE_CLUSTER_ZONE}" --project="${GKE_PROJECT}" username@"${RESOURCE_NAME}" \ --command="export START_LONGEVITY=${START_LONGEVITY} &&\ export STOP_LONGEVITY=${STOP_LONGEVITY} &&\ - bash -s" < ${SCRIPT_DIR}/remote-scripts/${SCRIPT}; retcode=$? + bash -s" <"${SCRIPT_DIR}"/remote-scripts/${SCRIPT} +retcode=$? -if [ $retcode -ne 0 ]; then - echo "Error running tests on VM" - exit 1 +if [ ${retcode} -ne 0 ]; then + echo "Error running tests on VM" + exit 1 fi if [ "${NFR}" = "true" ]; then ## Use rsync if running locally (faster); otherwise if in the pipeline don't download an SSH config if [ "${CI}" = "false" ]; then - gcloud compute config-ssh --ssh-config-file ngf-gcp.ssh > /dev/null - rsync -ave 'ssh -F ngf-gcp.ssh' username@${RESOURCE_NAME}.${GKE_CLUSTER_ZONE}.${GKE_PROJECT}:~/nginx-gateway-fabric/tests/results . + gcloud compute config-ssh --ssh-config-file ngf-gcp.ssh >/dev/null + rsync -ave 'ssh -F ngf-gcp.ssh' username@"${RESOURCE_NAME}"."${GKE_CLUSTER_ZONE}"."${GKE_PROJECT}":~/nginx-gateway-fabric/tests/results . else - gcloud compute scp --zone ${GKE_CLUSTER_ZONE} --project=${GKE_PROJECT} --recurse username@${RESOURCE_NAME}:~/nginx-gateway-fabric/tests/results . + gcloud compute scp --zone "${GKE_CLUSTER_ZONE}" --project="${GKE_PROJECT}" --recurse username@"${RESOURCE_NAME}":~/nginx-gateway-fabric/tests/results . fi fi ## If tearing down the longevity test, we need to collect logs from gcloud and add to the results if [ "${STOP_LONGEVITY}" = "true" ]; then version=${NGF_VERSION} - if [ "$version" = "" ]; then + if [ "${version}" = "" ]; then version=${TAG} fi results="${SCRIPT_DIR}/../results/longevity/$version/$version.md" - printf "\n## Error Logs\n\n" >> $results + printf "\n## Error Logs\n\n" >>"${results}" ## ngf error logs - ngfErrText=$(gcloud logging read --project=${GKE_PROJECT} 'resource.labels.cluster_name='"${RESOURCE_NAME}"' AND resource.type=k8s_container AND resource.labels.container_name=nginx-gateway AND labels."k8s-pod/app_kubernetes_io/instance"=ngf-longevity AND severity=ERROR AND SEARCH("error")' --format "value(textPayload)") - ngfErrJSON=$(gcloud logging read --project=${GKE_PROJECT} 'resource.labels.cluster_name='"${RESOURCE_NAME}"' AND resource.type=k8s_container AND resource.labels.container_name=nginx-gateway AND labels."k8s-pod/app_kubernetes_io/instance"=ngf-longevity AND severity=ERROR AND SEARCH("error")' --format "value(jsonPayload)") - printf "### nginx-gateway\n$ngfErrText\n$ngfErrJSON\n\n" >> $results + ngfErrText=$(gcloud logging read --project="${GKE_PROJECT}" 'resource.labels.cluster_name='"${RESOURCE_NAME}"' AND resource.type=k8s_container AND resource.labels.container_name=nginx-gateway AND labels."k8s-pod/app_kubernetes_io/instance"=ngf-longevity AND severity=ERROR AND SEARCH("error")' --format "value(textPayload)") + ngfErrJSON=$(gcloud logging read --project="${GKE_PROJECT}" 'resource.labels.cluster_name='"${RESOURCE_NAME}"' AND resource.type=k8s_container AND resource.labels.container_name=nginx-gateway AND labels."k8s-pod/app_kubernetes_io/instance"=ngf-longevity AND severity=ERROR AND SEARCH("error")' --format "value(jsonPayload)") + printf "### nginx-gateway\n%s\n%s\n\n" "${ngfErrText}" "${ngfErrJSON}" >>"${results}" ## nginx error logs - ngxErr=$(gcloud logging read --project=${GKE_PROJECT} 'resource.labels.cluster_name='"${RESOURCE_NAME}"' AND resource.type=k8s_container AND resource.labels.container_name=nginx AND labels."k8s-pod/app_kubernetes_io/instance"=ngf-longevity AND severity=ERROR AND SEARCH("`[warn]`") OR SEARCH("`[error]`") OR SEARCH("`[emerg]`")' --format "value(textPayload)") - printf "### nginx\n$ngxErr\n\n" >> $results + ngxErr=$(gcloud logging read --project="${GKE_PROJECT}" 'resource.labels.cluster_name='"${RESOURCE_NAME}"' AND resource.type=k8s_container AND resource.labels.container_name=nginx AND labels."k8s-pod/app_kubernetes_io/instance"=ngf-longevity AND severity=ERROR AND SEARCH("`[warn]`") OR SEARCH("`[error]`") OR SEARCH("`[emerg]`")' --format "value(textPayload)") + printf "### nginx\n%s\n\n" "${ngxErr}" >>"${results}" ## nginx non-200 responses (also filter out 499 since wrk cancels connections) - ngxNon200=$(gcloud logging read --project=${GKE_PROJECT} 'resource.labels.cluster_name='"${RESOURCE_NAME}"' AND resource.type=k8s_container AND resource.labels.container_name=nginx AND labels."k8s-pod/app_kubernetes_io/instance"=ngf-longevity AND "GET" "HTTP/1.1" -"200" -"499" -"client prematurely closed connection"' --format "value(textPayload)") - printf "$ngxNon200\n\n" >> $results + ngxNon200=$(gcloud logging read --project="${GKE_PROJECT}" 'resource.labels.cluster_name='"${RESOURCE_NAME}"' AND resource.type=k8s_container AND resource.labels.container_name=nginx AND labels."k8s-pod/app_kubernetes_io/instance"=ngf-longevity AND "GET" "HTTP/1.1" -"200" -"499" -"client prematurely closed connection"' --format "value(textPayload)") + printf "%s\n\n" "${ngxNon200}" >>"${results}" fi diff --git a/tests/scripts/sync-files-to-vm.sh b/tests/scripts/sync-files-to-vm.sh index 5055555ce6..f8ecaabc6e 100755 --- a/tests/scripts/sync-files-to-vm.sh +++ b/tests/scripts/sync-files-to-vm.sh @@ -1,9 +1,11 @@ #!/usr/bin/env bash +set -eo pipefail + source scripts/vars.env NGF_DIR=$(dirname "$PWD") -gcloud compute config-ssh --ssh-config-file ngf-gcp.ssh > /dev/null +gcloud compute config-ssh --ssh-config-file ngf-gcp.ssh >/dev/null -rsync -ave 'ssh -F ngf-gcp.ssh' ${NGF_DIR} username@${RESOURCE_NAME}.${GKE_CLUSTER_ZONE}.${GKE_PROJECT}:~ +rsync -ave 'ssh -F ngf-gcp.ssh' "${NGF_DIR}" username@"${RESOURCE_NAME}"."${GKE_CLUSTER_ZONE}"."${GKE_PROJECT}":~ diff --git a/tests/suite/scripts/longevity-wrk.sh b/tests/suite/scripts/longevity-wrk.sh index 7d758c0976..e7d3a6b23a 100755 --- a/tests/suite/scripts/longevity-wrk.sh +++ b/tests/suite/scripts/longevity-wrk.sh @@ -4,6 +4,6 @@ SVC_IP=$(kubectl -n nginx-gateway get svc ngf-longevity-nginx-gateway-fabric -o echo "${SVC_IP} cafe.example.com" | sudo tee -a /etc/hosts -nohup wrk -t2 -c100 -d96h http://cafe.example.com/coffee &> ~/coffee.txt & +nohup wrk -t2 -c100 -d96h http://cafe.example.com/coffee &>~/coffee.txt & -nohup wrk -t2 -c100 -d96h https://cafe.example.com/tea &> ~/tea.txt & +nohup wrk -t2 -c100 -d96h https://cafe.example.com/tea &>~/tea.txt &