Skip to content

Commit

Permalink
Test Chart installation for Plus and remote images (#2060)
Browse files Browse the repository at this point in the history
Problem: We're not testing the Helm Chart installation with NGINX Plus
and with pre-built images

Solution: Add installation steps for both
  • Loading branch information
lucacome authored May 31, 2024
1 parent c8bbcde commit 118488e
Show file tree
Hide file tree
Showing 6 changed files with 203 additions and 113 deletions.
121 changes: 28 additions & 93 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -31,20 +31,33 @@ jobs:
outputs:
go_path: ${{ steps.vars.outputs.go_path }}
min_k8s_version: ${{ steps.vars.outputs.min_k8s_version }}
k8s_latest: ${{ steps.vars.outputs.k8s_latest }}
helm_changes: ${{ steps.filter.outputs.charts }}
steps:
- name: Checkout Repository
uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
with:
fetch-depth: 0

- name: Setup Golang Environment
uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1
with:
go-version: stable

- name: Check for changes
uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
id: filter
with:
filters: |
charts:
- charts/nginx-gateway-fabric/**/*
- name: Output Variables
id: vars
run: |
echo "go_path=$(go env GOPATH)" >> $GITHUB_OUTPUT
echo "min_k8s_version=1.25.16" >> $GITHUB_OUTPUT
echo "min_k8s_version=v1.25.16" >> $GITHUB_OUTPUT
echo "k8s_latest=$(grep -m1 'FROM kindest/node' <tests/Dockerfile | awk -F'[:]' '{print $2}')" >> $GITHUB_OUTPUT
- name: Check if go.mod and go.sum are up to date
run: go mod tidy && git diff --exit-code -- go.mod go.sum
Expand Down Expand Up @@ -211,7 +224,7 @@ jobs:
fail-fast: false
matrix:
image: [nginx, plus]
k8s-version: ["${{ needs.vars.outputs.min_k8s_version }}", "latest"]
k8s-version: ["${{ needs.vars.outputs.min_k8s_version }}", "${{ needs.vars.outputs.k8s_latest }}"]
uses: ./.github/workflows/functional.yml
with:
image: ${{ matrix.image }}
Expand All @@ -226,7 +239,7 @@ jobs:
fail-fast: false
matrix:
image: [nginx, plus]
k8s-version: ["${{ needs.vars.outputs.min_k8s_version }}", "latest"]
k8s-version: ["${{ needs.vars.outputs.min_k8s_version }}", "${{ needs.vars.outputs.k8s_latest }}"]
enable-experimental: [true, false]
uses: ./.github/workflows/conformance.yml
with:
Expand All @@ -238,96 +251,18 @@ jobs:

helm-tests:
name: Helm Tests
runs-on: ubuntu-22.04
needs: [vars, build-oss]
steps:
- name: Checkout Repository
uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6

- name: Fetch Cached Artifacts
uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
with:
path: ${{ github.workspace }}/dist
key: nginx-gateway-fabric-${{ github.run_id }}-${{ github.run_number }}

- name: Docker Buildx
uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0

- name: NGF Docker meta
id: ngf-meta
uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1
with:
images: |
name=ghcr.io/nginxinc/nginx-gateway-fabric
tags: |
type=semver,pattern={{version}}
type=edge
type=ref,event=pr
type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }}
- name: NGINX Docker meta
id: nginx-meta
uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1
with:
images: |
name=ghcr.io/nginxinc/nginx-gateway-fabric/nginx
tags: |
type=semver,pattern={{version}}
type=edge
type=ref,event=pr
type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }}
- name: Build NGF Docker Image
uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0
with:
file: build/Dockerfile
tags: ${{ steps.ngf-meta.outputs.tags }}
context: "."
target: goreleaser
load: true
cache-from: type=gha,scope=ngf
pull: true

- name: Build NGINX Docker Image
uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0
with:
file: build/Dockerfile.nginx
tags: ${{ steps.nginx-meta.outputs.tags }}
context: "."
load: true
cache-from: type=gha,scope=nginx
pull: true
build-args: |
NJS_DIR=internal/mode/static/nginx/modules/src
NGINX_CONF_DIR=internal/mode/static/nginx/conf
BUILD_AGENT=gha
- name: Deploy Kubernetes
id: k8s
run: |
kube_config=${{ github.workspace }}/kube-${{ github.run_id }}-helm
make create-kind-cluster KIND_KUBE_CONFIG=${kube_config}
echo "KUBECONFIG=${kube_config}" >> "$GITHUB_ENV"
kind load docker-image ghcr.io/nginxinc/nginx-gateway-fabric:${{ steps.ngf-meta.outputs.version }} ghcr.io/nginxinc/nginx-gateway-fabric/nginx:${{ steps.nginx-meta.outputs.version }}
kubectl kustomize config/crd/gateway-api/standard | kubectl apply -f -
- name: Install Chart
run: >
helm install
helm-$(echo ${{ steps.ngf-meta.outputs.version }} | tr '.' '-')
.
--wait
--create-namespace
--set nginxGateway.image.repository=ghcr.io/nginxinc/nginx-gateway-fabric
--set nginxGateway.image.tag=${{ steps.ngf-meta.outputs.version }}
--set nginxGateway.image.pullPolicy=Never
--set nginxGateway.productTelemetry.enable=false
--set nginx.image.repository=ghcr.io/nginxinc/nginx-gateway-fabric/nginx
--set nginx.image.tag=${{ steps.nginx-meta.outputs.version }}
--set nginx.image.pullPolicy=Never
--set service.type=NodePort
-n nginx-gateway
working-directory: ${{ github.workspace }}/charts/nginx-gateway-fabric
needs: [vars, build-oss, build-plus]
strategy:
fail-fast: false
matrix:
image: [nginx, plus]
k8s-version: ["${{ needs.vars.outputs.min_k8s_version }}", "${{ needs.vars.outputs.k8s_latest }}"]
uses: ./.github/workflows/helm.yml
with:
image: ${{ matrix.image }}
k8s-version: ${{ matrix.k8s-version }}
secrets: inherit
if: ${{ needs.vars.outputs.helm_changes == 'true' || github.event_name == 'schedule' }}

publish-helm:
name: Package and Publish Helm Chart
Expand Down
11 changes: 5 additions & 6 deletions .github/workflows/conformance.yml
Original file line number Diff line number Diff line change
Expand Up @@ -129,23 +129,22 @@ jobs:
- name: Deploy Kubernetes
id: k8s
run: |
k8s_version=${{ inputs.k8s-version }}
make create-kind-cluster KIND_KUBE_CONFIG=${{ github.workspace }}/kube-${{ github.run_id }} ${{ ! contains(inputs.k8s-version, 'latest') && 'KIND_IMAGE=kindest/node:v${k8s_version}' || '' }}
echo "KUBECONFIG=${{ github.workspace }}/kube-${{ github.run_id }}" >> "$GITHUB_ENV"
working-directory: ./tests
kind create cluster --name ${{ github.run_id }} --image=kindest/node:${{ inputs.k8s-version }}
kind load docker-image ${{ join(fromJSON(steps.ngf-meta.outputs.json).tags, ' ') }} ${{ join(fromJSON(steps.nginx-meta.outputs.json).tags, ' ') }} --name ${{ github.run_id }}
- name: Setup conformance tests
run: |
ngf_prefix=ghcr.io/nginxinc/nginx-gateway-fabric
ngf_tag=${{ steps.ngf-meta.outputs.version }}
if [ ${{ github.event_name }} == "schedule" ]; then export GW_API_VERSION=main; fi
if [ ${{ inputs.enable-experimental }} == "true" ]; then export ENABLE_EXPERIMENTAL=true; fi
make install-ngf-local-no-build${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag}
make helm-install-local${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag}
make deploy-updated-provisioner PREFIX=${ngf_prefix} TAG=${ngf_tag}
working-directory: ./tests

- name: Run conformance tests
run: |
make run-conformance-tests CONFORMANCE_TAG=${{ github.sha }} NGF_VERSION=${{ github.ref_name }}
make run-conformance-tests CONFORMANCE_TAG=${{ github.sha }} NGF_VERSION=${{ github.ref_name }} CLUSTER_NAME=${{ github.run_id }}
core_result=$(cat conformance-profile.yaml | yq '.profiles[0].core.result')
extended_result=$(cat conformance-profile.yaml | yq '.profiles[0].extended.result')
if [ "${core_result}" == "failure" ] || [ "${extended_result}" == "failure" ]; then echo "Conformance test failed, see above for details." && exit 2; fi
Expand Down
13 changes: 2 additions & 11 deletions .github/workflows/functional.yml
Original file line number Diff line number Diff line change
Expand Up @@ -101,17 +101,8 @@ jobs:
- name: Deploy Kubernetes
id: k8s
run: |
k8s_version=${{ inputs.k8s-version }}
make create-kind-cluster KIND_KUBE_CONFIG=${{ github.workspace }}/kube-${{ github.run_id }} ${{ ! contains(inputs.k8s-version, 'latest') && 'KIND_IMAGE=kindest/node:v${k8s_version}' || '' }}
echo "KUBECONFIG=${{ github.workspace }}/kube-${{ github.run_id }}" >> "$GITHUB_ENV"
- name: Setup functional tests
id: setup
run: |
ngf_prefix=ghcr.io/nginxinc/nginx-gateway-fabric
ngf_tag=${{ steps.ngf-meta.outputs.version }}
make load-images${{ inputs.image == 'plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag}
working-directory: ./tests
kind create cluster --name ${{ github.run_id }} --image=kindest/node:${{ inputs.k8s-version }}
kind load docker-image ${{ join(fromJSON(steps.ngf-meta.outputs.json).tags, ' ') }} ${{ join(fromJSON(steps.nginx-meta.outputs.json).tags, ' ') }} --name ${{ github.run_id }}
- name: Run functional telemetry tests
run: |
Expand Down
166 changes: 166 additions & 0 deletions .github/workflows/helm.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,166 @@
name: Helm Testing

on:
workflow_call:
inputs:
image:
required: true
type: string
k8s-version:
required: true
type: string

permissions:
contents: read

jobs:
helm-tests-local:
name: Helm Tests Local
runs-on: ubuntu-22.04
if: ${{ github.event_name != 'schedule' }}
steps:
- name: Checkout Repository
uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
with:
fetch-depth: 0

- name: Fetch Cached Artifacts
uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
with:
path: ${{ github.workspace }}/dist
key: nginx-gateway-fabric-${{ github.run_id }}-${{ github.run_number }}

- name: Docker Buildx
uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0

- name: NGF Docker meta
id: ngf-meta
uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1
with:
images: |
name=ghcr.io/nginxinc/nginx-gateway-fabric
tags: |
type=semver,pattern={{version}}
type=edge
type=schedule
type=ref,event=pr
type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }}
- name: NGINX Docker meta
id: nginx-meta
uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1
with:
images: |
name=ghcr.io/nginxinc/nginx-gateway-fabric/${{ inputs.image == 'plus' && 'nginx-plus' || inputs.image }}
tags: |
type=semver,pattern={{version}}
type=edge
type=schedule
type=ref,event=pr
type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }}
- name: Build NGF Docker Image
uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0
with:
file: build/Dockerfile
tags: ${{ steps.ngf-meta.outputs.tags }}
context: "."
target: goreleaser
load: true
cache-from: type=gha,scope=ngf
pull: true

- name: Build NGINX Docker Image
uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0
with:
file: build/Dockerfile${{ inputs.image == 'nginx' && '.nginx' || '' }}${{ inputs.image == 'plus' && '.nginxplus' || ''}}
tags: ${{ steps.nginx-meta.outputs.tags }}
context: "."
load: true
cache-from: type=gha,scope=${{ inputs.image }}
pull: true
build-args: |
NJS_DIR=internal/mode/static/nginx/modules/src
NGINX_CONF_DIR=internal/mode/static/nginx/conf
BUILD_AGENT=gha
- name: Install cloud-provider-kind
run: |
curl -fsSL -O https://github.com/kubernetes-sigs/cloud-provider-kind/releases/download/v0.2.0/cloud-provider-kind_0.2.0_linux_amd64.tar.gz
tar -xvf cloud-provider-kind_0.2.0_linux_amd64.tar.gz
- name: Run cloud-provider-kind
run: ./cloud-provider-kind & > cloud-provider-kind.log 2>&1

- name: Deploy Kubernetes
id: k8s
run: |
kind create cluster --name ${{ github.run_id }} --image=kindest/node:${{ inputs.k8s-version }}
kind load docker-image ${{ join(fromJSON(steps.ngf-meta.outputs.json).tags, ' ') }} ${{ join(fromJSON(steps.nginx-meta.outputs.json).tags, ' ') }} --name ${{ github.run_id }}
kubectl kustomize config/crd/gateway-api/standard | kubectl apply -f -
- name: Set up Python
uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
with:
python-version: "3.x"
check-latest: true

- name: Set up chart-testing
uses: helm/chart-testing-action@e6669bcd63d7cb57cb4380c33043eebe5d111992 # v2.6.1

- name: Install Chart
run: |
ct install --config .ct.yaml --helm-extra-set-args="--set=nginxGateway.image.tag=${{ steps.ngf-meta.outputs.version }} \
--set=nginx.image.repository=ghcr.io/nginxinc/nginx-gateway-fabric/nginx${{ inputs.image == 'plus' && '-plus' || ''}} \
--set=nginx.plus=${{ inputs.image == 'plus' }} \
--set=nginx.image.tag=${{ steps.nginx-meta.outputs.version }} \
--set=nginxGateway.image.pullPolicy=Never \
--set=nginx.image.pullPolicy=Never \
--set=nginxGateway.productTelemetry.enable=false"
helm-test-remote:
name: Helm Tests Remote
runs-on: ubuntu-22.04
if: ${{ github.event_name == 'schedule' }}
steps:
- name: Checkout Repository
uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
with:
fetch-depth: 0

- name: Install cloud-provider-kind
run: |
curl -fsSL -O https://github.com/kubernetes-sigs/cloud-provider-kind/releases/download/v0.2.0/cloud-provider-kind_0.2.0_linux_amd64.tar.gz
tar -xvf cloud-provider-kind_0.2.0_linux_amd64.tar.gz
- name: Run cloud-provider-kind
run: ./cloud-provider-kind & > cloud-provider-kind.log 2>&1

- name: Deploy Kubernetes
id: k8s
run: |
kind create cluster --name ${{ github.run_id }} --image=kindest/node:${{ inputs.k8s-version }}
kubectl kustomize config/crd/gateway-api/standard | kubectl apply -f -
kubectl create namespace nginx-gateway
- name: Create k8s secret
if: ${{ inputs.image == 'plus' }}
run: |
kubectl create secret docker-registry nginx-plus-registry-secret --docker-server=private-registry.nginx.com --docker-username=${{ secrets.JWT_PLUS_REGISTRY }} --docker-password=none -n nginx-gateway
- name: Set up Python
uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
with:
python-version: "3.x"
check-latest: true

- name: Set up chart-testing
uses: helm/chart-testing-action@e6669bcd63d7cb57cb4380c33043eebe5d111992 # v2.6.1

- name: Install Chart
run: |
ct install --config .ct.yaml --all --namespace nginx-gateway --helm-extra-set-args="--set=nginxGateway.image.tag=nightly \
--set=nginx.plus=${{ inputs.image == 'plus' }} \
--set=nginx.image.tag=nightly \
--set=nginxGateway.productTelemetry.enable=false \
${{ inputs.image == 'plus' && '--set=serviceAccount.imagePullSecret=nginx-plus-registry-secret --set=nginx.image.repository=private-registry.nginx.com/nginx-gateway-fabric/nginx-plus' || '' }}"
2 changes: 0 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@ NGINX_PREFIX ?= $(PREFIX)/nginx## The name of the nginx image. For example: ngin
NGINX_PLUS_PREFIX ?= $(PREFIX)/nginx-plus## The name of the nginx plus image. For example: nginx-gateway-fabric/nginx-plus
TAG ?= $(VERSION:v%=%)## The tag of the image. For example, 1.1.0
TARGET ?= local## The target of the build. Possible values: local and container
KIND_KUBE_CONFIG=$${HOME}/.kube/kind/config## The location of the kind kubeconfig
OUT_DIR ?= build/out## The folder where the binary will be stored
GOARCH ?= amd64## The architecture of the image and/or binary. For example: amd64 or arm64
GOOS ?= linux## The OS of the image and/or binary. For example: linux or darwin
Expand Down Expand Up @@ -149,7 +148,6 @@ deps: ## Add missing and remove unused modules, verify deps and download them to
create-kind-cluster: ## Create a kind cluster
$(eval KIND_IMAGE=$(shell grep -m1 'FROM kindest/node' <$(SELF_DIR)tests/Dockerfile | awk -F'[ ]' '{print $$2}'))
kind create cluster --image $(KIND_IMAGE)
kind export kubeconfig --kubeconfig $(KIND_KUBE_CONFIG)

.PHONY: delete-kind-cluster
delete-kind-cluster: ## Delete kind cluster
Expand Down
Loading

0 comments on commit 118488e

Please sign in to comment.