Add NGINX plus Dockerfile, add make command, add metrics (#1394)

* Add nginx plus Dockerfile, add make command, add metrics
nginx · Dec 19, 2023 · c02f0e8 · c02f0e8
1 parent e659685
commit c02f0e8
Show file tree

Hide file tree

Showing 19 changed files with 224 additions and 22 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -215,6 +215,7 @@ jobs:
           build-args: |
             NJS_DIR=internal/mode/static/nginx/modules/src
             NGINX_CONF_DIR=internal/mode/static/nginx/conf
+            BUILD_AGENT=gha
 
       - name: Deploy Kubernetes
         id: k8s

diff --git a/.gitignore b/.gitignore
@@ -37,3 +37,7 @@ internal/mode/static/nginx/modules/coverage
 
 # MacOS Finder
 .DS_Store
+
+# Certs and keys
+*.crt
+*.key
diff --git a/Makefile b/Makefile
@@ -6,6 +6,8 @@ MANIFEST_DIR = $(shell pwd)/deploy/manifests
 CHART_DIR = $(shell pwd)/deploy/helm-chart
 NGINX_CONF_DIR = internal/mode/static/nginx/conf
 NJS_DIR = internal/mode/static/nginx/modules/src
+NGINX_DOCKER_BUILD_PLUS_ARGS = --secret id=nginx-repo.crt,src=nginx-repo.crt --secret id=nginx-repo.key,src=nginx-repo.key
+BUILD_AGENT=local
 
 # go build flags - should not be overridden by the user
 GO_LINKER_FlAGS_VARS = -X main.version=${VERSION} -X main.commit=${GIT_COMMIT} -X main.date=${DATE}
@@ -15,6 +17,7 @@ GO_LINKER_FLAGS = $(GO_LINKER_FLAGS_OPTIMIZATIONS) $(GO_LINKER_FlAGS_VARS)
 # variables that can be overridden by the user
 PREFIX ?= nginx-gateway-fabric## The name of the NGF image. For example, nginx-gateway-fabric
 NGINX_PREFIX ?= $(PREFIX)/nginx## The name of the nginx image. For example: nginx-gateway-fabric/nginx
+NGINX_PLUS_PREFIX ?= $(PREFIX)/nginxplus## The name of the nginx plus image. For example: nginx-gateway-fabric/nginxplus
 TAG ?= $(VERSION:v%=%)## The tag of the image. For example, 0.3.0
 TARGET ?= local## The target of the build. Possible values: local and container
 KIND_KUBE_CONFIG=$${HOME}/.kube/kind/config## The location of the kind kubeconfig
@@ -23,7 +26,7 @@ GOARCH ?= amd64## The architecture of the image and/or binary. For example: amd6
 GOOS ?= linux## The OS of the image and/or binary. For example: linux or darwin
 override HELM_TEMPLATE_COMMON_ARGS += --set creator=template --set nameOverride=nginx-gateway## The common options for the Helm template command.
 override HELM_TEMPLATE_EXTRA_ARGS_FOR_ALL_MANIFESTS_FILE += --set service.create=false## The options to be passed to the full Helm templating command only.
-override NGINX_DOCKER_BUILD_OPTIONS += --build-arg NJS_DIR=$(NJS_DIR) --build-arg NGINX_CONF_DIR=$(NGINX_CONF_DIR)
+override NGINX_DOCKER_BUILD_OPTIONS += --build-arg NJS_DIR=$(NJS_DIR) --build-arg NGINX_CONF_DIR=$(NGINX_CONF_DIR) --build-arg BUILD_AGENT=$(BUILD_AGENT)
 .DEFAULT_GOAL := help
 
 .PHONY: help
@@ -34,6 +37,9 @@ help: Makefile ## Display this help
 .PHONY: build-images
 build-images: build-ngf-image build-nginx-image ## Build the NGF and nginx docker images
 
+.PHONY: build-images-with-plus
+build-images-with-plus: build-ngf-image build-nginx-plus-image ## Build the NGF and NGINX Plus docker images
+
 .PHONY: build-ngf-image
 build-ngf-image: check-for-docker build ## Build the NGF docker image
 	docker build --platform linux/$(GOARCH) --target $(strip $(TARGET)) -f build/Dockerfile -t $(strip $(PREFIX)):$(strip $(TAG)) .
@@ -42,6 +48,10 @@ build-ngf-image: check-for-docker build ## Build the NGF docker image
 build-nginx-image: check-for-docker ## Build the custom nginx image
 	docker build --platform linux/$(GOARCH) $(strip $(NGINX_DOCKER_BUILD_OPTIONS)) -f build/Dockerfile.nginx -t $(strip $(NGINX_PREFIX)):$(strip $(TAG)) .
 
+.PHONY: build-nginx-plus-image
+build-nginx-plus-image: check-for-docker ## Build the custom nginx plus image
+	docker build --platform linux/$(GOARCH) $(strip $(NGINX_DOCKER_BUILD_OPTIONS)) $(strip $(NGINX_DOCKER_BUILD_PLUS_ARGS))  -f build/Dockerfile.nginxplus -t $(strip $(NGINX_PLUS_PREFIX)):$(strip $(TAG)) .
+
 .PHONY: check-for-docker
 check-for-docker: ## Check if Docker is installed
 	@docker -v || (code=$$?; printf "\033[0;31mError\033[0m: there was a problem with Docker\n"; exit $$code)

diff --git a/README.md b/README.md
@@ -62,17 +62,17 @@ the [Issue Lifecycle](ISSUE_LIFECYCLE.md) document for information on issue crea
 
 The following table lists the software versions NGINX Gateway Fabric supports.
 
-| NGINX Gateway Fabric | Gateway API | Kubernetes | NGINX OSS |
-|----------------------|-------------|------------|-----------|
-| Edge                 | 1.0.0       | 1.23+      | 1.25.3    |
-| 1.1.0                | 1.0.0       | 1.23+      | 1.25.3    |
-| 1.0.0                | 0.8.1       | 1.23+      | 1.25.2    |
-| 0.6.0                | 0.8.0       | 1.23+      | 1.25.2    |
-| 0.5.0                | 0.7.1       | 1.21+      | 1.25.x *  |
-| 0.4.0                | 0.7.1       | 1.21+      | 1.25.x *  |
-| 0.3.0                | 0.6.2       | 1.21+      | 1.23.x *  |
-| 0.2.0                | 0.5.1       | 1.21+      | 1.21.x *  |
-| 0.1.0                | 0.5.0       | 1.19+      | 1.21.3    |
+| NGINX Gateway Fabric | Gateway API | Kubernetes | NGINX OSS | NGINX Plus |
+|----------------------|-------------|------------|-----------|------------|
+| Edge                 | 1.0.0       | 1.23+      | 1.25.3    | R30        |
+| 1.1.0                | 1.0.0       | 1.23+      | 1.25.3    | n/a        |
+| 1.0.0                | 0.8.1       | 1.23+      | 1.25.2    | n/a        |
+| 0.6.0                | 0.8.0       | 1.23+      | 1.25.2    | n/a        |
+| 0.5.0                | 0.7.1       | 1.21+      | 1.25.x *  | n/a        |
+| 0.4.0                | 0.7.1       | 1.21+      | 1.25.x *  | n/a        |
+| 0.3.0                | 0.6.2       | 1.21+      | 1.23.x *  | n/a        |
+| 0.2.0                | 0.5.1       | 1.21+      | 1.21.x *  | n/a        |
+| 0.1.0                | 0.5.0       | 1.19+      | 1.21.3    | n/a        |
 
 \*the installation manifests use the minor version of NGINX container image (e.g. 1.25) and the patch version is not
 specified. This means that the latest available patch version is used.

diff --git a/build/Dockerfile.nginx b/build/Dockerfile.nginx
@@ -3,6 +3,7 @@ FROM nginx:1.25.3-alpine
 
 ARG NJS_DIR
 ARG NGINX_CONF_DIR
+ARG BUILD_AGENT
 
 RUN apk update && apk upgrade && apk add --no-cache libcap \
     && mkdir -p /var/lib/nginx /usr/lib/nginx/modules \
@@ -15,4 +16,6 @@ COPY ${NGINX_CONF_DIR}/nginx.conf /etc/nginx/nginx.conf
 
 RUN chown -R 101:1001 /etc/nginx /var/cache/nginx /var/lib/nginx
 
+LABEL org.nginx.ngf.image.build.agent="${BUILD_AGENT}"
+
 USER 101:1001
diff --git a/build/Dockerfile.nginxplus b/build/Dockerfile.nginxplus
@@ -0,0 +1,34 @@
+# syntax=docker/dockerfile:1.4
+FROM alpine:3.18
+
+ARG NGINX_PLUS_VERSION=R30
+ARG NJS_DIR
+ARG NGINX_CONF_DIR
+ARG BUILD_AGENT
+
+RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/apk/cert.pem,mode=0644 \
+	--mount=type=secret,id=nginx-repo.key,dst=/etc/apk/cert.key,mode=0644 \
+	addgroup -g 1001 -S nginx \
+    && adduser -S -D -H -u 101 -h /var/cache/nginx -s /sbin/nologin -G nginx -g nginx nginx \
+	&& wget -nv -O /etc/apk/keys/nginx_signing.rsa.pub https://cs.nginx.com/static/keys/nginx_signing.rsa.pub \
+	&& printf "%s\n" "https://pkgs.nginx.com/plus/${NGINX_PLUS_VERSION}/alpine/v$(grep -E -o '^[0-9]+\.[0-9]+' /etc/alpine-release)/main" >> /etc/apk/repositories \
+	&& apk add --no-cache nginx-plus nginx-plus-module-njs libcap \
+	&& ldconfig /usr/local/lib/ \
+    && mkdir -p /var/lib/nginx /usr/lib/nginx/modules \
+    && setcap 'cap_net_bind_service=+ep' /usr/sbin/nginx \
+    && setcap -v 'cap_net_bind_service=+ep' /usr/sbin/nginx \
+    && apk del libcap \
+	# forward request and error logs to docker log collector
+    && ln -sf /dev/stdout /var/log/nginx/access.log \
+    && ln -sf /dev/stderr /var/log/nginx/error.log
+
+COPY ${NJS_DIR}/httpmatches.js /usr/lib/nginx/modules/njs/httpmatches.js
+COPY ${NGINX_CONF_DIR}/nginx-plus.conf /etc/nginx/nginx.conf
+
+RUN chown -R 101:1001 /etc/nginx /var/cache/nginx /var/lib/nginx
+
+USER 101:1001
+
+LABEL org.nginx.ngf.image.build.agent="${BUILD_AGENT}"
+
+CMD ["nginx", "-g", "daemon off;"]
diff --git a/cmd/gateway/commands.go b/cmd/gateway/commands.go
@@ -55,6 +55,7 @@ func createStaticModeCommand() *cobra.Command {
 		healthPortFlag             = "health-port"
 		leaderElectionDisableFlag  = "leader-election-disable"
 		leaderElectionLockNameFlag = "leader-election-lock-name"
+		plusFlag                   = "nginx-plus"
 	)
 
 	// flag values
@@ -92,6 +93,8 @@ func createStaticModeCommand() *cobra.Command {
 			validator: validateResourceName,
 			value:     "nginx-gateway-leader-election-lock",
 		}
+
+		plus bool
 	)
 
 	cmd := &cobra.Command{
@@ -160,6 +163,7 @@ func createStaticModeCommand() *cobra.Command {
 					LockName: leaderElectionLockName.String(),
 					Identity: podName,
 				},
+				Plus: plus,
 			}
 
 			if err := static.StartManager(conf); err != nil {
@@ -266,6 +270,13 @@ func createStaticModeCommand() *cobra.Command {
 			"A Lease object with this name will be created in the same Namespace as the controller.",
 	)
 
+	cmd.Flags().BoolVar(
+		&plus,
+		plusFlag,
+		false,
+		"Use NGINX Plus",
+	)
+
 	return cmd
 }
 

diff --git a/deploy/helm-chart/README.md b/deploy/helm-chart/README.md
@@ -9,6 +9,8 @@
     - [Installing the Chart via Sources](#installing-the-chart-via-sources)
       - [Pulling the Chart](#pulling-the-chart)
       - [Installing the Chart](#installing-the-chart-1)
+    - [Custom installation options](#custom-installation-options)
+      - [Service type](#service-type)
   - [Upgrading the Chart](#upgrading-the-chart)
     - [Upgrading the Gateway Resources](#upgrading-the-gateway-resources)
     - [Upgrading the CRDs](#upgrading-the-crds)
@@ -275,7 +277,7 @@ kubectl delete -f https://github.com/kubernetes-sigs/gateway-api/releases/downlo
 The following tables lists the configurable parameters of the NGINX Gateway Fabric chart and their default values.
 
 | Parameter                                         | Description                                                                                                                                                                                              | Default Value                                                                                                   |
-|---------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------|
+| ------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------- |
 | `nginxGateway.image.repository`                   | The repository for the NGINX Gateway Fabric image.                                                                                                                                                       | ghcr.io/nginxinc/nginx-gateway-fabric                                                                           |
 | `nginxGateway.image.tag`                          | The tag for the NGINX Gateway Fabric image.                                                                                                                                                              | edge                                                                                                            |
 | `nginxGateway.image.pullPolicy`                   | The `imagePullPolicy` for the NGINX Gateway Fabric image.                                                                                                                                                | Always                                                                                                          |
@@ -294,6 +296,7 @@ The following tables lists the configurable parameters of the NGINX Gateway Fabr
 | `nginx.image.repository`                          | The repository for the NGINX image.                                                                                                                                                                      | ghcr.io/nginxinc/nginx-gateway-fabric/nginx                                                                     |
 | `nginx.image.tag`                                 | The tag for the NGINX image.                                                                                                                                                                             | edge                                                                                                            |
 | `nginx.image.pullPolicy`                          | The `imagePullPolicy` for the NGINX image.                                                                                                                                                               | Always                                                                                                          |
+| `nginx.plus`                                      | Is NGINX Plus image being used                                                                                                                                                                           | false                                                                                                           |
 | `nginx.lifecycle`                                 | The `lifecycle` of the nginx container.                                                                                                                                                                  | {}                                                                                                              |
 | `nginx.extraVolumeMounts`                         | Extra `volumeMounts` for the nginx container.                                                                                                                                                            | {}                                                                                                              |
 | `terminationGracePeriodSeconds`                   | The termination grace period of the NGINX Gateway Fabric pod.                                                                                                                                            | 30                                                                                                              |

diff --git a/deploy/helm-chart/templates/deployment.yaml b/deploy/helm-chart/templates/deployment.yaml
@@ -31,6 +31,9 @@ spec:
         - --gatewayclass={{ .Values.nginxGateway.gatewayClassName }}
         - --config={{ include "nginx-gateway.config-name" . }}
         - --service={{ include "nginx-gateway.fullname" . }}
+        {{- if .Values.nginx.plus }}
+        - --nginx-plus
+        {{- end }}
         {{- if .Values.metrics.enable }}
         - --metrics-port={{ .Values.metrics.port }}
         {{- if .Values.metrics.secure  }}

diff --git a/deploy/helm-chart/values.yaml b/deploy/helm-chart/values.yaml
@@ -58,6 +58,9 @@ nginx:
     tag: edge
     pullPolicy: Always
 
+  ## Is NGINX Plus image being used
+  plus: false
+
   ## The lifecycle of the nginx container.
   lifecycle: {}
 

diff --git a/docs/developer/quickstart.md b/docs/developer/quickstart.md
@@ -48,7 +48,7 @@ Follow these steps to set up your development environment.
    make deps
    ```
 
-## Build the Binary and Image
+## Build the Binary and Images
 
 ### Build the Binary
 
@@ -70,6 +70,19 @@ make TAG=$(whoami) build-images
 
 This will build the docker images `nginx-gateway-fabric:<your-user>` and `nginx-gateway-fabric/nginx:<your-user>`.
 
+### Build the Images with NGINX Plus
+
+> Note: You will need a valid NGINX Plus license certificate and key named `nginx-repo.crt` and `nginx-repo.key` in the
+> root of this repo to build the NGINX Plus image.
+
+To build the NGINX Gateway Fabric and NGINX Plus container images from source run the following make command:
+
+```makefile
+make TAG=$(whoami) build-images-with-plus
+```
+
+This will build the docker images `nginx-gateway-fabric:<your-user>` and `nginx-gateway-fabric/nginxplus:<your-user>`.
+
 ## Deploy on Kind
 
 1. Create a `kind` cluster:
@@ -84,6 +97,12 @@ This will build the docker images `nginx-gateway-fabric:<your-user>` and `nginx-
    kind load docker-image nginx-gateway-fabric:$(whoami) nginx-gateway-fabric/nginx:$(whoami)
    ```
 
+   or
+
+   ```shell
+   kind load docker-image nginx-gateway-fabric:$(whoami) nginx-gateway-fabric/nginxplus:$(whoami)
+   ```
+
 3. Install Gateway API CRDs:
 
    ```shell
@@ -98,7 +117,13 @@ This will build the docker images `nginx-gateway-fabric:<your-user>` and `nginx-
       helm install my-release ./deploy/helm-chart --create-namespace --wait --set service.type=NodePort --set nginxGateway.image.repository=nginx-gateway-fabric --set nginxGateway.image.tag=$(whoami) --set nginxGateway.image.pullPolicy=Never --set nginx.image.repository=nginx-gateway-fabric/nginx --set nginx.image.tag=$(whoami) --set nginx.image.pullPolicy=Never -n nginx-gateway
       ```
 
-      > For more information on helm configuration options see the Helm [README](../../deploy/helm-chart/README.md).
+   - To install NGINX Plus with Helm (where your release name is `my-release`):
+
+      ```shell
+      helm install my-release ./deploy/helm-chart --create-namespace --wait --set service.type=NodePort --set nginxGateway.image.repository=nginx-gateway-fabric --set nginxGateway.image.tag=$(whoami) --set nginxGateway.image.pullPolicy=Never --set nginx.image.repository=nginx-gateway-fabric/nginxplus --set nginx.image.tag=$(whoami) --set nginx.image.pullPolicy=Never --set nginx.plus=true -n nginx-gateway
+      ```
+
+   > For more information on Helm configuration options see the Helm [README](../../deploy/helm-chart/README.md).
 
    - To install with manifests:
 
@@ -109,6 +134,15 @@ This will build the docker images `nginx-gateway-fabric:<your-user>` and `nginx-
       kubectl apply -f deploy/manifests/service/nodeport.yaml
       ```
 
+   - To install NGINX Plus with manifests:
+
+      ```shell
+      make generate-manifests HELM_TEMPLATE_COMMON_ARGS="--set nginxGateway.image.repository=nginx-gateway-fabric --set nginxGateway.image.tag=$(whoami) --set nginxGateway.image.pullPolicy=Never --set nginx.image.repository=nginx-gateway-fabric/nginxplus --set nginx.image.tag=$(whoami) --set nginx.image.pullPolicy=Never --set nginx.plus=true"
+      kubectl apply -f deploy/manifests/crds
+      kubectl apply -f deploy/manifests/nginx-gateway.yaml
+      kubectl apply -f deploy/manifests/service/nodeport.yaml
+      ```
+
 ### Run Examples
 
 To make sure NGF is running properly, try out the [examples](/examples).

diff --git a/go.mod b/go.mod
@@ -9,6 +9,7 @@ require (
 	github.com/go-logr/logr v1.3.0
 	github.com/google/go-cmp v0.6.0
 	github.com/maxbrunsfeld/counterfeiter/v6 v6.7.0
+	github.com/nginxinc/nginx-plus-go-client v0.10.0
 	github.com/nginxinc/nginx-prometheus-exporter v0.11.0
 	github.com/onsi/ginkgo/v2 v2.13.2
 	github.com/onsi/gomega v1.30.0
@@ -61,7 +62,6 @@ require (
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
-	github.com/nginxinc/nginx-plus-go-client v0.10.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_model v0.5.0 // indirect

diff --git a/internal/mode/static/config/config.go b/internal/mode/static/config/config.go
@@ -26,6 +26,8 @@ type Config struct {
 	LeaderElection LeaderElection
 	// UpdateGatewayClassStatus enables updating the status of the GatewayClass resource.
 	UpdateGatewayClassStatus bool
+	// Plus indicates whether NGINX Plus is being used.
+	Plus bool
 	// MetricsConfig specifies the metrics config.
 	MetricsConfig MetricsConfig
 	// HealthConfig specifies the health probe config.

diff --git a/internal/mode/static/manager.go b/internal/mode/static/manager.go
@@ -63,6 +63,7 @@ func init() {
 	utilruntime.Must(apiext.AddToScheme(scheme))
 }
 
+// nolint:gocyclo
 func StartManager(cfg config.Config) error {
 	options := manager.Options{
 		Scheme:  scheme,
@@ -147,7 +148,12 @@ func StartManager(cfg config.Config) error {
 
 	if cfg.MetricsConfig.Enabled {
 		constLabels := map[string]string{"class": cfg.GatewayClassName}
-		ngxCollector, err := collectors.NewNginxMetricsCollector(constLabels)
+		var ngxCollector prometheus.Collector
+		if cfg.Plus {
+			ngxCollector, err = collectors.NewNginxPlusMetricsCollector(constLabels)
+		} else {
+			ngxCollector, err = collectors.NewNginxMetricsCollector(constLabels)
+		}
 		if err != nil {
 			return fmt.Errorf("cannot create nginx metrics collector: %w", err)
 		}