Adds env var to set certbot acme server

this is required for test suite to use dns certbot request
without talking to live or staging letsencrypt servers or
production level dns providers. This is a backwards port
from the v3 branch and opens the door for a full certificate
cypress test

Author: jc21

backend/internal/certificate.js

@@ -20,6 +20,7 @@ const internalHost     = require('./host');
 
 
 const letsencryptStaging = config.useLetsencryptStaging();
+const letsencryptServer  = config.useLetsencryptServer();
 const letsencryptConfig  = '/etc/letsencrypt.ini';
 const certbotCommand     = 'certbot';
 
@@ -838,7 +839,8 @@ const internalCertificate = {
 			'--email "' + certificate.meta.letsencrypt_email + '" ' +
 			'--preferred-challenges "dns,http" ' +
 			'--domains "' + certificate.domain_names.join(',') + '" ' +
-			(letsencryptStaging ? '--staging' : '');
+			(letsencryptStaging ? '--staging' : '') +
+			(letsencryptServer !== null ? `--server '${letsencryptServer}'` : '');
 
 		logger.info('Command:', cmd);
 

backend/lib/config.js

@@ -180,5 +180,15 @@ module.exports = {
 	 */
 	useLetsencryptStaging: function () {
 		return !!process.env.LE_STAGING;
+	},
+
+	/**
+	 * @returns {string|null}
+	 */
+	useLetsencryptServer: function () {
+		if (process.env.LE_SERVER) {
+			return process.env.LE_SERVER;
+		}
+		return null;
 	}
 };

docker/Dockerfile

@@ -3,6 +3,8 @@
 
 # This file assumes that the frontend has been built using ./scripts/frontend-build
 
+FROM nginxproxymanager/testca AS testca
+FROM letsencrypt/pebble AS pebbleca
 FROM nginxproxymanager/nginx-full:certbot-node
 
 ARG TARGETPLATFORM
@@ -45,6 +47,8 @@ RUN yarn install \
 
 # add late to limit cache-busting by modifications
 COPY docker/rootfs /
+COPY --from=pebbleca /test/certs/pebble.minica.pem /etc/ssl/certs/pebble.minica.pem
+COPY --from=testca /home/step/certs/root_ca.crt /etc/ssl/certs/NginxProxyManager.crt
 
 # Remove frontend service not required for prod, dev nginx config as well
 RUN rm -rf /etc/s6-overlay/s6-rc.d/user/contents.d/frontend /etc/nginx/conf.d/dev.conf \

docker/dev/Dockerfile

@@ -1,7 +1,10 @@
+FROM nginxproxymanager/testca AS testca
+FROM letsencrypt/pebble AS pebbleca
 FROM nginxproxymanager/nginx-full:certbot-node
 LABEL maintainer="Jamie Curnow <[email protected]>"
 
-# See: https://github.com/just-containers/s6-overlay/blob/master/README.md
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
 ENV SUPPRESS_NO_CONFIG_WARNING=1 \
 	S6_BEHAVIOUR_IF_STAGE2_FAILS=1 \
 	S6_CMD_WAIT_FOR_SERVICES_MAXTIME=0 \
@@ -17,17 +20,20 @@ RUN echo "fs.file-max = 65535" > /etc/sysctl.conf \
 	&& rm -rf /var/lib/apt/lists/*
 
 # Task
-RUN cd /usr \
-	&& curl -sL https://taskfile.dev/install.sh | sh \
-	&& cd /root
+WORKDIR /usr
+RUN curl -sL https://taskfile.dev/install.sh | sh
+WORKDIR /root
 
 COPY rootfs /
-RUN rm -f /etc/nginx/conf.d/production.conf
-RUN chmod 644 /etc/logrotate.d/nginx-proxy-manager
-
-# s6 overlay
 COPY scripts/install-s6 /tmp/install-s6
-RUN /tmp/install-s6 "${TARGETPLATFORM}" && rm -f /tmp/install-s6
+RUN rm -f /etc/nginx/conf.d/production.conf \
+	&& chmod 644 /etc/logrotate.d/nginx-proxy-manager \
+	&& /tmp/install-s6 "${TARGETPLATFORM}" \
+	&& rm -f /tmp/install-s6
+
+# Certs for testing purposes
+COPY --from=pebbleca /test/certs/pebble.minica.pem /etc/ssl/certs/pebble.minica.pem
+COPY --from=testca /home/step/certs/root_ca.crt /etc/ssl/certs/NginxProxyManager.crt
 
 EXPOSE 80 81 443
 ENTRYPOINT [ "/init" ]

docker/docker-compose.ci.yml

@@ -9,6 +9,9 @@ services:
     environment:
       DEBUG: 'true'
       FORCE_COLOR: 1
+      # Required for DNS Certificate provisioning in CI
+      LE_SERVER: 'https://ca.internal/acme/acme/directory'
+      REQUESTS_CA_BUNDLE: '/etc/ssl/certs/NginxProxyManager.crt'
     volumes:
       - 'npm_data_ci:/data'
       - 'npm_le_ci:/etc/letsencrypt'

docker/docker-compose.dev.yml

@@ -33,12 +33,20 @@ services:
       DB_MYSQL_NAME: 'npm'
       # DB_SQLITE_FILE: "/data/database.sqlite"
       # DISABLE_IPV6: "true"
+      # Required for DNS Certificate provisioning testing:
+      LE_SERVER: 'https://ca.internal/acme/acme/directory'
+      REQUESTS_CA_BUNDLE: '/etc/ssl/certs/NginxProxyManager.crt'
     volumes:
       - npm_data:/data
       - le_data:/etc/letsencrypt
+      - './dev/resolv.conf:/etc/resolv.conf:ro'
       - ../backend:/app
       - ../frontend:/app/frontend
       - ../global:/app/global
+    healthcheck:
+      test: ["CMD", "/usr/bin/check-health"]
+      interval: 10s
+      timeout: 3s
     depends_on:
       - db
     working_dir: /app
@@ -58,6 +66,23 @@ services:
     volumes:
       - db_data:/var/lib/mysql
 
+  stepca:
+    image: jc21/testca
+    volumes:
+      - './dev/resolv.conf:/etc/resolv.conf:ro'
+      - '/etc/localtime:/etc/localtime:ro'
+    networks:
+      nginx_proxy_manager:
+        aliases:
+          - ca.internal
+
+  dnsrouter:
+    image: jc21/dnsrouter
+    volumes:
+      - ./dev/dnsrouter-config.json.tmp:/dnsrouter-config.json:ro
+    networks:
+      - nginx_proxy_manager
+
   swagger:
     image: swaggerapi/swagger-ui:latest
     container_name: npm_swagger
@@ -74,19 +99,80 @@ services:
     container_name: npm_squid
     volumes:
       - './dev/squid.conf:/etc/squid/squid.conf:ro'
+      - './dev/resolv.conf:/etc/resolv.conf:ro'
       - '/etc/localtime:/etc/localtime:ro'
     networks:
       - nginx_proxy_manager
     ports:
       - 8128:3128
 
+  pdns:
+    image: pschiffe/pdns-mysql
+    volumes:
+      - '/etc/localtime:/etc/localtime:ro'
+    environment:
+      PDNS_master: 'yes'
+      PDNS_api: 'yes'
+      PDNS_api_key: 'npm'
+      PDNS_webserver: 'yes'
+      PDNS_webserver_address: '0.0.0.0'
+      PDNS_webserver_password: 'npm'
+      PDNS_webserver-allow-from: '127.0.0.0/8,192.0.0.0/8,10.0.0.0/8,172.0.0.0/8'
+      PDNS_version_string: 'anonymous'
+      PDNS_default_ttl: 1500
+      PDNS_allow_axfr_ips: '127.0.0.0/8,192.0.0.0/8,10.0.0.0/8,172.0.0.0/8'
+      PDNS_gmysql_host: pdns-db
+      PDNS_gmysql_port: 3306
+      PDNS_gmysql_user: pdns
+      PDNS_gmysql_password: pdns
+      PDNS_gmysql_dbname: pdns
+    depends_on:
+      - pdns-db
+    networks:
+      nginx_proxy_manager:
+        aliases:
+          - ns1.pdns
+          - ns2.pdns
+
+  pdns-db:
+    image: mariadb
+    environment:
+      MYSQL_ROOT_PASSWORD: 'pdns'
+      MYSQL_DATABASE: 'pdns'
+      MYSQL_USER: 'pdns'
+      MYSQL_PASSWORD: 'pdns'
+    volumes:
+      - 'pdns_mysql:/var/lib/mysql'
+      - '/etc/localtime:/etc/localtime:ro'
+      - './dev/pdns-db.sql:/docker-entrypoint-initdb.d/01_init.sql:ro'
+    networks:
+      - nginx_proxy_manager
+
+  cypress:
+    image: "npm_dev_cypress"
+    build:
+      context: ../
+      dockerfile: test/cypress/Dockerfile
+    environment:
+      HTTP_PROXY: 'squid:3128'
+      HTTPS_PROXY: 'squid:3128'
+    volumes:
+      - '../test/results:/results'
+      - './dev/resolv.conf:/etc/resolv.conf:ro'
+      - '/etc/localtime:/etc/localtime:ro'
+    command: cypress run --browser chrome --config-file=cypress/config/ci.js
+    networks:
+      - nginx_proxy_manager
+
 volumes:
   npm_data:
     name: npm_core_data
   le_data:
     name: npm_le_data
   db_data:
     name: npm_db_data
+  pdns_mysql:
+    name: npm_pdns_mysql
 
 networks:
   nginx_proxy_manager:

scripts/.common.sh

@@ -15,3 +15,13 @@ COMPOSE_PROJECT_NAME="npmdev"
 COMPOSE_FILE="docker/docker-compose.dev.yml"
 
 export COMPOSE_FILE COMPOSE_PROJECT_NAME
+
+# $1: container_name
+get_container_ip () {
+	local container_name=$1
+	local container
+	local ip
+	container=$(docker-compose ps --all -q "${container_name}" | tail -n1)
+	ip=$(docker inspect --format='{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$container")
+	echo "$ip"
+}

scripts/cypress-dev

@@ -0,0 +1,13 @@
+#!/bin/bash -e
+
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+. "$DIR/.common.sh"
+
+# Ensure docker-compose exists
+if hash docker-compose 2>/dev/null; then
+	cd "${DIR}/.."
+	rm -rf "$DIR/../test/results"
+	docker-compose up --build cypress
+else
+	echo -e "${RED}❯ docker-compose command is not available${RESET}"
+fi

scripts/start-dev

@@ -7,8 +7,43 @@ DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 if hash docker-compose 2>/dev/null; then
 	cd "${DIR}/.."
 	echo -e "${BLUE}❯ ${CYAN}Starting Dev Stack ...${RESET}"
+	echo -e "${BLUE}❯ $(docker-compose config)${RESET}"
 
-	docker-compose up -d --remove-orphans --force-recreate --build
+	# Bring up a stack, in steps so we can inject IPs everywhere
+	docker-compose up -d pdns pdns-db
+	PDNS_IP=$(get_container_ip "pdns")
+	echo -e "${BLUE}❯ ${YELLOW}PDNS IP is ${PDNS_IP}${RESET}"
+
+	# adjust the dnsrouter config
+	LOCAL_DNSROUTER_CONFIG="$DIR/../docker/dev/dnsrouter-config.json"
+	rm -rf "$LOCAL_DNSROUTER_CONFIG.tmp"
+	# IMPORTANT: changes to dnsrouter-config.json will affect this line:
+	jq --arg a "$PDNS_IP" '.servers[0].upstreams[1].upstream = $a' "$LOCAL_DNSROUTER_CONFIG" > "$LOCAL_DNSROUTER_CONFIG.tmp"
+
+	docker-compose up -d dnsrouter
+	DNSROUTER_IP=$(get_container_ip "dnsrouter")
+	echo -e "${BLUE}❯ ${YELLOW}DNS Router IP is ${DNSROUTER_IP}"
+
+	if [ "${DNSROUTER_IP:-}" = "" ]; then
+		echo -e "${RED}❯ ERROR: DNS Router IP is not set${RESET}"
+		exit 1
+	fi
+
+	# mount the resolver
+	LOCAL_RESOLVE="$DIR/../docker/dev/resolv.conf"
+	rm -rf "${LOCAL_RESOLVE}"
+	printf "nameserver %s\noptions ndots:0" "${DNSROUTER_IP}" > "${LOCAL_RESOLVE}"
+
+	# bring up all remaining containers, except cypress!
+	docker-compose up -d --remove-orphans stepca squid
+	docker-compose pull db
+	docker-compose up -d --remove-orphans --pull=never fullstack
+	docker-compose up -d --remove-orphans swagger
+
+	# docker-compose up -d --remove-orphans --force-recreate --build
+
+	# wait for main container to be healthy
+	bash "$DIR/wait-healthy" "$(docker-compose ps --all -q fullstack)" 120
 
 	echo ""
 	echo -e "${CYAN}Admin UI:     http://127.0.0.1:3081${RESET}"