NOISSUE - Implementation of gRPC mTLS (absmach#1848)

Rebase with master and squash commits add: rootCA and clientCA in grpc server add: rootCA and client certificate in grpc client add: docker-compose for grpc-mtls and make target for mtls cert generation fix: typo in makefile fix: loadCertFile function in internal/clients/grpc/connect.go fix: env.parser test remove: commented lines add: make commands update: make commands and grpc clients fix: typo in makefile fix: loadCertFile function in internal/clients/grpc/connect.go remove: commented lines update: make commands and grpc clients update: make commands and docker-compose add: end of line fix: typos in makefile add: end of line fix: typos in makefile revert: grafana port in .env change: loadCertFile function change: certficate logic change: env name and update in compose file fix: makefile remove: tls env var change: ioutil to os for ReadFile change loadfile remove: test which is no needed fix: docker project name single docker-compose file single docker-compose file single docker-compose file fix space and new lines fix makefile add: GRPC_TLS varaible and imporved logging in gRPC Client fix mtls and tls env vars fix mtls and tls env vars grpc_mtls fix docker-compose fix makefile fix const name to go idomatic --------- Signed-off-by: Arvindh <[email protected]>
AryanGodara · Aug 16, 2023 · fde4350 · fde4350
1 parent 9dbe87f
commit fde4350
Show file tree

Hide file tree

Showing 32 changed files with 949 additions and 275 deletions.
diff --git a/Makefile b/Makefile
@@ -13,7 +13,14 @@ GOARCH ?= amd64
 VERSION ?= $(shell git describe --abbrev=0 --tags)
 COMMIT ?= $(shell git rev-parse HEAD)
 TIME ?= $(shell date +%F_%T)
-
+USER_REPO ?= $(shell git remote get-url origin | sed -e 's/.*\/\([^/]*\)\/\([^/]*\).*/\1_\2/' )
+BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD  2>/dev/null || git describe --tags --abbrev=0  2>/dev/null )
+empty:=
+space:= $(empty) $(empty)
+DOCKER_PROJECT ?= $(shell echo $(subst $(space),,$(USER_REPO)_$(BRANCH)) | tr -c -s '[:alnum:][=-=]' '_')
+DOCKER_COMPOSE_COMMANDS_SUPPORTED := up down config
+DEFAULT_DOCKER_COMPOSE_COMMAND  := up
+GRPC_MTLS_CERT_FILES_EXISTS = 0
 ifneq ($(MF_BROKER_TYPE),)
     MF_BROKER_TYPE := $(MF_BROKER_TYPE)
 else
@@ -54,9 +61,36 @@ define make_docker_dev
 		-f docker/Dockerfile.dev ./build
 endef
 
+ADDON_SERVICES = bootstrap cassandra-reader cassandra-writer certs \
+					influxdb-reader influxdb-writer lora-adapter mongodb-reader mongodb-writer \
+					opcua-adapter postgres-reader postgres-writer provision smpp-notifier smtp-notifier \
+					timescale-reader timescale-writer twins
+
+EXTERNAL_SERVICES = vault prometheus
+
+ifneq ($(filter run%,$(firstword $(MAKECMDGOALS))),)
+  temp_args := $(wordlist 2,$(words $(MAKECMDGOALS)),$(MAKECMDGOALS))
+  DOCKER_COMPOSE_COMMAND := $(if $(filter $(DOCKER_COMPOSE_COMMANDS_SUPPORTED),$(temp_args)), $(filter $(DOCKER_COMPOSE_COMMANDS_SUPPORTED),$(temp_args)), $(DEFAULT_DOCKER_COMPOSE_COMMAND))
+  $(eval $(DOCKER_COMPOSE_COMMAND):;@)
+endif
+
+ifneq ($(filter run_addons%,$(firstword $(MAKECMDGOALS))),)
+  temp_args := $(wordlist 2,$(words $(MAKECMDGOALS)),$(MAKECMDGOALS))
+  RUN_ADDON_ARGS :=  $(if $(filter-out $(DOCKER_COMPOSE_COMMANDS_SUPPORTED),$(temp_args)), $(filter-out $(DOCKER_COMPOSE_COMMANDS_SUPPORTED),$(temp_args)),$(ADDON_SERVICES) $(EXTERNAL_SERVICES))
+  $(eval $(RUN_ADDON_ARGS):;@)
+endif
+
+ifneq ("$(wildcard docker/ssl/certs/*-grpc-*)","")
+GRPC_MTLS_CERT_FILES_EXISTS = 1
+else
+GRPC_MTLS_CERT_FILES_EXISTS = 0
+endif
+
+FILTERED_SERVICES = $(filter-out $(RUN_ADDON_ARGS), $(SERVICES))
+
 all: $(SERVICES)
 
-.PHONY: all $(SERVICES) dockers dockers_dev latest release
+.PHONY: all $(SERVICES) dockers dockers_dev latest release run run_addons grpc_mtls_certs check_mtls check_certs
 
 clean:
 	rm -rf ${BUILD_DIR}
@@ -81,7 +115,7 @@ proto:
 	protoc -I. --go_out=. --go_opt=paths=source_relative --go-grpc_out=. --go-grpc_opt=paths=source_relative users/policies/*.proto
 	protoc -I. --go_out=. --go_opt=paths=source_relative --go-grpc_out=. --go-grpc_opt=paths=source_relative things/policies/*.proto
 
-$(SERVICES):
+$(FILTERED_SERVICES):
 	$(call compile_service,$(@))
 
 $(DOCKERS):
@@ -117,7 +151,45 @@ release:
 rundev:
 	cd scripts && ./run.sh
 
-run:
+grpc_mtls_certs:
+	$(MAKE) -C docker/ssl users_grpc_certs things_grpc_certs
+
+check_tls:
+ifeq ($(GRPC_TLS),true)
+	@unset GRPC_MTLS
+	@echo "gRPC TLS is enabled"
+	GRPC_MTLS=
+else
+	@unset GRPC_TLS
+	GRPC_TLS=
+endif
+
+check_mtls:
+ifeq ($(GRPC_MTLS),true)
+	@unset GRPC_TLS
+	@echo "gRPC MTLS is enabled"
+	GRPC_TLS=
+else
+	@unset GRPC_MTLS
+	GRPC_MTLS=
+endif
+
+check_certs: check_mtls check_tls
+ifeq ($(GRPC_MTLS_CERT_FILES_EXISTS),0)
+ifeq ($(filter true,$(GRPC_MTLS) $(GRPC_TLS)),true)
+ifeq ($(filter $(DEFAULT_DOCKER_COMPOSE_COMMAND),$(DOCKER_COMPOSE_COMMAND)),$(DEFAULT_DOCKER_COMPOSE_COMMAND))
+	$(MAKE) -C docker/ssl users_grpc_certs things_grpc_certs
+endif
+endif
+endif
+
+run: check_certs
 	sed -i "s,file: brokers/.*.yml,file: brokers/${MF_BROKER_TYPE}.yml," docker/docker-compose.yml
 	sed -i "s,MF_BROKER_URL=.*,MF_BROKER_URL=$$\{MF_$(shell echo ${MF_BROKER_TYPE} | tr 'a-z' 'A-Z')_URL\}," docker/.env
-	docker-compose -f docker/docker-compose.yml up
+	docker-compose -f docker/docker-compose.yml -p $(DOCKER_PROJECT) $(DOCKER_COMPOSE_COMMAND) $(args)
+
+run_addons: check_certs
+	$(foreach SVC,$(RUN_ADDON_ARGS),$(if $(filter $(SVC),$(ADDON_SERVICES) $(EXTERNAL_SERVICES)),,$(error Invalid Service $(SVC))))
+	@for SVC in $(RUN_ADDON_ARGS); do \
+		MF_ADDONS_CERTS_PATH_PREFIX="../."  docker-compose -f docker/addons/$$SVC/docker-compose.yml -p $(DOCKER_PROJECT) --env-file ./docker/.env $(DOCKER_COMPOSE_COMMAND) $(args) & \
+	done
diff --git a/cmd/things/main.go b/cmd/things/main.go
@@ -60,7 +60,7 @@ const (
 	envPrefixCache     = "MF_THINGS_CACHE_"
 	envPrefixES        = "MF_THINGS_ES_"
 	envPrefixHTTP      = "MF_THINGS_HTTP_"
-	envPrefixGRPC      = "MF_THINGS_GRPC_"
+	envPrefixGRPC      = "MF_THINGS_AUTH_GRPC_"
 	defDB              = "things"
 	defSvcHTTPPort     = "9000"
 	defSvcAuthGRPCPort = "7000"

diff --git a/docker/.env b/docker/.env
@@ -53,8 +53,9 @@ MF_USERS_HTTP_SERVER_CERT=
 MF_USERS_HTTP_SERVER_KEY=
 MF_USERS_GRPC_HOST=users
 MF_USERS_GRPC_PORT=7001
-MF_USERS_GRPC_SERVER_CERT=
-MF_USERS_GRPC_SERVER_KEY=
+MF_USERS_GRPC_SERVER_CERT=${GRPC_MTLS:+./ssl/certs/users-grpc-server.crt}${GRPC_TLS:+./ssl/certs/users-grpc-server.crt}
+MF_USERS_GRPC_SERVER_KEY=${GRPC_MTLS:+./ssl/certs/users-grpc-server.key}${GRPC_TLS:+./ssl/certs/users-grpc-server.key}
+MF_USERS_GRPC_SERVER_CA_CERTS=${GRPC_MTLS:+./ssl/certs/ca.crt}${GRPC_TLS:+./ssl/certs/ca.crt}
 MF_USERS_DB_HOST=users-db
 MF_USERS_DB_PORT=5432
 MF_USERS_DB_USER=mainflux
@@ -73,8 +74,9 @@ MF_USERS_INSTANCE_ID=
 #### Users Client Config
 MF_USERS_GRPC_URL=users:7001
 MF_USERS_GRPC_TIMEOUT=1s
-MF_USERS_GRPC_CLIENT_TLS=
-MF_USERS_GRPC_CA_CERT=
+MF_USERS_GRPC_CLIENT_CERT=${GRPC_MTLS:+./ssl/certs/users-grpc-client.crt}
+MF_USERS_GRPC_CLIENT_KEY=${GRPC_MTLS:+./ssl/certs/users-grpc-client.key}
+MF_USERS_GRPC_CLIENT_CA_CERTS=${GRPC_MTLS:+./ssl/certs/ca.crt}
 
 ### Email utility
 MF_EMAIL_HOST=smtp.mailtrap.io
@@ -92,12 +94,11 @@ MF_THINGS_STANDALONE_TOKEN=
 MF_THINGS_CACHE_KEY_DURATION=10m
 MF_THINGS_HTTP_HOST=things
 MF_THINGS_HTTP_PORT=9000
-MF_THINGS_HTTP_SERVER_CERT=
-MF_THINGS_HTTP_SERVER_KEY=
-MF_THINGS_GRPC_HOST=things
-MF_THINGS_GRPC_PORT=7000
-MF_THINGS_GRPC_SERVER_CERT=
-MF_THINGS_GRPC_SERVER_KEY=
+MF_THINGS_AUTH_GRPC_HOST=things
+MF_THINGS_AUTH_GRPC_PORT=7000
+MF_THINGS_AUTH_GRPC_SERVER_CERT=${GRPC_MTLS:+./ssl/certs/things-grpc-server.crt}${GRPC_TLS:+./ssl/certs/things-grpc-server.crt}
+MF_THINGS_AUTH_GRPC_SERVER_KEY=${GRPC_MTLS:+./ssl/certs/things-grpc-server.key}${GRPC_TLS:+./ssl/certs/things-grpc-server.key}
+MF_THINGS_AUTH_GRPC_SERVER_CA_CERTS=${GRPC_MTLS:+./ssl/certs/ca.crt}${GRPC_TLS:+./ssl/certs/ca.crt}
 MF_THINGS_ES_URL=es-redis:${MF_REDIS_TCP_PORT}
 MF_THINGS_ES_PASS=
 MF_THINGS_ES_DB=0
@@ -119,8 +120,9 @@ MF_THINGS_INSTANCE_ID=
 MF_THINGS_URL=http://things:9000
 MF_THINGS_AUTH_GRPC_URL=things:7000
 MF_THINGS_AUTH_GRPC_TIMEOUT=1s
-MF_THINGS_AUTH_GRPC_CLIENT_TLS=
-MF_THINGS_AUTH_GRPC_CA_CERTS=
+MF_THINGS_AUTH_GRPC_CLIENT_CERT=${GRPC_MTLS:+./ssl/certs/things-grpc-client.crt}
+MF_THINGS_AUTH_GRPC_CLIENT_KEY=${GRPC_MTLS:+./ssl/certs/things-grpc-client.key}
+MF_THINGS_AUTH_GRPC_CLIENT_CA_CERTS=${GRPC_MTLS:+./ssl/certs/ca.crt}
 
 ### HTTP
 MF_HTTP_ADAPTER_LOG_LEVEL=debug
@@ -259,7 +261,7 @@ MF_VAULT_CA_L=Belgrade
 
 ### LoRa
 MF_LORA_ADAPTER_LOG_LEVEL=debug
-MF_LORA_ADAPTER_MESSAGES_URL=tcp://lora.mqtt.mainflux.io:1883
+MF_LORA_ADAPTER_MESSAGES_URL=tcp://mainflux-mqtt:1883
 MF_LORA_ADAPTER_MESSAGES_TOPIC=application/+/device/+/event/up
 MF_LORA_ADAPTER_MESSAGES_USER=
 MF_LORA_ADAPTER_MESSAGES_PASS=
@@ -501,3 +503,4 @@ MF_GRAFANA_ADMIN_PASSWORD=mainflux
 
 # Docker image tag
 MF_RELEASE_TAG=latest
+
diff --git a/docker/addons/bootstrap/docker-compose.yml b/docker/addons/bootstrap/docker-compose.yml
@@ -9,8 +9,7 @@
 version: "3.7"
 
 networks:
-  docker_mainflux-base-net:
-    external: true
+  mainflux-base-net:
 
 volumes:
   mainflux-bootstrap-db-volume:
@@ -26,7 +25,7 @@ services:
       POSTGRES_PASSWORD: ${MF_BOOTSTRAP_DB_PASS}
       POSTGRES_DB: ${MF_BOOTSTRAP_DB_NAME}
     networks:
-      - docker_mainflux-base-net
+      - mainflux-base-net
     volumes:
       - mainflux-bootstrap-db-volume:/var/lib/postgresql/data
 
@@ -60,11 +59,28 @@ services:
       MF_BOOTSTRAP_DB_SSL_ROOT_CERT: ${MF_BOOTSTRAP_DB_SSL_ROOT_CERT}
       MF_AUTH_GRPC_URL: ${MF_USERS_GRPC_URL}
       MF_AUTH_GRPC_TIMEOUT: ${MF_USERS_GRPC_TIMEOUT}
-      MF_AUTH_GRPC_CLIENT_TLS: ${MF_USERS_GRPC_CLIENT_TLS}
-      MF_AUTH_GRPC_CA_CERT: ${MF_USERS_GRPC_CA_CERT}
+      MF_AUTH_GRPC_CLIENT_CERT: ${MF_USERS_GRPC_CLIENT_CERT:+/users-grpc-client.crt}
+      MF_AUTH_GRPC_CLIENT_KEY: ${MF_USERS_GRPC_CLIENT_KEY:+/users-grpc-client.key}
+      MF_AUTH_GRPC_SERVER_CA_CERTS: ${MF_USERS_GRPC_SERVER_CA_CERTS:+/users-grpc-server-ca.crt}
       MF_THINGS_URL: ${MF_THINGS_URL}
       MF_JAEGER_URL: ${MF_JAEGER_URL}
       MF_SEND_TELEMETRY: ${MF_SEND_TELEMETRY}
       MF_BOOTSTRAP_INSTANCE_ID: ${MF_BOOTSTRAP_INSTANCE_ID}
     networks:
-      - docker_mainflux-base-net
+      - mainflux-base-net
+    volumes:
+      - type: bind
+        source: ${MF_ADDONS_CERTS_PATH_PREFIX}${MF_USERS_GRPC_CLIENT_CERT:-./ssl/certs/dummy/client_cert}
+        target: /users-grpc-client${MF_USERS_GRPC_CLIENT_CERT:+.crt}
+        bind:
+          create_host_path: true
+      - type: bind
+        source: ${MF_ADDONS_CERTS_PATH_PREFIX}${MF_USERS_GRPC_CLIENT_KEY:-./ssl/certs/dummy/client_key}
+        target: /users-grpc-client${MF_USERS_GRPC_CLIENT_KEY:+.key}
+        bind:
+          create_host_path: true
+      - type: bind
+        source: ${MF_ADDONS_CERTS_PATH_PREFIX}${MF_USERS_GRPC_SERVER_CA_CERTS:-./ssl/certs/dummy/server_ca}
+        target: /users-grpc-server-ca${MF_USERS_GRPC_SERVER_CA_CERTS:+.crt}
+        bind:
+          create_host_path: true
diff --git a/docker/addons/cassandra-reader/docker-compose.yml b/docker/addons/cassandra-reader/docker-compose.yml
@@ -9,8 +9,7 @@
 version: "3.7"
 
 networks:
-  docker_mainflux-base-net:
-    external: true
+  mainflux-base-net:
 
 services:
   cassandra-reader:
@@ -30,18 +29,51 @@ services:
       MF_CASSANDRA_PASS: ${MF_CASSANDRA_PASS}
       MF_THINGS_AUTH_GRPC_URL: ${MF_THINGS_AUTH_GRPC_URL}
       MF_THINGS_AUTH_GRPC_TIMEOUT: ${MF_THINGS_AUTH_GRPC_TIMEOUT}
-      MF_THINGS_AUTH_GRPC_CLIENT_TLS: ${MF_THINGS_AUTH_GRPC_CLIENT_TLS}
-      MF_THINGS_AUTH_GRPC_CA_CERTS: ${MF_THINGS_AUTH_GRPC_CA_CERTS}
+      MF_THINGS_AUTH_GRPC_CLIENT_CERT: ${MF_THINGS_AUTH_GRPC_CLIENT_CERT:+/things-grpc-client.crt}
+      MF_THINGS_AUTH_GRPC_CLIENT_KEY: ${MF_THINGS_AUTH_GRPC_CLIENT_KEY:+/things-grpc-client.key}
+      MF_THINGS_AUTH_GRPC_SERVER_CA_CERTS: ${MF_THINGS_AUTH_GRPC_SERVER_CA_CERTS:+/things-grpc-server-ca.crt}
       MF_AUTH_GRPC_URL: ${MF_USERS_GRPC_URL}
       MF_AUTH_GRPC_TIMEOUT : ${MF_USERS_GRPC_TIMEOUT}
-      MF_AUTH_GRPC_CLIENT_TLS: ${MF_USERS_GRPC_CLIENT_TLS}
-      MF_AUTH_GRPC_CA_CERT: ${MF_USERS_GRPC_CA_CERT}
+      MF_AUTH_GRPC_CLIENT_CERT: ${MF_USERS_GRPC_CLIENT_CERT:+/users-grpc-client.crt}
+      MF_AUTH_GRPC_CLIENT_KEY: ${MF_USERS_GRPC_CLIENT_KEY:+/users-grpc-client.key}
+      MF_AUTH_GRPC_SERVER_CA_CERTS: ${MF_USERS_GRPC_SERVER_CA_CERTS:+/users-grpc-server-ca.crt}
       MF_JAEGER_URL: ${MF_JAEGER_URL}
       MF_SEND_TELEMETRY: ${MF_SEND_TELEMETRY}
       MF_CASSANDRA_READER_INSTANCE_ID: ${MF_CASSANDRA_READER_INSTANCE_ID}
     ports:
       - ${MF_CASSANDRA_READER_HTTP_PORT}:${MF_CASSANDRA_READER_HTTP_PORT}
     networks:
-      - docker_mainflux-base-net
+      - mainflux-base-net
     volumes:
       - ../../ssl/certs:/etc/ssl/certs
+      # Users gRPC client certificates
+      - type: bind
+        source: ${MF_ADDONS_CERTS_PATH_PREFIX}${MF_USERS_GRPC_CLIENT_CERT:-./ssl/certs/dummy/client_cert}
+        target: /users-grpc-client${MF_USERS_GRPC_CLIENT_CERT:+.crt}
+        bind:
+          create_host_path: true
+      - type: bind
+        source: ${MF_ADDONS_CERTS_PATH_PREFIX}${MF_USERS_GRPC_CLIENT_KEY:-./ssl/certs/dummy/client_key}
+        target: /users-grpc-client${MF_USERS_GRPC_CLIENT_KEY:+.key}
+        bind:
+          create_host_path: true
+      - type: bind
+        source: ${MF_ADDONS_CERTS_PATH_PREFIX}${MF_USERS_GRPC_SERVER_CA_CERTS:-./ssl/certs/dummy/server_ca}
+        target: /users-grpc-server-ca${MF_USERS_GRPC_SERVER_CA_CERTS:+.crt}
+        bind:
+          create_host_path: true
+      - type: bind
+        source: ${MF_ADDONS_CERTS_PATH_PREFIX}${MF_THINGS_AUTH_GRPC_CLIENT_CERT:-./ssl/certs/dummy/client_cert}
+        target: /things-grpc-client${MF_THINGS_AUTH_GRPC_CLIENT_CERT:+.crt}
+        bind:
+          create_host_path: true
+      - type: bind
+        source: ${MF_ADDONS_CERTS_PATH_PREFIX}${MF_THINGS_AUTH_GRPC_CLIENT_KEY:-./ssl/certs/dummy/client_key}
+        target: /things-grpc-client${MF_THINGS_AUTH_GRPC_CLIENT_KEY:+.key}
+        bind:
+          create_host_path: true
+      - type: bind
+        source: ${MF_ADDONS_CERTS_PATH_PREFIX}${MF_THINGS_AUTH_GRPC_SERVER_CA_CERTS:-./ssl/certs/dummy/server_ca}
+        target: /things-grpc-server-ca${MF_THINGS_AUTH_GRPC_SERVER_CA_CERTS:+.crt}
+        bind:
+          create_host_path: true
diff --git a/docker/addons/cassandra-writer/docker-compose.yml b/docker/addons/cassandra-writer/docker-compose.yml
@@ -9,8 +9,7 @@
 version: "3.7"
 
 networks:
-  docker_mainflux-base-net:
-    external: true
+  mainflux-base-net:
 
 volumes:
   mainflux-cassandra-volume:
@@ -21,12 +20,23 @@ services:
     container_name: mainflux-cassandra
     restart: on-failure
     networks:
-      - docker_mainflux-base-net
+      - mainflux-base-net
     ports:
       - ${MF_CASSANDRA_PORT}:${MF_CASSANDRA_PORT}
     volumes:
       - mainflux-cassandra-volume:/var/lib/cassandra
 
+  cassandra-init-keyspace:
+    image: cassandra:3.11.10
+    depends_on:
+      - cassandra
+    restart: on-failure
+    volumes:
+      - ./init.sh:/init.sh
+    entrypoint: ["/init.sh"]
+    networks:
+      - mainflux-base-net
+
   cassandra-writer:
     image: mainflux/cassandra-writer:${MF_RELEASE_TAG}
     container_name: mainflux-cassandra-writer
@@ -52,6 +62,6 @@ services:
     ports:
       - ${MF_CASSANDRA_WRITER_HTTP_PORT}:${MF_CASSANDRA_WRITER_HTTP_PORT}
     networks:
-      - docker_mainflux-base-net
+      - mainflux-base-net
     volumes:
       - ./config.toml:/config.toml
diff --git a/docker/addons/cassandra-writer/init.sh b/docker/addons/cassandra-writer/init.sh
@@ -1,3 +1,8 @@
-docker-compose -f docker/addons/cassandra-writer/docker-compose.yml --env-file docker/.env up -d
-sleep 20
-docker exec mainflux-cassandra cqlsh -e "CREATE KEYSPACE IF NOT EXISTS mainflux WITH replication = {'class':'SimpleStrategy','replication_factor':'1'};"
+#!/usr/bin/env bash
+until printf "" 2>>/dev/null >>/dev/tcp/mainflux-cassandra/9042; do
+    sleep 5;
+    echo "Waiting for cassandra...";
+done
+
+echo "Creating keyspace and table..."
+cqlsh mainflux-cassandra  -e "CREATE KEYSPACE IF NOT EXISTS mainflux WITH replication = {'class':'SimpleStrategy','replication_factor':'1'};"