prepare automatic security scanning (#122)

* add todo for docker wait
* add makefile target to scan containers with trivy
* pin version of trivy in travis file
* add trivy cache to persistent storage of travis (commented since not part of ci for the moment)
* reorg travis file + remove goss as it is not used
* add a fixed version for the base image
* add file with tags to gitignore
* introduce tag-all target
* remove after_failure step
* add ignore file for trivy
* store the tag for the builder as well
* remove tag_file after completing scans
* replace manual build and publish commands with generated ones
* update kdav builder to resolve CVE-2019-3855
* ignore CVE-2019-3855 which is thrown in kdav build container
* exclude the build-webapp-demo from build-all

.gitignore

@@ -6,3 +6,4 @@ data/*
 .env
 *.env
 docker-compose.override.yml
+build.tags

.travis.yml

@@ -2,23 +2,20 @@ sudo: required
 language: bash
 env:
   global:
-  - GOSS_VERSION=0.3.6
-  - GOSS_PATH=/usr/local/bin/goss
-  - HADOLINT_VERSION=1.16.0
+  - HADOLINT_VERSION=1.16.3
   - DOCKER_COMPOSE_VERSION=1.23.2
+  - TRIVY_VERSION=0.1.1
   - secure: iSwQW1ytg9/ntqlF1nMzYcg0ouT3TifuAzauu//vWMiRfRthAi0bLuz3nBvlkQwtUk/iF3+smdOtwvjlmW7wWdwdf9tzpsyVKVYcS/+1MbxnGXE4OyNLkUJ7KASRk4otfsujMDNO95q/m04sOLJ721dsOWR6dv+5MNJ3LrushsbFfuStHmM1cNyUR6NuPy4g/x4oppv23rbSXU/qS7ULUsOTEUuTsmgvvKQRZiiOFaOgzeHCIEdrX6Dpsx6DPtYQ5az88q6CrkkTaw7GhP1qBXAGNX03NeHPd7YZvsgePoZJEJ/jTRsZVx9LxwkmnVTJDqthgqTGXTBJIvow3oICjKLf/DhURvkHaAJPu+Nxyvxo2xgYaa0Zbau5fmhEblyKU8Q9g+ZXsdjC5uy/vqJjg1rZD9BZjbKXRP9nb5VpxLdzcWE80XpEj7tHMfF4bN7LvIHZ81wINtZdZeFLVW53YzIO0NAoRCDk1SmR6N11T1uE8FrBzO80oETUMud2zYTx9U+J0m/qsNK+fOz2GtxwI3mlU0/bgVlcFE6865lOPuRwcTOhDwGqeWsLbBYsYXaJhqktn6XKiZ/BEeJLx6Z/CvyNXbzexn1i4wyVZAK7xxkhjxFPnWFU9WPan4ibkGLsS9sFsUTLVa4oBszkTO6q5NU7vIycdgJpfZlkdL2V0EA=
   - secure: kj/KcPck6RHSQdcN29+OoxSufHX8KgMXs/ekVUsgcXfWb8iwo0UbfGwyPf+oy1vvjO65e0xsdGHN6Vk++opJT1qaAMpIInfh3+otXmDrT4Uq0s+vBkyQ/EPNeTy6oWK28y5+IVrR3Nd4FMK8CQ4FKzqKAAOQDkusI1182tRL9wDPnCbUD92cNcTPh7aHccSflkBOzw0G6d0v3RFIseOdYMA4DN72YfUV6RHVgOz7PSPmZ9p9lza1Fdbd1fBYoqBapzm3tIWiaU20OkyYNorZzsT+afTTpfHIb5ku+emNCiKDORuX4XQHDiS+PtqDNJRL2WsOsudVf9ckd9wpTkDj5rFnVex7GtS4z47kLDahzNWMQs4gnpDVUi3jbGeU/62EXdiAmuWs0A2kUSPYZwAKVbfIDlp3tAy0dzGivnBfTdN/TYVRm0IDRJZZNp964Tu3rGLazbRCYpGTIYz4KlMRrIN4QJj8JMmvcaOidp/xQJL+MkZTNY653VFHYeu61XEUV3RkGkkhZL967w+VuhkULDppslKExsJzXXX6ITauLu6hqAj+fWrn0WDxn/Km+sx9aJaBNqg4egT8mX5+WeDdoV+3NyODjbYUaEPKSuUkW/Skm+VGlYeyc9apahTSDe1H/W2KUcramkMT17IdPqXTqvlo+HSR97IGoE37OWKdoVM=
   - secure: k5V2o5xIGGQ2vlWaCfWHAn68z7k/FSL8bXgow6/x0svxmsvDxJzRrpnM3xn681ogUEoQP1hQeHWeR0tg88RcDFmjzEObMjVd7Av289YIQ/W6hmFFb+SCa+TmAe49ybPLZA2UNygC/zqH5N6U5iMYsyPrChw4oUv9X9lfDJUz08crRVwffm/JwcEfV1tH722I2WUcEpxKYyqymK9CaO3e2UTXnPaASNOPuZ2v0T3D1lvla+XRNG+JJ6+BJjBRkzMMg584IaBIqGVf9tlImZkGfYmVWUVvBfpuHMSU9OC4CJXBRqy6K/nUlw5bDDsGFbLGA9Tg1qgLzAZsPCSMSCC2Gq0rLxuihudWEJ9e8dnRLIbt+Zxlqa2s7DQ2FTWyofQfR4GL6cD4uSoSh+k9ij6PeJMSEzplaO01Fyh87uRbcVBxwktIXeVuJsBG8uQ2wdWjQ41g4noDHzsV1duJ1nz9b6JRH7Vbp8bKXow3K+EtlFfa9GcD4I64oksbWH+hx+PBBf0qEdUzZnHmw2vEqJyjdlCoQ1k7pX6c9rxzNiKIb8Hsmhu1r7DCNYBYZIZ1pGhVBilxrr9QiU0hGpRsON0QOzTobz6TohW9w+LNgBMPMizLRFi3r14Nqel8GIWcQUP/RBTiXb8Lr+D9oq0oY1Up4QyfEq1SfkJ1yD4qzCOhb5I=
 services:
   - docker
 before_install:
-  - sudo curl -L https://github.com/aelsabbahy/goss/releases/download/v$GOSS_VERSION/goss-linux-amd64 -o /usr/local/bin/goss
-  - sudo curl -L https://github.com/aelsabbahy/goss/releases/download/v$GOSS_VERSION/dgoss -o /usr/local/bin/dgoss
   - sudo curl -L https://github.com/hadolint/hadolint/releases/download/v$HADOLINT_VERSION/hadolint-$(uname -s)-$(uname -m) -o /usr/local/bin/hadolint
   - sudo rm /usr/local/bin/docker-compose
-  - sudo curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` -o /usr/local/bin/docker-compose
-  - sudo chmod +rx /usr/local/bin/goss
-  - sudo chmod +rx /usr/local/bin/dgoss
+  - sudo curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` -o /usr/local/bin/docker-compose 
+  - wget https://github.com/knqyf263/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz
+  - sudo tar zxvf trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz -C /usr/local/bin trivy
   - sudo chmod +rx /usr/local/bin/hadolint
   - sudo chmod +rx /usr/local/bin/docker-compose
   - sudo apt update && sudo apt install -y expect
@@ -29,10 +26,11 @@ install:
   - make build-all
 script:
   - make test-ci
-after_failure:
-  - timeout 3s docker-compose logs
 deploy:
   - provider: script
     script: make publish
     on:
       branch: master
+#cache: # uncomment to not load/upload trivy files each time
+#  directories:
+#    - $HOME/.cache/trivy

.trivyignore

@@ -0,0 +1,8 @@
+# we're not using systemd
+CVE-2017-1000082
+
+# tar setuid issue
+CVE-2005-2541
+
+# libssh2 issue not relevant since not using ssh
+CVE-2019-3855

Makefile

@@ -22,20 +22,29 @@ RELEASE_KEY_DOWNLOAD := 0
 DOWNLOAD_COMMUNITY_PACKAGES := 1
 
 COMPOSE_FILE := docker-compose.yml
+TAG_FILE := build.tags
 -include .env
 export
 
 # convert lowercase componentname to uppercase
 COMPONENT = $(shell echo $(component) | tr a-z A-Z)
 
-.PHONY: all
+.PHONY: default
+default: help
+
+.PHONY: help
+help:
+	@eval $$(sed -r -n 's/^([a-zA-Z0-9_-]+):.*?## (.*)$$/printf "\\033[36m%-30s\\033[0m %s\\n" "\1" "\2" ;/; ta; b; :a p' $(MAKEFILE_LIST) | sort)
+
+.PHONY: build-all
 all: build-all
 
-build-all: build-base build-core build-kdav build-konnect build-kwmserver build-ldap build-ldap-demo build-meet build-php build-playground build-scheduler build-ssl build-utils build-web build-webapp build-zpush
+build-all:
+	make $(shell grep -o ^build-.*: Makefile | grep -Ev 'build-all|build-simple|build-builder|build-webapp-demo' | uniq | sed s/://g | xargs)
 
 .PHONY: build
 build: component ?= base
-build:
+build: ## Helper target to build a given image. Defaults to the "base" image.
 ifdef TRAVIS
 	@echo "fetching previous build to warm up build cache (only on travis)"
 	docker pull  $(docker_repo)/kopano_$(component) || true
@@ -62,7 +71,7 @@ endif
 
 .PHONY: build-simple
 build-simple: component ?= ssl
-build-simple:
+build-simple: ## Helper target to build a simplified image (no Kopano repo integration).
 ifdef TRAVIS
 	@echo "fetching previous build to warm up build cache (only on travis)"
 	docker pull  $(docker_repo)/kopano_$(component) || true
@@ -76,7 +85,7 @@ endif
 
 .PHONY: build-builder
 build-builder: component ?= kdav
-build-builder:
+build-builder: ## Helper target for images with a build stage.
 ifdef TRAVIS
 	@echo "fetching previous build to warm up build cache (only on travis)"
 	docker pull  $(docker_repo)/kopano_$(component):builder || true
@@ -99,8 +108,9 @@ endif
 	--cache-from $(docker_repo)/kopano_$(component) \
 	--cache-from $(docker_repo)/kopano_$(component):builder \
 	-t $(docker_repo)/kopano_$(component):builder $(component)/
+	@echo $(docker_repo)/kopano_$(component):builder >> $(TAG_FILE)
 
-build-base:
+build-base: ## Build new base image.
 	docker pull debian:stretch
 	component=base make build
 
@@ -151,19 +161,22 @@ build-web:
 build-webapp: build-php
 	component=webapp make build
 
-# replaces the actual kopano_webapp container with one that has login hints for demo.kopano.com
-build-webapp-demo:
+build-webapp-demo: ## Replaces the actual kopano_webapp container with one that has login hints for demo.kopano.com.
 	docker build \
 		-f webapp/Dockerfile.demo \
 		-t $(docker_repo)/kopano_webapp webapp/
 
 build-zpush:
 	component=zpush make build
 
+tag-all: build-all ## Helper target to create tags for all images.
+	make $(shell grep -o ^tag-.*: Makefile | grep -Ev 'tag-all|tag-container' | uniq | sed s/://g | xargs)
+
 tag-container: component ?= base
-tag-container:
+tag-container: ## Helper target to tag a given image. Defaults to the base image.
 	@echo 'create tag $($(component)_version)'
 	docker tag $(docker_repo)/kopano_$(component) $(docker_repo)/kopano_$(component):${$(component)_version}
+	@echo $(docker_repo)/kopano_$(component):${$(component)_version} >> $(TAG_FILE)
 	@echo 'create tag latest'
 	docker tag $(docker_repo)/kopano_$(component) $(docker_repo)/kopano_$(component):latest
 	git commit -m 'ci: committing changes for $(component)' -- $(component) || true
@@ -242,14 +255,15 @@ tag-zpush:
 	component=zpush make tag-container
 
 # Docker publish
-repo-login:
+repo-login: ## Login at hub.docker.com
 	@docker login -u $(docker_login) -p $(docker_pwd)
 
 .PHONY: publish
-publish: repo-login publish-base publish-core publish-kdav publish-konnect publish-kwmserver publish-ldap publish-ldap-demo publish-meet publish-php publish-playground publish-python publish-scheduler publish-ssl publish-utils publish-web publish-webapp publish-zpush
+publish: repo-login
+	make $(shell grep -o ^publish-.*: Makefile | grep -Ev 'publish-container' | uniq | sed s/://g | xargs)
 
 publish-container: component ?= base
-publish-container:
+publish-container: ## Helper target to push a given image to a registry. Defaults to the base image.
 	@echo 'publish latest to $(docker_repo)/kopano_$(component)'
 	docker push $(docker_repo)/kopano_$(component):${$(component)_version}
 	docker push $(docker_repo)/kopano_$(component):latest
@@ -295,7 +309,7 @@ publish-scheduler: build-scheduler tag-scheduler
 publish-ssl: build-ssl tag-ssl
 	component=scheduler make publish-container
 
-publish-utils: build-core build-utils tag-utils
+publish-utils: build-utils tag-utils
 	component=utils make publish-container
 
 publish-web: build-web tag-web
@@ -320,32 +334,35 @@ clean:
 	docker-compose -f $(COMPOSE_FILE) down -v --remove-orphans || true
 
 .PHONY: test
-test:
+test: ## Build and start new containers for testing (also deletes existing data volumes).
 	docker-compose -f $(COMPOSE_FILE) down -v --remove-orphans || true
 	make build-all
 	docker-compose -f $(COMPOSE_FILE) build
 	docker-compose -f $(COMPOSE_FILE) up -d
 	docker-compose -f $(COMPOSE_FILE) ps
 
-test-update-env:
+test-update-env: ## Recreate containers based on updated .env.
 	docker-compose -f $(COMPOSE_FILE) up -d
 
-test-ci:
+test-ci: ## Test if all containers start up
 	docker-compose -f $(COMPOSE_FILE) -f tests/test-container.yml build
 	docker-compose -f $(COMPOSE_FILE) -f tests/test-container.yml up -d
 	docker-compose -f $(COMPOSE_FILE) -f tests/test-container.yml ps
+	# TODO this just echos the exit code of the kopano_test container. if this is not 0 we should do something with it.
 	docker wait kopano_test_1
 	docker logs --tail 10 kopano_test_1
 	docker-compose -f $(COMPOSE_FILE) -f tests/test-container.yml stop 2>/dev/null
 	docker rm kopano_test_1
 
-test-quick:
+test-security: ## Scan containers with Trivy for known security risks (not part of CI workflow for now).
+	cat $(TAG_FILE) | xargs -I % sh -c 'trivy --exit-code 0 --severity HIGH --quiet --auto-refresh %'
+	cat $(TAG_FILE) | xargs -I % sh -c 'trivy --exit-code 1 --severity CRITICAL --quiet --auto-refresh %'
+	rm $(TAG_FILE)
+
+test-quick: ## Similar to test target, but does not delete existing data volumes and does not rebuild images.
 	docker-compose -f $(COMPOSE_FILE) stop || true
 	docker-compose -f $(COMPOSE_FILE) up -d
 	docker-compose -f $(COMPOSE_FILE) ps
 
 test-stop:
 	docker-compose -f $(COMPOSE_FILE) stop || true
-
-.PHONY: default
-default: build-all

base/Dockerfile

@@ -3,6 +3,8 @@ FROM debian:stretch
 LABEL [email protected] \
       version="2.0"
 
+ENV BASE_VERSION=1.0
+
 RUN mkdir -p /kopano/repo /kopano/data /kopano/helper /kopano/path
 WORKDIR /kopano/repo
 
@@ -46,6 +48,6 @@ ARG RELEASE_KEY_DOWNLOAD=0
 # get common utilities
 COPY create-kopano-repo.sh /kopano/helper/
 COPY kcconf.py /kopano/
-RUN date +%s > /kopano/buildversion
+RUN echo $BASE_VERSION > /kopano/buildversion
 
 SHELL [ "/bin/bash", "-c"]