feat: add image scanning and refactor Makefile (#335)

fix: add fixes for vuln
Azure · Jan 5, 2021 · 147a9bc · 147a9bc
1 parent 4a5f7b7
commit 147a9bc
Show file tree

Hide file tree

Showing 14 changed files with 201 additions and 149 deletions.
diff --git a/.pipelines/e2e-tests-template.yaml b/.pipelines/e2e-tests-template.yaml
@@ -31,7 +31,6 @@ jobs:
             version: 1.15
 
         - script: |
-            sudo add-apt-repository ppa:rmescandon/yq && sudo apt-get -o Acquire::Retries=30 update && sudo apt-get install yq
             sudo apt-get -o Acquire::Retries=30 update && sudo apt-get -o Acquire::Retries=30 install -y bats
             bats --version
 
@@ -57,7 +56,8 @@ jobs:
             identitySubscriptionID: $(SUBSCRIPTION_ID)
 
         - script: |
-            kubectl create ns dev            
+            export REGISTRY="${REGISTRY:-$(REGISTRY_NAME).azurecr.io/k8s/csi/secrets-store}"
+
             make e2e-test
           displayName: "Run e2e tests"
           env:
@@ -73,6 +73,9 @@ jobs:
             ${{ if eq(osType, 'windows') }}:
               TEST_WINDOWS: true
 
-        - template: templates/teardown.yaml
+        - template: templates/get-logs.yaml
+          parameters:
+            logUID: $(CLUSTER_TYPE)-$(CLUSTER_CONFIG)
 
+        - template: templates/teardown.yaml
         - template: templates/cleanup-images.yaml
diff --git a/.pipelines/load-tests-template.yaml b/.pipelines/load-tests-template.yaml
@@ -48,10 +48,10 @@ jobs:
         displayName: "Check cluster's health"
 
       - script: |
-          export REGISTRY="${REGISTRY:-$(REGISTRY_NAME).azurecr.io/secrets-store-csi-driver-provider-azure}"
+          export REGISTRY="${REGISTRY:-$(REGISTRY_NAME).azurecr.io/k8s/csi/secrets-store}"
 
           helm install csi manifest_staging/charts/csi-secrets-store-provider-azure --namespace $NAMESPACE --wait --timeout=10m -v=5 --debug \
-            --set image.repository=${REGISTRY} \
+            --set image.repository=${REGISTRY}/provider-azure \
             --set image.tag=${IMAGE_VERSION} \
             --set secrets-store-csi-driver.enableSecretRotation=true \
             --dependency-update
@@ -101,15 +101,9 @@ jobs:
         condition: succeededOrFailed()
         displayName: "Get stats"
 
-      - script: |
-          mkdir -p $(Build.ArtifactStagingDirectory)/logs
-          kubectl logs -n kube-system -l app.kubernetes.io/name=secrets-store-csi-driver -c secrets-store --tail=-1 > $(Build.ArtifactStagingDirectory)/logs/secrets-store-csi-driver.json
-          kubectl logs -n kube-system -l app.kubernetes.io/name=secrets-store-csi-driver -c node-driver-registrar --tail=-1 > $(Build.ArtifactStagingDirectory)/logs/node-driver-registrar.json
-          kubectl logs -n kube-system -l app.kubernetes.io/name=secrets-store-csi-driver -c liveness-probe --tail=-1 > $(Build.ArtifactStagingDirectory)/logs/liveness-probe.json
-          kubectl logs -n kube-system -l app.kubernetes.io/name=csi-secrets-store-provider-azure --tail=-1 > $(Build.ArtifactStagingDirectory)/logs/csi-secrets-store-provider-azure.json
-        workingDirectory: "$(perf-tests.repo.path)"
-        condition: succeededOrFailed()
-        displayName: "Get logs"
+      - template: templates/get-logs.yaml
+        parameters:
+          logUID: load-test
 
       - template: templates/publish-load-test-result.yaml
 

diff --git a/.pipelines/templates/build-images.yaml b/.pipelines/templates/build-images.yaml
@@ -18,9 +18,8 @@ steps:
       echo "Image version: ${IMAGE_VERSION}"
 
       export IMAGE_VERSION="${IMAGE_VERSION}"
-      export IMAGE_TAG="${IMAGE_VERSION}"
       echo "##vso[task.setvariable variable=IMAGE_VERSION]${IMAGE_VERSION}"
-      echo "##vso[task.setvariable variable=IMAGE_TAG]${IMAGE_VERSION}"
 
+      az acr login -n $(REGISTRY_NAME)
       make e2e-bootstrap
     displayName: "Build and push azure keyvault provider image"
diff --git a/.pipelines/templates/cleanup-images.yaml b/.pipelines/templates/cleanup-images.yaml
@@ -1,11 +1,17 @@
 steps:
   - script: |
-      # Allow errors in case the images do not exist
-      set +e
-      az account set -s=$(SUBSCRIPTION_ID)
-      az acr login -n $(REGISTRY_NAME)
-      az acr repository delete --name $(REGISTRY_NAME) --image secrets-store-csi-driver-provider-azure:${IMAGE_VERSION}-linux-amd64 -y || true
-      az acr repository delete --name $(REGISTRY_NAME) --image secrets-store-csi-driver-provider-azure:${IMAGE_VERSION}-windows-1809-amd64 -y || true
-      az acr repository delete --name $(REGISTRY_NAME) --image secrets-store-csi-driver-provider-azure:${IMAGE_VERSION} -y || true
+      # an empty tag will result in deleting the whole repo.
+      if [[ -n "${IMAGE_VERSION:-}" ]]; then
+        # Allow errors in case the images do not exist
+        set +e
+        az account set -s=$(SUBSCRIPTION_ID)
+        az acr login -n $(REGISTRY_NAME)
+        az acr repository delete --name $(REGISTRY_NAME) --image k8s/csi/secrets-store/provider-azure:${IMAGE_VERSION}-linux-amd64 -y || true
+        az acr repository delete --name $(REGISTRY_NAME) --image k8s/csi/secrets-store/provider-azure:${IMAGE_VERSION}-windows-1809-amd64 -y || true
+        az acr repository delete --name $(REGISTRY_NAME) --image k8s/csi/secrets-store/provider-azure:${IMAGE_VERSION}-windows-1903-amd64 -y || true
+        az acr repository delete --name $(REGISTRY_NAME) --image k8s/csi/secrets-store/provider-azure:${IMAGE_VERSION}-windows-1909-amd64 -y || true
+        az acr repository delete --name $(REGISTRY_NAME) --image k8s/csi/secrets-store/provider-azure:${IMAGE_VERSION}-windows-2004-amd64 -y || true
+        az acr repository delete --name $(REGISTRY_NAME) --image k8s/csi/secrets-store/provider-azure:${IMAGE_VERSION} -y || true
+      fi
     condition: always()
     displayName: "Cleanup"
diff --git a/.pipelines/templates/get-logs.yaml b/.pipelines/templates/get-logs.yaml
@@ -0,0 +1,19 @@
+parameters:
+- name: logUID
+  displayName: 'uid for log directory'
+  type: string
+
+steps:
+  - script: |
+      mkdir -p $(Build.ArtifactStagingDirectory)/logs/${{ parameters.logUID }}
+      kubectl logs -n kube-system -l app.kubernetes.io/name=secrets-store-csi-driver -c secrets-store --tail=-1 > $(Build.ArtifactStagingDirectory)/logs/${{ parameters.logUID }}/secrets-store-csi-driver.json
+      kubectl logs -n kube-system -l app.kubernetes.io/name=secrets-store-csi-driver -c node-driver-registrar --tail=-1 > $(Build.ArtifactStagingDirectory)/logs/${{ parameters.logUID }}/node-driver-registrar.json
+      kubectl logs -n kube-system -l app.kubernetes.io/name=secrets-store-csi-driver -c liveness-probe --tail=-1 > $(Build.ArtifactStagingDirectory)/logs/${{ parameters.logUID }}/liveness-probe.json
+      kubectl logs -n kube-system -l app.kubernetes.io/name=csi-secrets-store-provider-azure --tail=-1 > $(Build.ArtifactStagingDirectory)/logs/${{ parameters.logUID }}/csi-secrets-store-provider-azure.json
+
+      kubectl get pods -A -o wide
+    condition: always()
+    displayName: "Get logs"
+  - task: PublishBuildArtifacts@1
+    displayName: Save artifacts
+    condition: succeededOrFailed()
diff --git a/.pipelines/templates/scan-images.yaml b/.pipelines/templates/scan-images.yaml
@@ -0,0 +1,12 @@
+steps:
+  - script: |
+      export REGISTRY="e2e"
+      export IMAGE_VERSION="test"
+      OUTPUT_TYPE=docker  make container
+      wget https://github.com/aquasecurity/trivy/releases/download/v$(TRIVY_VERSION)/trivy_$(TRIVY_VERSION)_Linux-64bit.tar.gz
+      tar zxvf trivy_$(TRIVY_VERSION)_Linux-64bit.tar.gz
+      # show all vulnerabilities in the logs
+      ./trivy "${REGISTRY}/provider-azure:${IMAGE_VERSION}"
+      
+      ./trivy --exit-code 1 --ignore-unfixed --severity MEDIUM,HIGH,CRITICAL "${REGISTRY}/provider-azure:${IMAGE_VERSION}" || exit 1
+    displayName: "Scan images for vulnerability"
diff --git a/Dockerfile b/Dockerfile
@@ -1,6 +1,3 @@
-ARG TARGETARCH
-ARG TARGETOS
-
 FROM us.gcr.io/k8s-artifacts-prod/build-image/debian-base-amd64:buster-v1.2.0
 COPY ./_output/secrets-store-csi-driver-provider-azure /bin/
 RUN chmod a+x /bin/secrets-store-csi-driver-provider-azure
@@ -9,4 +6,7 @@ RUN chmod a+x /bin/secrets-store-csi-driver-provider-azure
 RUN apt-mark unhold apt && \
     clean-install ca-certificates cifs-utils mount apt libapt-pkg5.0 libp11-kit0 
 
+LABEL maintainers="aramase"
+LABEL description="Secrets Store CSI Driver Provider Azure"
+
 ENTRYPOINT ["/bin/secrets-store-csi-driver-provider-azure"]
diff --git a/Makefile b/Makefile
@@ -1,34 +1,56 @@
 -include secrets.env
 export $(shell test -f secrets.env && sed 's/=.*//' secrets.env)
 
+ORG_PATH=github.com/Azure
+PROJECT_NAME := secrets-store-csi-driver-provider-azure
+REPO_PATH="$(ORG_PATH)/$(PROJECT_NAME)"
+
 REGISTRY_NAME ?= upstreamk8sci
-REGISTRY ?= $(REGISTRY_NAME).azurecr.io
-DOCKER_IMAGE ?= $(REGISTRY)/public/k8s/csi/secrets-store/provider-azure
+REPO_PREFIX ?= k8s/csi/secrets-store
+REGISTRY ?= $(REGISTRY_NAME).azurecr.io/$(REPO_PREFIX)
 IMAGE_VERSION ?= 0.0.11
-IMAGE_NAME ?= secrets-store-csi-driver-provider-azure
+IMAGE_NAME ?= provider-azure
+IMAGE_TAG ?= $(REGISTRY)/$(IMAGE_NAME):$(IMAGE_VERSION)
 
+# build variables
 BUILD_DATE=$$(date +%Y-%m-%d-%H:%M)
 BUILD_COMMIT := $$(git rev-parse --short HEAD)
-GO_FILES=$(shell go list ./...)
-ORG_PATH=github.com/Azure
-REPO_PATH="$(ORG_PATH)/$(IMAGE_NAME)"
-E2E_IMAGE_TAG=$(REGISTRY)/$(IMAGE_NAME):$(IMAGE_VERSION)
-
 BUILD_DATE_VAR := $(REPO_PATH)/pkg/version.BuildDate
 BUILD_VERSION_VAR := $(REPO_PATH)/pkg/version.BuildVersion
 VCS_VAR := $(REPO_PATH)/pkg/version.Vcs
+LDFLAGS ?= "-X $(BUILD_DATE_VAR)=$(BUILD_DATE) -X $(BUILD_VERSION_VAR)=$(IMAGE_VERSION) -X $(VCS_VAR)=$(BUILD_COMMIT)"
 
+GO_FILES=$(shell go list ./...)
 ALL_DOCS := $(shell find . -name '*.md' -type f | sort)
 TOOLS_MOD_DIR := ./tools
 TOOLS_DIR := $(abspath ./.tools)
 
-LDFLAGS ?= "-X $(BUILD_DATE_VAR)=$(BUILD_DATE) -X $(BUILD_VERSION_VAR)=$(IMAGE_VERSION) -X $(VCS_VAR)=$(BUILD_COMMIT)"
-
-GO111MODULE ?= on
-export GO111MODULE
+GO111MODULE = on
 DOCKER_CLI_EXPERIMENTAL = enabled
 export GOPATH GOBIN GO111MODULE DOCKER_CLI_EXPERIMENTAL
 
+# Generate all combination of all OS, ARCH, and OSVERSIONS for iteration
+ALL_OS = linux windows
+ALL_ARCH.linux = amd64
+ALL_OS_ARCH.linux = $(foreach arch, ${ALL_ARCH.linux}, linux-$(arch))
+ALL_ARCH.windows = amd64
+ALL_OSVERSIONS.windows := 1809 1903 1909 2004
+ALL_OS_ARCH.windows = $(foreach arch, $(ALL_ARCH.windows), $(foreach osversion, ${ALL_OSVERSIONS.windows}, windows-${osversion}-${arch}))
+ALL_OS_ARCH = $(foreach os, $(ALL_OS), ${ALL_OS_ARCH.${os}})
+
+# The current context of image building
+# The architecture of the image
+ARCH ?= amd64
+# OS Version for the Windows images: 1809, 1903, 1909, 2004
+OSVERSION ?= 1809
+# Output type of docker buildx build
+OUTPUT_TYPE ?= registry
+BUILDKIT_VERSION ?= v0.8.1
+
+# E2E test variables
+KIND_VERSION ?= 0.8.1
+KIND_K8S_VERSION ?= 1.18.2
+
 $(TOOLS_DIR)/golangci-lint: $(TOOLS_MOD_DIR)/go.mod $(TOOLS_MOD_DIR)/go.sum $(TOOLS_MOD_DIR)/tools.go
 	cd $(TOOLS_MOD_DIR) && \
 	go build -o $(TOOLS_DIR)/golangci-lint github.com/golangci/golangci-lint/cmd/golangci-lint
@@ -42,71 +64,90 @@ lint: $(TOOLS_DIR)/golangci-lint $(TOOLS_DIR)/misspell
 	$(TOOLS_DIR)/golangci-lint run --timeout=5m -v
 	$(TOOLS_DIR)/misspell $(ALL_DOCS)
 
+.PHONY: unit-test
+unit-test:
+	go test $(GO_FILES) -v
+
 .PHONY: build
-build: setup
-	@echo "Building..."
-	$Q GOOS=linux CGO_ENABLED=0 go build -ldflags ${LDFLAGS} -o _output/secrets-store-csi-driver-provider-azure ./cmd/
+build:
+	CGO_ENABLED=0 GOOS=linux go build -a -ldflags ${LDFLAGS} -o _output/secrets-store-csi-driver-provider-azure ./cmd/
 
 .PHONY: build-windows
 build-windows:
-	@echo "Building windows binary..."
 	CGO_ENABLED=0 GOOS=windows go build -a -ldflags ${LDFLAGS} -o _output/secrets-store-csi-driver-provider-azure.exe ./cmd/
 
-image:
-	@echo "Building docker image..."
-	docker buildx build --no-cache -t $(DOCKER_IMAGE):$(IMAGE_VERSION) -f Dockerfile --platform="linux/amd64" --output "type=docker,push=false" .
-
-.PHONY: build-container-windows
-build-container-windows:
-	@echo "Building windows docker image..."
-	docker buildx build --no-cache -t $(DOCKER_IMAGE):$(IMAGE_VERSION) -f windows.Dockerfile --platform="windows/amd64" --output "type=docker,push=false" .
-
-push: image
-	docker push $(DOCKER_IMAGE):$(IMAGE_VERSION)
-
-setup: clean
-	@echo "Setup..."
-	$Q go env
-
+.PHONY: build-darwin
+build-darwin:
+	CGO_ENABLED=0 GOOS=darwin go build -a -ldflags ${LDFLAGS} -o _output/secrets-store-csi-driver-provider-azure.exe ./cmd/
+
+.PHONY: container
+container: build
+	docker build --no-cache -t $(IMAGE_TAG) -f Dockerfile .
+
+.PHONY: container-linux
+container-linux: docker-buildx-builder
+	docker buildx build --no-cache --output=type=$(OUTPUT_TYPE) --platform="linux/$(ARCH)" \
+			-t $(IMAGE_TAG)-linux-$(ARCH) -f Dockerfile .
+
+.PHONY: container-windows
+container-windows: docker-buildx-builder
+	docker buildx build --no-cache --output=type=$(OUTPUT_TYPE) --platform="windows/amd64" \
+	 		-t $(IMAGE_TAG)-windows-$(OSVERSION)-$(ARCH) --build-arg OSVERSION=$(OSVERSION) -f windows.Dockerfile .
+
+.PHONY: docker-buildx-builder
+docker-buildx-builder:
+	@if ! DOCKER_CLI_EXPERIMENTAL=enabled docker buildx ls | grep -q container-builder; then\
+		DOCKER_CLI_EXPERIMENTAL=enabled docker buildx create --name container-builder --use --driver-opt image=moby/buildkit:$(BUILDKIT_VERSION);\
+	fi
+
+.PHONY: container-all
+container-all: build build-windows
+	$(MAKE) container-linux
+	for osversion in $(ALL_OSVERSIONS.windows); do \
+		OSVERSION=$${osversion} $(MAKE) container-windows; \
+	done
+
+.PHONY: push-manifest
+push-manifest:
+	docker manifest create --amend $(IMAGE_TAG) $(foreach osarch, $(ALL_OS_ARCH), $(IMAGE_TAG)-${osarch})
+	# add "os.version" field to windows images (based on https://github.com/kubernetes/kubernetes/blob/master/build/pause/Makefile)
+	set -x; \
+	registry_prefix=$(shell (echo ${REGISTRY} | grep -Eq ".*\/.*") && echo "" || echo "docker.io/"); \
+	manifest_image_folder=`echo "$${registry_prefix}${IMAGE_TAG}" | sed "s|/|_|g" | sed "s/:/-/"`; \
+	for arch in $(ALL_ARCH.windows); do \
+		for osversion in $(ALL_OSVERSIONS.windows); do \
+			BASEIMAGE=mcr.microsoft.com/windows/nanoserver:$${osversion}; \
+			full_version=`docker manifest inspect $${BASEIMAGE} | jq -r '.manifests[0].platform["os.version"]'`; \
+			sed -i -r "s/(\"os\"\:\"windows\")/\0,\"os.version\":\"$${full_version}\"/" "${HOME}/.docker/manifests/$${manifest_image_folder}/$${manifest_image_folder}-windows-$${osversion}-$${arch}"; \
+		done; \
+	done
+	docker manifest push --purge $(IMAGE_TAG)
+	docker manifest inspect $(IMAGE_TAG)
+
+.PHONY: clean
 clean:
+	go clean -r -x
 	-rm -rf _output
 
 .PHONY: mod
 mod:
 	@go mod tidy
 
-.PHONY: unit-test
-unit-test:
-	go test $(GO_FILES) -v
-
-KIND_VERSION ?= 0.6.0
-KIND_K8S_VERSION ?= 1.18.2
-
 .PHONY: e2e-bootstrap
 e2e-bootstrap: install-helm
 ifdef CI_KIND_CLUSTER
 		curl -LO https://storage.googleapis.com/kubernetes-release/release/v${KIND_K8S_VERSION}/bin/linux/amd64/kubectl && chmod +x ./kubectl && sudo mv kubectl /usr/local/bin/
 		make setup-kind
 endif
-	docker pull $(E2E_IMAGE_TAG) || make e2e-container
+	docker pull $(IMAGE_TAG) || make e2e-container
 
 .PHONY: e2e-container
-e2e-container: build build-windows
-	docker buildx rm container-builder || true
-	# only moby/buildkit:foreign-mediatype works on building Windows image now
-	# https://github.com/moby/buildkit/pull/1879
-	# Github issue: https://github.com/moby/buildkit/issues/1877
-	docker buildx create --use --name=container-builder --driver-opt image=moby/buildkit:v0.7.2
+e2e-container:
 ifdef CI_KIND_CLUSTER
-		DOCKER_IMAGE=$(REGISTRY)/$(IMAGE_NAME) make image
-		kind load docker-image --name kind $(REGISTRY)/$(IMAGE_NAME):$(IMAGE_VERSION)
+	$(MAKE) build container
+	kind load docker-image --name kind $(IMAGE_TAG)
 else
-		az acr login --name $(REGISTRY_NAME)
-		docker buildx build --no-cache -t $(E2E_IMAGE_TAG)-linux-amd64 -f Dockerfile --platform="linux/amd64" --push .
-		docker buildx build --no-cache -t $(E2E_IMAGE_TAG)-windows-1809-amd64 -f windows.Dockerfile --platform="windows/amd64" --push .
-		docker manifest create $(E2E_IMAGE_TAG) $(E2E_IMAGE_TAG)-linux-amd64 $(E2E_IMAGE_TAG)-windows-1809-amd64
-		docker manifest inspect $(E2E_IMAGE_TAG)
-		docker manifest push --purge $(E2E_IMAGE_TAG)
+	$(MAKE) container-all push-manifest
 endif
 
 .PHONY: e2e-test
@@ -116,19 +157,20 @@ e2e-test:
 .PHONY: setup-kind
 setup-kind:
 	curl -L https://github.com/kubernetes-sigs/kind/releases/download/v${KIND_VERSION}/kind-linux-amd64 --output kind && chmod +x kind && sudo mv kind /usr/local/bin/
-	kind create cluster --image kindest/node:v${KIND_K8S_VERSION} --config test/kind-config.yaml
+	# Check for existing kind cluster
+	if [ $$(kind get clusters) ]; then kind delete cluster; fi
+	# using kind config to create cluster for testing custom cloud environments
+	TERM=dumb kind create cluster --image kindest/node:v${KIND_K8S_VERSION} --config test/kind-config.yaml
 
 .PHONY: install-helm
 install-helm:
-	curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash
+	helm version --short | grep -q v3 || (curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash)
 
 .PHONY: e2e-local-bootstrap
 e2e-local-bootstrap: build
 	kind create cluster --image kindest/node:v${KIND_K8S_VERSION} --config test/kind-config.yaml
 	make image
-	kind load docker-image --name kind $(DOCKER_IMAGE):$(IMAGE_VERSION)
-	# Create Dev namespace for local e2e-testing
-	kubectl create ns dev
+	kind load docker-image --name kind $(IMAGE_TAG)
 
 .PHONY: e2e-kind-cleanup
 e2e-kind-cleanup: