chore: compute coverage (#391)

.build/go.mk

@@ -2,13 +2,14 @@
 
 GO_BUILD 			= CGO_ENABLED=0 go build -installsuffix cgo
 GO_LIST 			= env GO111MODULE=on go list
-TEST_CMD 			= go test -v -timeout 600s -coverprofile=profile.coverprofile
-TEST_C_CMD 			= go test -c -coverprofile=profile.coverprofile
+TEST_C_CMD 			= go test -c
 TEST_RUN_ARGS 		= -test.v -test.timeout 600s -test.coverprofile=profile.coverprofile
 CURRENT_PACKAGE 	= $(shell $(GO_LIST))
 TARGET_DIST 		:= ./dist
 TARGET_RESULTS 		:= ./results
 
+PKGS_COMMA_SEP = go list -f '{{ join .Deps "\n" }}{{"\n"}}{{.ImportPath}}' . | grep github.com/ovh/venom | grep -v vendor | tr '\n' ',' | sed 's/,$$//'
+
 ##### =====> Clean <===== #####
 
 mk_go_clean: # clean target directory
@@ -63,7 +64,9 @@ TESTPKGS_C = $(foreach PKG, $(TESTPKGS), $(TESTPKGS_C_FILE))
 
 $(TESTPKGS_C): #main_test.go
 	$(info *** compiling test $@)
-	@cd $(dir $@) && $(TEST_C_CMD) -o bin.test .
+	@cd $(dir $@); \
+	TEMP=`$(PKGS_COMMA_SEP)`; \
+	$(TEST_C_CMD) -coverpkg $$TEMP -o bin.test .;
 
 ##### =====> Running Tests <===== #####
 
@@ -94,14 +97,18 @@ GO_COBERTURA = ${GOPATH}/bin/gocover-cobertura
 $(GO_COBERTURA):
 	go get -u github.com/t-yuki/gocover-cobertura
 
+GO_XUTOOLS = ${GOPATH}/bin/xutools
+$(GO_XUTOOLS):
+	go get -u github.com/richardlt/xutools
+
 mk_go_test: $(GO_COV_MERGE) $(GO_COBERTURA) $(GOFILES) $(TARGET_RESULTS) $(TESTPKGS_RESULTS)# Run tests
 	@echo "Generating unit tests coverage..."
 	@$(GO_COV_MERGE) `find ./ -name "*.coverprofile"` > $(TARGET_RESULTS)/cover.out
 	@$(GO_COBERTURA) < $(TARGET_RESULTS)/cover.out > $(TARGET_RESULTS)/coverage.xml
 	@go tool cover -html=$(TARGET_RESULTS)/cover.out -o=$(TARGET_RESULTS)/cover.html
 	@NB=$$(grep "^FAIL" `find . -type f -name "tests.log"`|grep -v ':0'|wc -l); echo "tests failed $$NB" && exit $$NB
 
-mk_go_test-xunit: $(GO_GOJUNIT) $(TARGET_RESULTS) # Generate test with xunit report
+mk_go_test-xunit: $(GO_GOJUNIT) $(GO_XUTOOLS) $(TARGET_RESULTS) # Generate test with xunit report
 	@echo "Generating xUnit Report..."
 	@for TST in `find . -name "tests.log"`; do \
 		if [ -s $$TST ]; then \
@@ -134,38 +141,16 @@ mk_go_test-xunit: $(GO_GOJUNIT) $(TARGET_RESULTS) # Generate test with xunit rep
 			mv $$XML $(TARGET_RESULTS)/`echo $$XML | sed 's|./||' | sed 's|/|_|g' | sed 's|_tests.log||'`; \
 		fi; \
 	done; \
-	rm -f $(TARGET_RESULTS)/report; \
-	for XML in `find . -name "*.tests-results.xml"`; do \
-		if [ -s $$XML ]; then \
-			if grep -q 'testsuite' $$XML; then \
-				echo "Generating report: " $$XML; \
-				echo "`xmllint --xpath "//testsuite/@name" $$XML | sed 's/name=//' | sed 's/"//g'`" \
-				"`xmllint --xpath "//testsuite/@tests" $$XML | sed 's/tests=//' | sed 's/"//g'` Tests :" \
-				"`xmllint --xpath "//testsuite/@errors" $$XML 2>/dev/null | sed 's/errors=//' | sed 's/"//g'` Errors ;"\
-				"`xmllint --xpath "//testsuite/@failures" $$XML 2>/dev/null | sed 's/failures=//' | sed 's/"//g'` Failures;" \
-				"`xmllint --xpath "//testsuite/@skip" $$XML 2>/dev/null | sed 's/skip=//' | sed 's/"//g'` Skipped;" \
-				>> $(TARGET_RESULTS)/report; \
-			fi; \
-		fi; \
-	done; \
+	xutools pretty --show-failures $(TARGET_RESULTS)/*.xml > $(TARGET_RESULTS)/report; \
+	xutools sort-duration $(TARGET_RESULTS)/*.xml > $(TARGET_RESULTS)/duration; \
 	if [ -e $(TARGET_RESULTS)/report ]; then \
+		echo "Report:"; \
 		cat $(TARGET_RESULTS)/report; \
 	fi; \
-	echo "#########################"; \
-	for XML in `find . -name "*.tests-results.xml"`; do \
-		if [ -s $$XML ]; then \
-			if grep -q 'errors' $$XML && grep -q 'testsuite' $$XML; then \
-				if [ "`xmllint --xpath "//testsuite/@errors" $$XML | sed 's/errors=//' | sed 's/"//g'`" -gt "0" ]; then  \
-					echo "	$$XML : Tests failed";  \
-				fi; \
-			fi; \
-			if grep -q 'failures' $$XML && grep -q 'testsuite' $$XML $$XML; then \
-				if [ "`xmllint --xpath "//testsuite/@failures" $$XML | sed 's/failures=//' | sed 's/"//g'`" -gt "0" ]; then  \
-					echo "	$$XML : Tests failed";  \
-				fi; \
-			fi; \
-		fi; \
-	done; \
+	if [ -e $(TARGET_RESULTS)/duration ]; then \
+		echo "Max duration:"; \
+		cat $(TARGET_RESULTS)/duration; \
+	fi; \
 	if [ -e $(TARGET_RESULTS)/fail ]; then \
 		echo "#########################"; \
 		echo "ERROR: Test compilation failure"; \

cmd/venom/main_test.go

@@ -0,0 +1,13 @@
+// +build testbincover
+
+package main
+
+import (
+	"testing"
+
+	"github.com/confluentinc/bincover"
+)
+
+func TestBincoverRunMain(t *testing.T) {
+	bincover.RunTest(main)
+}

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/Shopify/sarama v1.27.2
 	github.com/alexbrainman/odbc v0.0.0-20200426075526-f0492dfa1575
 	github.com/antonfisher/nested-logrus-formatter v1.3.0
+	github.com/confluentinc/bincover v0.2.0
 	github.com/fatih/color v1.10.0
 	github.com/fsamin/go-dump v1.2.0
 	github.com/fullstorydev/grpcurl v1.7.0

go.sum

@@ -110,6 +110,8 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX
 github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ=
 github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
+github.com/confluentinc/bincover v0.2.0 h1:WSS3MqzwJbosCLMOuF3tJ0pMpALzBfrm80Tb+/3gbQs=
+github.com/confluentinc/bincover v0.2.0/go.mod h1:qeI1wx0RxdGTZtrJY0HVlgJ4NqC/X2Z+fHbvy87tgHE=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
@@ -823,8 +825,6 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201118182958-a01c418693c7 h1:Z991aAXPjz0tLnj74pVXW3eWJ5lHMIBvbRfMq4M2jHA=
-golang.org/x/sys v0.0.0-20201118182958-a01c418693c7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4 h1:EZ2mChiOa8udjfp6rRmswTbtZN/QzUQp4ptM4rnjHvc=
 golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

tests/.gitignore

@@ -1,2 +1,9 @@
 *.cid
-*.log
+*.log
+venom.test
+*.args.file
+*.cover.out
+*.error.out
+*.coverprofile
+*.out
+*.test.out

tests/Makefile

@@ -4,6 +4,9 @@ endef
 
 docker-network := venom-test-net
 
+COVER_FILES = $(shell find . -name "*.coverprofile")
+PKGS_COMMA_SEP = go list -f '{{ join .Deps "\n" }}{{"\n"}}{{.ImportPath}}' . | grep github.com/ovh/venom | grep -v vendor | tr '\n' ',' | sed 's/,$$//'
+
 venom-postgres.cid:
 	$(call docker_run,postgres,postgres,-p 15432:5432 -e POSTGRES_PASSWORD=venom -e POSTGRES_USER=venom -e POSTGRES_DB=venom)
 venom-mysql.cid:
@@ -41,8 +44,17 @@ start-test-stack: venom-postgres.cid venom-mysql.cid venom-redis.cid venom-imap.
 stop-test-stack:
 	@for f in `ls -1 *.cid`; do docker stop `cat $${f}`; docker rm `cat $${f}`; done; rm -f *.cid; docker network rm $(docker-network)
 
+build-test-binary:
+	cd ../cmd/venom; \
+	TEMP=`$(PKGS_COMMA_SEP)`; \
+	CGO_ENABLED=1 go test -coverpkg $$TEMP -c . -o ../../tests/venom.test -tags testbincover;
+
+build-test-binary-docker:
+	docker run -v `pwd`/..:/workspace golang:1.16 sh -c "cd /workspace/tests && make build-test-binary"	
+
 run-test: wait-for-kafka
-	@(cd ../cmd/venom; go build && VENOM_VAR_MY_ENVAR=foo ./venom run --stop-on-failure -vv --lib-dir=lib_custom --var='array_from_var=["biz","buz"]' --var-from-file ../../tests/kafka/testVariables.yml  --var-from-file ../../tests/vars/vars.yml ../../tests/*.yml ../../tests/assertions/*.yml && cp ./venom ../../tests/vars_override/venom && cd ../../tests/vars_override && VENOM_VAR_foo=from-env VENOM_VAR_FOO2=from-env2 VENOM_VAR_FOO3=from-env3 ./venom run --var foo='from-cmd-arg' --var='array_from_var=["biz","buz"]' mytc.yml)
+	VENOM_VAR_MY_ENVAR=foo ./venom_wrapper.sh run -vv --format=xml --lib-dir=./lib_custom --var='array_from_var=["biz","buz"]' --var-from-file ./kafka/testVariables.yml  --var-from-file ./vars/vars.yml ./*.yml ./assertions/*.yml;
+	(cd ./vars_override && VENOM_VAR_foo=from-env VENOM_VAR_FOO2=from-env2 VENOM_VAR_FOO3=from-env3 ../venom_wrapper.sh run -vv --var foo='from-cmd-arg' --var='array_from_var=["biz","buz"]' ./mytc.yml);
 
 .PHONY: wait-for-kafka
 wait-for-kafka:
@@ -51,4 +63,11 @@ wait-for-kafka:
 	echo "\n\033[0;32mdone\033[0m"
 
 clean:
-	@rm -f *.prof *.log *.dump.json
+	@rm -f *.prof *.log *.dump.json *.args.file *.error.out *.out *.test.out *.coverprofile
+
+merge-coverage:
+	@docker run -v `pwd`:/workspace golang:1.16 sh -c "\
+			go get -u github.com/wadey/gocovmerge && \
+			cd /workspace && \
+      gocovmerge $(COVER_FILES) > /workspace/venom.cover.out \
+		"

tests/venom_wrapper.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+DIR=`dirname $0`
+NANO_TIMESTAMP=$(date +%s%N)
+
+# Prepare args file from current args
+rm -f $DIR/$NANO_TIMESTAMP.args.file
+for v in "$@"
+do
+  echo $v >> $DIR/$NANO_TIMESTAMP.args.file
+done
+
+# Exec venom test binary, capture logs and exit code
+$DIR/venom.test -test.coverprofile=$DIR/$NANO_TIMESTAMP.coverprofile -test.run="^TestBincoverRunMain$" -args-file=$DIR/$NANO_TIMESTAMP.args.file 1> $DIR/$NANO_TIMESTAMP.out 2> $DIR/$NANO_TIMESTAMP.error.out
+EXI=$?
+
+# Print venom log on stdout and go test log in a dedicated file
+END_CTL_OUT=false
+while IFS="" read -r p || [ -n "$p" ]
+do
+  if [[ $p == *"START_BINCOVER_METADATA" && "$p" != "START_BINCOVER_METADATA" ]]; then
+    P=`printf '%s' "$p" | sed -e "s/START_BINCOVER_METADATA$//"`
+    printf '%s\n' "$P"
+    END_CTL_OUT=true
+    echo "START_BINCOVER_METADATA" >> $DIR/$NANO_TIMESTAMP.test.out
+  else
+    if [ "$p" == "START_BINCOVER_METADATA" ]; then
+      END_CTL_OUT=true
+    fi
+    if $END_CTL_OUT; then
+      echo "$p" >> $DIR/$NANO_TIMESTAMP.test.out
+    else
+      printf '%s\n' "$p"
+    fi
+  fi
+done < $DIR/$NANO_TIMESTAMP.out
+
+exit $EXI