diff --git a/.build/go.mk b/.build/go.mk index b353f4fe..ad63e83b 100644 --- a/.build/go.mk +++ b/.build/go.mk @@ -2,13 +2,14 @@ GO_BUILD = CGO_ENABLED=0 go build -installsuffix cgo GO_LIST = env GO111MODULE=on go list -TEST_CMD = go test -v -timeout 600s -coverprofile=profile.coverprofile -TEST_C_CMD = go test -c -coverprofile=profile.coverprofile +TEST_C_CMD = go test -c TEST_RUN_ARGS = -test.v -test.timeout 600s -test.coverprofile=profile.coverprofile CURRENT_PACKAGE = $(shell $(GO_LIST)) TARGET_DIST := ./dist TARGET_RESULTS := ./results +PKGS_COMMA_SEP = go list -f '{{ join .Deps "\n" }}{{"\n"}}{{.ImportPath}}' . | grep github.com/ovh/venom | grep -v vendor | tr '\n' ',' | sed 's/,$$//' + ##### =====> Clean <===== ##### mk_go_clean: # clean target directory @@ -63,7 +64,9 @@ TESTPKGS_C = $(foreach PKG, $(TESTPKGS), $(TESTPKGS_C_FILE)) $(TESTPKGS_C): #main_test.go $(info *** compiling test $@) - @cd $(dir $@) && $(TEST_C_CMD) -o bin.test . + @cd $(dir $@); \ + TEMP=`$(PKGS_COMMA_SEP)`; \ + $(TEST_C_CMD) -coverpkg $$TEMP -o bin.test .; ##### =====> Running Tests <===== ##### @@ -94,6 +97,10 @@ GO_COBERTURA = ${GOPATH}/bin/gocover-cobertura $(GO_COBERTURA): go get -u github.com/t-yuki/gocover-cobertura +GO_XUTOOLS = ${GOPATH}/bin/xutools +$(GO_XUTOOLS): + go get -u github.com/richardlt/xutools + mk_go_test: $(GO_COV_MERGE) $(GO_COBERTURA) $(GOFILES) $(TARGET_RESULTS) $(TESTPKGS_RESULTS)# Run tests @echo "Generating unit tests coverage..." @$(GO_COV_MERGE) `find ./ -name "*.coverprofile"` > $(TARGET_RESULTS)/cover.out @@ -101,7 +108,7 @@ mk_go_test: $(GO_COV_MERGE) $(GO_COBERTURA) $(GOFILES) $(TARGET_RESULTS) $(TESTP @go tool cover -html=$(TARGET_RESULTS)/cover.out -o=$(TARGET_RESULTS)/cover.html @NB=$$(grep "^FAIL" `find . -type f -name "tests.log"`|grep -v ':0'|wc -l); echo "tests failed $$NB" && exit $$NB -mk_go_test-xunit: $(GO_GOJUNIT) $(TARGET_RESULTS) # Generate test with xunit report +mk_go_test-xunit: $(GO_GOJUNIT) $(GO_XUTOOLS) $(TARGET_RESULTS) # Generate test with xunit report @echo "Generating xUnit Report..." @for TST in `find . -name "tests.log"`; do \ if [ -s $$TST ]; then \ @@ -134,38 +141,16 @@ mk_go_test-xunit: $(GO_GOJUNIT) $(TARGET_RESULTS) # Generate test with xunit rep mv $$XML $(TARGET_RESULTS)/`echo $$XML | sed 's|./||' | sed 's|/|_|g' | sed 's|_tests.log||'`; \ fi; \ done; \ - rm -f $(TARGET_RESULTS)/report; \ - for XML in `find . -name "*.tests-results.xml"`; do \ - if [ -s $$XML ]; then \ - if grep -q 'testsuite' $$XML; then \ - echo "Generating report: " $$XML; \ - echo "`xmllint --xpath "//testsuite/@name" $$XML | sed 's/name=//' | sed 's/"//g'`" \ - "`xmllint --xpath "//testsuite/@tests" $$XML | sed 's/tests=//' | sed 's/"//g'` Tests :" \ - "`xmllint --xpath "//testsuite/@errors" $$XML 2>/dev/null | sed 's/errors=//' | sed 's/"//g'` Errors ;"\ - "`xmllint --xpath "//testsuite/@failures" $$XML 2>/dev/null | sed 's/failures=//' | sed 's/"//g'` Failures;" \ - "`xmllint --xpath "//testsuite/@skip" $$XML 2>/dev/null | sed 's/skip=//' | sed 's/"//g'` Skipped;" \ - >> $(TARGET_RESULTS)/report; \ - fi; \ - fi; \ - done; \ + xutools pretty --show-failures $(TARGET_RESULTS)/*.xml > $(TARGET_RESULTS)/report; \ + xutools sort-duration $(TARGET_RESULTS)/*.xml > $(TARGET_RESULTS)/duration; \ if [ -e $(TARGET_RESULTS)/report ]; then \ + echo "Report:"; \ cat $(TARGET_RESULTS)/report; \ fi; \ - echo "#########################"; \ - for XML in `find . -name "*.tests-results.xml"`; do \ - if [ -s $$XML ]; then \ - if grep -q 'errors' $$XML && grep -q 'testsuite' $$XML; then \ - if [ "`xmllint --xpath "//testsuite/@errors" $$XML | sed 's/errors=//' | sed 's/"//g'`" -gt "0" ]; then \ - echo " $$XML : Tests failed"; \ - fi; \ - fi; \ - if grep -q 'failures' $$XML && grep -q 'testsuite' $$XML $$XML; then \ - if [ "`xmllint --xpath "//testsuite/@failures" $$XML | sed 's/failures=//' | sed 's/"//g'`" -gt "0" ]; then \ - echo " $$XML : Tests failed"; \ - fi; \ - fi; \ - fi; \ - done; \ + if [ -e $(TARGET_RESULTS)/duration ]; then \ + echo "Max duration:"; \ + cat $(TARGET_RESULTS)/duration; \ + fi; \ if [ -e $(TARGET_RESULTS)/fail ]; then \ echo "#########################"; \ echo "ERROR: Test compilation failure"; \ diff --git a/cmd/venom/main_test.go b/cmd/venom/main_test.go new file mode 100644 index 00000000..62cb3260 --- /dev/null +++ b/cmd/venom/main_test.go @@ -0,0 +1,13 @@ +// +build testbincover + +package main + +import ( + "testing" + + "github.com/confluentinc/bincover" +) + +func TestBincoverRunMain(t *testing.T) { + bincover.RunTest(main) +} diff --git a/go.mod b/go.mod index 083147fa..f97f6e53 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/Shopify/sarama v1.27.2 github.com/alexbrainman/odbc v0.0.0-20200426075526-f0492dfa1575 github.com/antonfisher/nested-logrus-formatter v1.3.0 + github.com/confluentinc/bincover v0.2.0 github.com/fatih/color v1.10.0 github.com/fsamin/go-dump v1.2.0 github.com/fullstorydev/grpcurl v1.7.0 diff --git a/go.sum b/go.sum index ee43b065..d1f9fd6d 100644 --- a/go.sum +++ b/go.sum @@ -110,6 +110,8 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= +github.com/confluentinc/bincover v0.2.0 h1:WSS3MqzwJbosCLMOuF3tJ0pMpALzBfrm80Tb+/3gbQs= +github.com/confluentinc/bincover v0.2.0/go.mod h1:qeI1wx0RxdGTZtrJY0HVlgJ4NqC/X2Z+fHbvy87tgHE= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -823,8 +825,6 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201118182958-a01c418693c7 h1:Z991aAXPjz0tLnj74pVXW3eWJ5lHMIBvbRfMq4M2jHA= -golang.org/x/sys v0.0.0-20201118182958-a01c418693c7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4 h1:EZ2mChiOa8udjfp6rRmswTbtZN/QzUQp4ptM4rnjHvc= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/tests/.gitignore b/tests/.gitignore index 8156f146..41400310 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -1,2 +1,9 @@ *.cid -*.log \ No newline at end of file +*.log +venom.test +*.args.file +*.cover.out +*.error.out +*.coverprofile +*.out +*.test.out \ No newline at end of file diff --git a/tests/Makefile b/tests/Makefile index 8b6095cb..bc976b57 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -4,6 +4,9 @@ endef docker-network := venom-test-net +COVER_FILES = $(shell find . -name "*.coverprofile") +PKGS_COMMA_SEP = go list -f '{{ join .Deps "\n" }}{{"\n"}}{{.ImportPath}}' . | grep github.com/ovh/venom | grep -v vendor | tr '\n' ',' | sed 's/,$$//' + venom-postgres.cid: $(call docker_run,postgres,postgres,-p 15432:5432 -e POSTGRES_PASSWORD=venom -e POSTGRES_USER=venom -e POSTGRES_DB=venom) venom-mysql.cid: @@ -41,8 +44,17 @@ start-test-stack: venom-postgres.cid venom-mysql.cid venom-redis.cid venom-imap. stop-test-stack: @for f in `ls -1 *.cid`; do docker stop `cat $${f}`; docker rm `cat $${f}`; done; rm -f *.cid; docker network rm $(docker-network) +build-test-binary: + cd ../cmd/venom; \ + TEMP=`$(PKGS_COMMA_SEP)`; \ + CGO_ENABLED=1 go test -coverpkg $$TEMP -c . -o ../../tests/venom.test -tags testbincover; + +build-test-binary-docker: + docker run -v `pwd`/..:/workspace golang:1.16 sh -c "cd /workspace/tests && make build-test-binary" + run-test: wait-for-kafka - @(cd ../cmd/venom; go build && VENOM_VAR_MY_ENVAR=foo ./venom run --stop-on-failure -vv --lib-dir=lib_custom --var='array_from_var=["biz","buz"]' --var-from-file ../../tests/kafka/testVariables.yml --var-from-file ../../tests/vars/vars.yml ../../tests/*.yml ../../tests/assertions/*.yml && cp ./venom ../../tests/vars_override/venom && cd ../../tests/vars_override && VENOM_VAR_foo=from-env VENOM_VAR_FOO2=from-env2 VENOM_VAR_FOO3=from-env3 ./venom run --var foo='from-cmd-arg' --var='array_from_var=["biz","buz"]' mytc.yml) + VENOM_VAR_MY_ENVAR=foo ./venom_wrapper.sh run -vv --format=xml --lib-dir=./lib_custom --var='array_from_var=["biz","buz"]' --var-from-file ./kafka/testVariables.yml --var-from-file ./vars/vars.yml ./*.yml ./assertions/*.yml; + (cd ./vars_override && VENOM_VAR_foo=from-env VENOM_VAR_FOO2=from-env2 VENOM_VAR_FOO3=from-env3 ../venom_wrapper.sh run -vv --var foo='from-cmd-arg' --var='array_from_var=["biz","buz"]' ./mytc.yml); .PHONY: wait-for-kafka wait-for-kafka: @@ -51,4 +63,11 @@ wait-for-kafka: echo "\n\033[0;32mdone\033[0m" clean: - @rm -f *.prof *.log *.dump.json + @rm -f *.prof *.log *.dump.json *.args.file *.error.out *.out *.test.out *.coverprofile + +merge-coverage: + @docker run -v `pwd`:/workspace golang:1.16 sh -c "\ + go get -u github.com/wadey/gocovmerge && \ + cd /workspace && \ + gocovmerge $(COVER_FILES) > /workspace/venom.cover.out \ + " diff --git a/tests/venom_wrapper.sh b/tests/venom_wrapper.sh new file mode 100755 index 00000000..838ebd63 --- /dev/null +++ b/tests/venom_wrapper.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +DIR=`dirname $0` +NANO_TIMESTAMP=$(date +%s%N) + +# Prepare args file from current args +rm -f $DIR/$NANO_TIMESTAMP.args.file +for v in "$@" +do + echo $v >> $DIR/$NANO_TIMESTAMP.args.file +done + +# Exec venom test binary, capture logs and exit code +$DIR/venom.test -test.coverprofile=$DIR/$NANO_TIMESTAMP.coverprofile -test.run="^TestBincoverRunMain$" -args-file=$DIR/$NANO_TIMESTAMP.args.file 1> $DIR/$NANO_TIMESTAMP.out 2> $DIR/$NANO_TIMESTAMP.error.out +EXI=$? + +# Print venom log on stdout and go test log in a dedicated file +END_CTL_OUT=false +while IFS="" read -r p || [ -n "$p" ] +do + if [[ $p == *"START_BINCOVER_METADATA" && "$p" != "START_BINCOVER_METADATA" ]]; then + P=`printf '%s' "$p" | sed -e "s/START_BINCOVER_METADATA$//"` + printf '%s\n' "$P" + END_CTL_OUT=true + echo "START_BINCOVER_METADATA" >> $DIR/$NANO_TIMESTAMP.test.out + else + if [ "$p" == "START_BINCOVER_METADATA" ]; then + END_CTL_OUT=true + fi + if $END_CTL_OUT; then + echo "$p" >> $DIR/$NANO_TIMESTAMP.test.out + else + printf '%s\n' "$p" + fi + fi +done < $DIR/$NANO_TIMESTAMP.out + +exit $EXI