From 87eda8313006b65347ead59960cf30a4881aa2c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Samin?= Date: Sun, 11 Sep 2022 21:12:49 +0200 Subject: [PATCH] fix: string unescaped quotes * fix: string unescaped quotes close #581 close #396 Signed-off-by: francois samin --- cmd/venom/run/cmd.go | 2 +- go.mod | 4 ++-- go.sum | 9 ++------- process_files.go | 2 +- process_testcase.go | 30 ++++++++++++++++++++---------- read_partial.go | 17 ++++++++++------- tests/Makefile | 4 ++-- tests/grpc/Dockerfile | 2 +- tests/grpc/go.mod | 2 +- tests/json_quote.yml | 7 +++++++ tests/json_quote/file.json | 1 + types_executor.go | 2 +- venom.go | 2 +- 13 files changed, 50 insertions(+), 34 deletions(-) create mode 100644 tests/json_quote.yml create mode 100644 tests/json_quote/file.json diff --git a/cmd/venom/run/cmd.go b/cmd/venom/run/cmd.go index 940bda47..f8c3af16 100644 --- a/cmd/venom/run/cmd.go +++ b/cmd/venom/run/cmd.go @@ -10,8 +10,8 @@ import ( "strconv" "strings" - "github.com/ghodss/yaml" "github.com/mitchellh/go-homedir" + "github.com/rockbears/yaml" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/pflag" diff --git a/go.mod b/go.mod index cbe77c55..444b6e4c 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/ovh/venom -go 1.18 +go 1.19 require ( github.com/Azure/go-amqp v0.13.7 @@ -13,7 +13,6 @@ require ( github.com/fsamin/go-dump v1.8.0 github.com/fullstorydev/grpcurl v1.8.6 github.com/garyburd/redigo v1.6.3 - github.com/ghodss/yaml v1.0.0 github.com/go-sql-driver/mysql v1.6.0 github.com/go-testfixtures/testfixtures/v3 v3.8.1 github.com/golang/protobuf v1.5.2 @@ -33,6 +32,7 @@ require ( github.com/ovh/cds/sdk/interpolate v0.0.0-20220801141832-203422993fef github.com/ovh/go-ovh v1.1.0 github.com/pkg/errors v0.9.1 + github.com/rockbears/yaml v0.1.0 github.com/rubenv/sql-migrate v1.1.2 github.com/sclevine/agouti v3.0.1-0.20180306165625-6ada53bb069e+incompatible github.com/sijms/go-ora v1.2.1 diff --git a/go.sum b/go.sum index e402c1df..e02138d2 100644 --- a/go.sum +++ b/go.sum @@ -140,12 +140,6 @@ github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoD github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= -github.com/fsamin/go-dump v0.0.0-20220811134020-951ca28b6c23 h1:8EBsFJ40eEsF8M4/0E/Ilxq6GV2PolHzkt/0Or+chJo= -github.com/fsamin/go-dump v0.0.0-20220811134020-951ca28b6c23/go.mod h1:/xw/mwt7LWFJlTwxTjwKMv1W3gmKJsKD8zHS9+/czFY= -github.com/fsamin/go-dump v0.0.0-20220811151532-e083d29933b0 h1:D7FFK1RN9JfVhB+LZQ8/Ap1xyzE8LTFDwIWiqAzRAvI= -github.com/fsamin/go-dump v0.0.0-20220811151532-e083d29933b0/go.mod h1:/xw/mwt7LWFJlTwxTjwKMv1W3gmKJsKD8zHS9+/czFY= -github.com/fsamin/go-dump v1.7.0 h1:12qptOwAyJ6vuzERrcMhREy1fT3bVmlYhi2CibhdAJM= -github.com/fsamin/go-dump v1.7.0/go.mod h1:/xw/mwt7LWFJlTwxTjwKMv1W3gmKJsKD8zHS9+/czFY= github.com/fsamin/go-dump v1.8.0 h1:jy1qExf8rXFoIc3y9hGuBgk0C+2GqETnusw/uouZlO8= github.com/fsamin/go-dump v1.8.0/go.mod h1:/xw/mwt7LWFJlTwxTjwKMv1W3gmKJsKD8zHS9+/czFY= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -155,7 +149,6 @@ github.com/fullstorydev/grpcurl v1.8.6 h1:WylAwnPauJIofYSHqqMTC1eEfUIzqzevXyogBx github.com/fullstorydev/grpcurl v1.8.6/go.mod h1:WhP7fRQdhxz2TkL97u+TCb505sxfH78W1usyoB3tepw= github.com/garyburd/redigo v1.6.3 h1:HCeeRluvAgMusMomi1+6Y5dmFOdYV/JzoRrrbFlkGIc= github.com/garyburd/redigo v1.6.3/go.mod h1:rTb6epsqigu3kYKBnaF028A7Tf/Aw5s0cqA47doKKqw= -github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -489,6 +482,8 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/rockbears/yaml v0.1.0 h1:W6w12VtEhYjmRrJVO37Tk4g7pBrGvOxnUjrvRZkiZ4k= +github.com/rockbears/yaml v0.1.0/go.mod h1:jRD5iDgWRO/FJnrKAPIXsccaEOc3IHS8W6EynrhQUwU= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= diff --git a/process_files.go b/process_files.go index 352e0e57..55653c1b 100644 --- a/process_files.go +++ b/process_files.go @@ -8,7 +8,7 @@ import ( "strings" "time" - "github.com/ghodss/yaml" + "github.com/rockbears/yaml" "github.com/mattn/go-zglob" diff --git a/process_testcase.go b/process_testcase.go index 05d0375b..5c2a9e7a 100644 --- a/process_testcase.go +++ b/process_testcase.go @@ -9,9 +9,9 @@ import ( "strings" "time" - "github.com/ghodss/yaml" "github.com/ovh/cds/sdk/interpolate" "github.com/pkg/errors" + "github.com/rockbears/yaml" ) var varRegEx = regexp.MustCompile("{{.*}}") @@ -218,7 +218,12 @@ func (v *Venom) runTestSteps(ctx context.Context, tc *TestCase, tsIn *TestStepRe // if the value is not escaped, it will be used as is, and the json sent to unmarshall will be incorrect. // This also avoids injections into the json structure of a step for i := range vars { - vars[i] = strings.ReplaceAll(vars[i], "\"", "\\\"") + if strings.Contains(vars[i], `"`) { + x := strconv.Quote(vars[i]) + x = strings.TrimPrefix(x, `"`) + x = strings.TrimSuffix(x, `"`) + vars[i] = x + } } var content string @@ -234,7 +239,11 @@ func (v *Venom) runTestSteps(ctx context.Context, tc *TestCase, tsIn *TestStepRe } } - Info(ctx, "Step #%d-%d content is: %q", stepNumber, rangedIndex, content) + if ranged.Enabled { + Info(ctx, "Step #%d-%d content is: %s", stepNumber, rangedIndex, content) + } else { + Info(ctx, "Step #%d content is: %s", stepNumber, content) + } data, err := yaml.Marshal(rawStep) if err != nil { @@ -243,22 +252,23 @@ func (v *Venom) runTestSteps(ctx context.Context, tc *TestCase, tsIn *TestStepRe } tsResult.Raw = data + var step TestStep + if err := yaml.Unmarshal([]byte(content), &step); err != nil { + tsResult.appendError(err) + Error(ctx, "unable to parse step #%d: %v", stepNumber, err) + return + } + data2, err := yaml.JSONToYAML([]byte(content)) if err != nil { tsResult.appendError(err) - Error(ctx, "unable to marshal interpolated: %v", err) + Error(ctx, "unable to marshal step #%d to json: %v", stepNumber, err) } tsResult.Interpolated = data2 tsResult.Number = stepNumber tsResult.RangedIndex = rangedIndex tsResult.InputVars = vars - var step TestStep - if err := yaml.Unmarshal([]byte(content), &step); err != nil { - tsResult.appendError(err) - Error(ctx, "unable to unmarshal step: %v", err) - return - } tc.testSteps = append(tc.testSteps, step) var e ExecutorRunner diff --git a/read_partial.go b/read_partial.go index ab26d70e..6a09e8e3 100644 --- a/read_partial.go +++ b/read_partial.go @@ -7,8 +7,8 @@ import ( "unicode" "unicode/utf8" - "github.com/ghodss/yaml" "github.com/pkg/errors" + "github.com/rockbears/yaml" ) func getUserExecutorInputYML(ctx context.Context, btesIn []byte) (H, error) { @@ -17,8 +17,10 @@ func getUserExecutorInputYML(ctx context.Context, btesIn []byte) (H, error) { var result = map[string]interface{}{} var tmpResult = map[string]interface{}{} - if err := yaml.Unmarshal([]byte(btes), &tmpResult); err != nil { - return nil, err + if len(btes) > 0 { + if err := yaml.Unmarshal([]byte(btes), &tmpResult); err != nil { + return nil, err + } } for k, v := range tmpResult { result[k] = v @@ -33,11 +35,12 @@ func getVarFromPartialYML(ctx context.Context, btesIn []byte) (H, error) { Vars H `yaml:"vars" json:"vars"` } var partial partialVars - if err := yaml.Unmarshal([]byte(btes), &partial); err != nil { - Error(context.Background(), "file content: %s", string(btes)) - return nil, errors.Wrapf(err, "error while unmarshal - see venom.log") + if len(btes) > 0 { + if err := yaml.Unmarshal([]byte(btes), &partial); err != nil { + Error(context.Background(), "file content: %s", string(btes)) + return nil, errors.Wrapf(err, "error while unmarshal - see venom.log") + } } - return partial.Vars, nil } diff --git a/tests/Makefile b/tests/Makefile index fd22efda..32198ab1 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -73,7 +73,7 @@ build-test-binary: CGO_ENABLED=1 go test -coverpkg $$TEMP -c . -o ../../tests/venom.test -ldflags "-X github.com/ovh/venom.IsTest=true" -tags testbincover; build-test-binary-docker: - docker run -v `pwd`/..:/workspace golang:1.17 sh -c "cd /workspace/tests && make build-test-binary" + docker run -v `pwd`/..:/workspace golang:1.19 sh -c "cd /workspace/tests && make build-test-binary" run-test: generate-venom-pki VENOM_VAR_MY_ENVAR=foo ./venom_wrapper.sh run \ @@ -90,7 +90,7 @@ clean: @rm -f *.prof *.html *.xml *.log *.dump.json *.args.file *.error.out *.out *.test.out *.coverprofile merge-coverage: - @docker run -v `pwd`:/workspace golang:1.17 sh -c "\ + @docker run -v `pwd`:/workspace golang:1.19 sh -c "\ go install github.com/wadey/gocovmerge@latest && \ cd /workspace && \ gocovmerge $(COVER_FILES) > /workspace/venom.cover.out \ diff --git a/tests/grpc/Dockerfile b/tests/grpc/Dockerfile index 41966e53..42728c0a 100644 --- a/tests/grpc/Dockerfile +++ b/tests/grpc/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.17.5-bullseye +FROM golang:1.19-bullseye WORKDIR /app diff --git a/tests/grpc/go.mod b/tests/grpc/go.mod index 1cbdf327..1e97f64d 100644 --- a/tests/grpc/go.mod +++ b/tests/grpc/go.mod @@ -1,6 +1,6 @@ module github.com/ovh/venom/grpc-example-server -go 1.17 +go 1.19 require ( google.golang.org/grpc v1.43.0 diff --git a/tests/json_quote.yml b/tests/json_quote.yml new file mode 100644 index 00000000..ec2d2d99 --- /dev/null +++ b/tests/json_quote.yml @@ -0,0 +1,7 @@ +name: "parsing" +testcases: + - name: "test" + steps: + - type: readfile + path: json_quote/file.json + - script: echo {{.result.content}} diff --git a/tests/json_quote/file.json b/tests/json_quote/file.json new file mode 100644 index 00000000..78041a23 --- /dev/null +++ b/tests/json_quote/file.json @@ -0,0 +1 @@ +{"foo": "{\"bar\":\"baz\"}"} \ No newline at end of file diff --git a/types_executor.go b/types_executor.go index 0aa8b02b..12482b2f 100644 --- a/types_executor.go +++ b/types_executor.go @@ -7,10 +7,10 @@ import ( "reflect" "strings" - "github.com/ghodss/yaml" "github.com/gosimple/slug" "github.com/ovh/cds/sdk/interpolate" "github.com/pkg/errors" + "github.com/rockbears/yaml" ) // Executor execute a testStep. diff --git a/venom.go b/venom.go index fb90d531..45664680 100644 --- a/venom.go +++ b/venom.go @@ -15,9 +15,9 @@ import ( "github.com/confluentinc/bincover" "github.com/fatih/color" - "github.com/ghodss/yaml" "github.com/ovh/cds/sdk/interpolate" "github.com/pkg/errors" + "github.com/rockbears/yaml" log "github.com/sirupsen/logrus" "github.com/spf13/cast" )