From 0ef6ec9b3572ab279b333d15069eb446b47d7658 Mon Sep 17 00:00:00 2001 From: Justin King Date: Mon, 26 Aug 2024 20:26:09 -0700 Subject: [PATCH] Implement native conformance test runner (#1001) * Implement native conformance test runner Signed-off-by: Justin King --- WORKSPACE | 13 +- conformance/BUILD.bazel | 98 +++++++----- conformance/conformance_test.bzl | 58 +++++++ conformance/conformance_test.go | 265 +++++++++++++++++++++++++++++++ conformance/conformance_test.sh | 3 + conformance/go.mod | 20 +++ conformance/go.sum | 32 ++++ conformance/start.sh | 14 -- conformance/zip.sh | 69 -------- 9 files changed, 447 insertions(+), 125 deletions(-) create mode 100644 conformance/conformance_test.bzl create mode 100644 conformance/conformance_test.go create mode 100755 conformance/conformance_test.sh create mode 100644 conformance/go.mod create mode 100644 conformance/go.sum delete mode 100755 conformance/start.sh delete mode 100755 conformance/zip.sh diff --git a/WORKSPACE b/WORKSPACE index da3c8bd9..24f5e9e9 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -4,10 +4,10 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") http_archive( name = "io_bazel_rules_go", - sha256 = "099a9fb96a376ccbbb7d291ed4ecbdfd42f6bc822ab77ae6f1b5cb9e914e94fa", + sha256 = "56d8c5a5c91e1af73eca71a6fab2ced959b67c86d12ba37feedb0a2dfea441a6", urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.35.0/rules_go-v0.35.0.zip", - "https://github.com/bazelbuild/rules_go/releases/download/v0.35.0/rules_go-v0.35.0.zip", + "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.37.0/rules_go-v0.37.0.zip", + "https://github.com/bazelbuild/rules_go/releases/download/v0.37.0/rules_go-v0.37.0.zip", ], ) @@ -155,6 +155,13 @@ go_repository( version = "v0.0.0-20220722155223-a9213eeb770e", ) +go_repository( + name = "com_github_google_go_cmp", + importpath = "github.com/google/go-cmp", + sum = "h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=", + version = "v0.5.9", +) + # Run the dependencies at the end. These will silently try to import some # of the above repositories but at different versions, so ours must come first. go_rules_dependencies() diff --git a/conformance/BUILD.bazel b/conformance/BUILD.bazel index e1fd8b9b..52deaf52 100644 --- a/conformance/BUILD.bazel +++ b/conformance/BUILD.bazel @@ -1,4 +1,11 @@ -ALL_TESTS = [ +load("@io_bazel_rules_go//go:def.bzl", "go_test") +load("//conformance:conformance_test.bzl", "conformance_test") + +package( + licenses = ["notice"], # Apache 2.0 +) + +_ALL_TESTS = [ "@com_google_cel_spec//tests/simple:testdata/basic.textproto", "@com_google_cel_spec//tests/simple:testdata/comparisons.textproto", "@com_google_cel_spec//tests/simple:testdata/conversions.textproto", @@ -24,47 +31,60 @@ ALL_TESTS = [ "@com_google_cel_spec//tests/simple:testdata/wrappers.textproto", ] -sh_test( - name = "ct", - srcs = ["@com_google_cel_spec//tests:conftest.sh"], - args = [ - "$(location @com_google_cel_spec//tests/simple:simple_test)", - "--server=$(location //server/main:cel_server)", - # Tests that need to be removed as the spec has changed - "--skip_test=comparisons/eq_literal/eq_mixed_types_error,eq_list_elem_mixed_types_error,eq_map_value_mixed_types_error;ne_literal/ne_mixed_types_error", - "--skip_test=comparisons/in_list_literal/elem_in_mixed_type_list_error", - "--skip_test=comparisons/in_map_literal/key_in_mixed_key_type_map_error", - "--skip_test=macros/exists/list_elem_type_exhaustive,map_key_type_exhaustive", +_TESTS_TO_SKIP = [ + "comparisons/eq_literal/eq_mixed_types_error,eq_list_elem_mixed_types_error,eq_map_value_mixed_types_error", + "comparisons/ne_literal/ne_mixed_types_error", + "comparisons/in_list_literal/elem_in_mixed_type_list_error", + "comparisons/in_map_literal/key_in_mixed_key_type_map_error", + "macros/exists/list_elem_type_exhaustive,map_key_type_exhaustive", - # Failing conformance tests. - "--skip_test=fields/qualified_identifier_resolution/map_key_float,map_key_null,map_value_repeat_key", - "--skip_test=fields/qualified_identifier_resolution/map_value_repeat_key_heterogeneous", - "--skip_test=macros/map/map_extract_keys", - "--skip_test=timestamps/duration_converters/get_milliseconds", + # Failing conformance tests. + "fields/qualified_identifier_resolution/map_key_float,map_key_null,map_value_repeat_key", + "fields/qualified_identifier_resolution/map_value_repeat_key_heterogeneous", + "macros/map/map_extract_keys", + "timestamps/duration_converters/get_milliseconds", - # Future enhancments. - "--skip_test=enums/strong_proto2", - "--skip_test=enums/strong_proto3", - ] + ["$(location " + test + ")" for test in ALL_TESTS], - data = [ - "//server/main:cel_server", - "@com_google_cel_spec//tests/simple:simple_test", - ] + ALL_TESTS, + # Future enhancments. + "enums/strong_proto2", + "enums/strong_proto3", +] + +go_test( + name = "go_default_test", + size = "small", + srcs = [ + "conformance_test.go", + ], + tags = [ + "manual", + "notap", + ], + deps = [ + "//cel:go_default_library", + "//common:go_default_library", + "//common/types:go_default_library", + "//common/types/ref:go_default_library", + "//ext:go_default_library", + "@com_github_google_go_cmp//cmp:go_default_library", + "@com_google_cel_spec//proto/test/v1:testpb_go_proto", + "@com_google_cel_spec//proto/test/v1/proto2:test_all_types_go_proto", + "@com_google_cel_spec//proto/test/v1/proto3:test_all_types_go_proto", + "@io_bazel_rules_go//go/runfiles", + "@org_golang_google_genproto_googleapis_api//expr/v1alpha1:go_default_library", + "@org_golang_google_protobuf//encoding/prototext:go_default_library", + "@org_golang_google_protobuf//testing/protocmp:go_default_library", + ], ) -# ct_dashboard is a target for the conformance dashboard and includes all simple textproto files, including those that are broken. -sh_test( - name = "ct_dashboard", - srcs = ["@com_google_cel_spec//tests:conftest-nofail.sh"], - args = [ - "$(location @com_google_cel_spec//tests/simple:simple_test)", - "--server=$(location //server/main:cel_server)", +conformance_test( + name = "conformance", + dashboard = False, + data = _ALL_TESTS, + skip_tests = _TESTS_TO_SKIP, +) - # Failing due to a GCB builder issue - "--skip_test=timestamps/timestamp_selectors_tz/getDate,getDayOfMonth_name_neg,getDayOfMonth_name_pos,getDayOfYear,getMinutes", - ] + ["$(location " + test + ")" for test in ALL_TESTS], - data = [ - "//server/main:cel_server", - "@com_google_cel_spec//tests/simple:simple_test", - ] + ALL_TESTS, +conformance_test( + name = "conformance_dashboard", + dashboard = True, + data = _ALL_TESTS, ) diff --git a/conformance/conformance_test.bzl b/conformance/conformance_test.bzl new file mode 100644 index 00000000..b66686b4 --- /dev/null +++ b/conformance/conformance_test.bzl @@ -0,0 +1,58 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This module contains build rules for generating the conformance test targets. +""" + +# Converts the list of tests to skip from the format used by the original Go test runner to a single +# flag value where each test is separated by a comma. It also performs expansion, for example +# `foo/bar,baz` becomes two entries which are `foo/bar` and `foo/baz`. +def _expand_tests_to_skip(tests_to_skip): + result = [] + for test_to_skip in tests_to_skip: + comma = test_to_skip.find(",") + if comma == -1: + result.append(test_to_skip) + continue + slash = test_to_skip.rfind("/", 0, comma) + if slash == -1: + slash = 0 + else: + slash = slash + 1 + for part in test_to_skip[slash:].split(","): + result.append(test_to_skip[0:slash] + part) + return result + +def _conformance_test_args(data, skip_tests, dashboard): + args = [] + args.append("--skip_tests={}".format(",".join(_expand_tests_to_skip(skip_tests)))) + args.append("--tests={}".format(",".join(["$(location " + test + ")" for test in data]))) + if dashboard: + args.append("--dashboard") + return args + +def conformance_test(name, data, dashboard, skip_tests = []): + native.sh_test( + name = name, + size = "small", + srcs = ["//conformance:conformance_test.sh"], + args = ["$(location //conformance:go_default_test)"] + _conformance_test_args(data, skip_tests, dashboard), + data = ["//conformance:go_default_test"] + data, + tags = [ + "guitar", + "manual", + "notap", + ] if dashboard else [], + ) diff --git a/conformance/conformance_test.go b/conformance/conformance_test.go new file mode 100644 index 00000000..9ec36056 --- /dev/null +++ b/conformance/conformance_test.go @@ -0,0 +1,265 @@ +package conformance + +import ( + "errors" + "flag" + "fmt" + "log" + "os" + "strings" + "testing" + + "github.com/bazelbuild/rules_go/go/runfiles" + "github.com/google/cel-go/cel" + "github.com/google/cel-go/common" + "github.com/google/cel-go/common/types" + "github.com/google/cel-go/common/types/ref" + "github.com/google/cel-go/ext" + "github.com/google/cel-spec/proto/test/v1/testpb" + "github.com/google/go-cmp/cmp" + "google.golang.org/protobuf/encoding/prototext" + "google.golang.org/protobuf/testing/protocmp" + + test2pb "github.com/google/cel-spec/proto/test/v1/proto2/test_all_types" + test3pb "github.com/google/cel-spec/proto/test/v1/proto3/test_all_types" + exprpb "google.golang.org/genproto/googleapis/api/expr/v1alpha1" +) + +type testsFlag []string + +func (t *testsFlag) String() string { + return strings.Join(([]string)(*t), ",") +} + +func (t *testsFlag) Set(v string) error { + *t = strings.Split(v, ",") + for i, v := range *t { + (*t)[i] = strings.TrimSpace(v) + } + return nil +} + +func (t *testsFlag) Get() any { + return ([]string)(*t) +} + +type skipTestsFlag []string + +func (t *skipTestsFlag) String() string { + return strings.Join(([]string)(*t), ",") +} + +func (t *skipTestsFlag) Set(v string) error { + *t = strings.Split(v, ",") + for i, v := range *t { + (*t)[i] = strings.TrimSpace(v) + } + return nil +} + +func (t *skipTestsFlag) Get() any { + return ([]string)(*t) +} + +var ( + dashboard bool + tests testsFlag + skipTests skipTestsFlag + + envWithMacros *cel.Env + envNoMacros *cel.Env +) + +func init() { + flag.BoolVar(&dashboard, "dashboard", false, "Dashboard.") + flag.Var(&tests, "tests", "Paths to run, separate by a comma.") + flag.Var(&skipTests, "skip_tests", "Tests to skip, separate by a comma.") + + var err error + envWithMacros, err = cel.NewCustomEnv(cel.StdLib(), cel.OptionalTypes(), + cel.EagerlyValidateDeclarations(true), + cel.EnableErrorOnBadPresenceTest(true), + cel.Types(&test2pb.TestAllTypes{}, &test3pb.TestAllTypes{}), cel.Macros(cel.StandardMacros...), ext.Bindings(), ext.Encoders(), ext.Math(), ext.Protos(), ext.Strings()) + if err != nil { + log.Fatalf("cel.NewCustomEnv() = %v", err) + } + envNoMacros, err = cel.NewCustomEnv(cel.StdLib(), cel.OptionalTypes(), + cel.EagerlyValidateDeclarations(true), + cel.EnableErrorOnBadPresenceTest(true), + cel.Types(&test2pb.TestAllTypes{}, &test3pb.TestAllTypes{}), ext.Bindings(), ext.Encoders(), ext.Math(), ext.Protos(), ext.Strings(), cel.ClearMacros()) + if err != nil { + log.Fatalf("cel.NewCustomEnv() = %v", err) + } +} + +func TestMain(m *testing.M) { + flag.Parse() + code := m.Run() + if dashboard { + code = 0 + } + os.Exit(code) +} + +func shouldSkipTest(s string) bool { + for _, t := range skipTests { + if strings.HasPrefix(s, t) { + n := s[len(t):] + if n == "" || strings.HasPrefix(n, "/") { + return true + } + } + } + return false +} + +func refValueToExprValue(res ref.Val) (*exprpb.ExprValue, error) { + if types.IsUnknown(res) { + return &exprpb.ExprValue{ + Kind: &exprpb.ExprValue_Unknown{ + Unknown: &exprpb.UnknownSet{ + Exprs: res.Value().([]int64), + }, + }}, nil + } + v, err := cel.RefValueToValue(res) + if err != nil { + return nil, err + } + return &exprpb.ExprValue{ + Kind: &exprpb.ExprValue_Value{Value: v}}, nil +} + +func exprValueToRefValue(adapter types.Adapter, ev *exprpb.ExprValue) (ref.Val, error) { + switch ev.Kind.(type) { + case *exprpb.ExprValue_Value: + return cel.ValueToRefValue(adapter, ev.GetValue()) + case *exprpb.ExprValue_Error: + // An error ExprValue is a repeated set of statuspb.Status + // messages, with no convention for the status details. + // To convert this to a types.Err, we need to convert + // these Status messages to a single string, and be + // able to decompose that string on output so we can + // round-trip arbitrary ExprValue messages. + // TODO(jimlarson) make a convention for this. + return types.NewErr("XXX add details later"), nil + case *exprpb.ExprValue_Unknown: + var unk *types.Unknown + for _, id := range ev.GetUnknown().GetExprs() { + if unk == nil { + unk = types.NewUnknown(id, nil) + } + unk = types.MergeUnknowns(types.NewUnknown(id, nil), unk) + } + return unk, nil + } + return nil, errors.New("unknown ExprValue kind") +} + +func conformanceTest(t *testing.T, name string, pb *testpb.SimpleTest) { + if shouldSkipTest(name) { + t.SkipNow() + return + } + var env *cel.Env + if pb.GetDisableMacros() { + env = envNoMacros + } else { + env = envWithMacros + } + src := common.NewStringSource(pb.GetExpr(), pb.GetName()) + ast, iss := env.ParseSource(src) + if err := iss.Err(); err != nil { + t.Fatal(err) + } + var opts []cel.EnvOption + if pb.GetContainer() != "" { + opts = append(opts, cel.Container(pb.GetContainer())) + } + opts = append(opts, cel.Declarations(pb.GetTypeEnv()...)) + var err error + env, err = env.Extend(opts...) + if err != nil { + t.Fatal(err) + } + if !pb.GetDisableCheck() { + ast, iss = env.Check(ast) + if err := iss.Err(); err != nil { + t.Fatal(err) + } + } + program, err := env.Program(ast) + if err != nil { + t.Fatal(err) + } + act := make(map[string]any, len(pb.GetBindings())) + for k, v := range pb.GetBindings() { + act[k], err = exprValueToRefValue(env.CELTypeAdapter(), v) + if err != nil { + t.Fatal(err) + } + } + ret, _, err := program.Eval(act) + switch m := pb.GetResultMatcher().(type) { + case *testpb.SimpleTest_Value: + if err != nil { + t.Fatalf("program.Eval(): got %v, want nil", err) + } + val, err := refValueToExprValue(ret) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(&exprpb.ExprValue{Kind: &exprpb.ExprValue_Value{Value: m.Value}}, val, protocmp.Transform(), protocmp.SortRepeatedFields(&exprpb.MapValue{}, "entries")); diff != "" { + t.Errorf("program.Eval() diff (-want +got):\n%s", diff) + } + case *testpb.SimpleTest_EvalError: + if err == nil && types.IsError(ret) { + err = ret.(*types.Err).Unwrap() + } + if err == nil { + t.Errorf("program.Eval(): got nil, want %v", m.EvalError) + } + default: + t.Errorf("unexpected matcher kind: %T", pb.GetResultMatcher()) + } +} + +func TestConformance(t *testing.T) { + var files []*testpb.SimpleTestFile + for _, path := range tests { + path = strings.TrimPrefix(path, "external/") + f, err := runfiles.RlocationFrom(path, "com_google_cel_spec") + if err != nil { + log.Fatalf("failed to find runfile %q: %v", path, err) + } + b, err := os.ReadFile(f) + if err != nil { + log.Fatalf("failed to read file %q: %v", f, err) + } + file := &testpb.SimpleTestFile{} + err = prototext.Unmarshal(b, file) + if err != nil { + log.Fatalf("failed to parse file %q: %v", path, err) + } + files = append(files, file) + } + for _, file := range files { + for _, section := range file.GetSection() { + for _, test := range section.GetTest() { + name := fmt.Sprintf("%s/%s/%s", file.GetName(), section.GetName(), test.GetName()) + if test.GetResultMatcher() == nil { + test.ResultMatcher = &testpb.SimpleTest_Value{ + Value: &exprpb.Value{ + Kind: &exprpb.Value_BoolValue{ + BoolValue: true, + }, + }, + } + } + t.Run(name, func(t *testing.T) { + conformanceTest(t, name, test) + }) + } + } + } +} diff --git a/conformance/conformance_test.sh b/conformance/conformance_test.sh new file mode 100755 index 00000000..1b565af1 --- /dev/null +++ b/conformance/conformance_test.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +exec $@ diff --git a/conformance/go.mod b/conformance/go.mod new file mode 100644 index 00000000..da4fc485 --- /dev/null +++ b/conformance/go.mod @@ -0,0 +1,20 @@ +module github.com/google/cel-go/conformance + +go 1.18 + +require ( + github.com/bazelbuild/rules_go v0.38.1 + github.com/google/cel-go v0.21.0 + github.com/google/cel-spec v0.14.0 + github.com/google/go-cmp v0.5.9 + google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5 + google.golang.org/protobuf v1.33.0 +) + +require ( + github.com/antlr4-go/antlr/v4 v4.13.0 // indirect + github.com/stoewer/go-strcase v1.2.0 // indirect + golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc // indirect + golang.org/x/text v0.13.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5 // indirect +) diff --git a/conformance/go.sum b/conformance/go.sum new file mode 100644 index 00000000..65d05f75 --- /dev/null +++ b/conformance/go.sum @@ -0,0 +1,32 @@ +github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= +github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= +github.com/bazelbuild/rules_go v0.38.1 h1:YGNsLhWe18Ielebav7cClP3GMwBxBE+xEArLHtmXDx8= +github.com/bazelbuild/rules_go v0.38.1/go.mod h1:TMHmtfpvyfsxaqfL9WnahCsXMWDMICTw7XeK9yVb+YU= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/cel-go v0.21.0 h1:cl6uW/gxN+Hy50tNYvI691+sXxioCnstFzLp2WO4GCI= +github.com/google/cel-go v0.21.0/go.mod h1:rHUlWCcBKgyEk+eV03RPdZUekPp6YcJwV0FxuUksYxc= +github.com/google/cel-spec v0.14.0 h1:vVw8oKDC6TTksmM5qwOxx2r+PLDUDV16eqLzeMWVenk= +github.com/google/cel-spec v0.14.0/go.mod h1:sBeqYG7I0bX68Z49T0ydlXLxJK1+TBX8musTpcSjMcY= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stoewer/go-strcase v1.2.0 h1:Z2iHWqGXH00XYgqDmNgQbIBxf3wrNq0F3feEy0ainaU= +github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc h1:mCRnTeVUjcrhlRmO0VK8a6k6Rrf6TF9htwo2pJVSjIU= +golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5 h1:nIgk/EEq3/YlnmVVXVnm14rC2oxgs1o0ong4sD/rd44= +google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5/go.mod h1:5DZzOUPCLYL3mNkQ0ms0F3EuUNZ7py1Bqeq6sxzI7/Q= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5 h1:eSaPbMR4T7WfH9FvABk36NBMacoTUKdWCvV0dx+KfOg= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5/go.mod h1:zBEcrKX2ZOcEkHWxBPAIvYUWOKKMIhYcmNiUIu2ji3I= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/conformance/start.sh b/conformance/start.sh deleted file mode 100755 index 2ff170e2..00000000 --- a/conformance/start.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -quote='"' -comma="," -startdate=$(date +%s) -timestamp='"timestamp": ' -time_string="$timestamp$startdate$comma" - -pull='"pull": "' -pull_string="$pull$1$quote" - -echo "{" > started.json -echo "$time_string" >> started.json -echo "$pull_string" >> started.json -echo "}" >> started.json diff --git a/conformance/zip.sh b/conformance/zip.sh deleted file mode 100755 index eab4b3ab..00000000 --- a/conformance/zip.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/bash -mkdir -p artifacts -touch artifacts/junit_01.xml - -input="./bazel-out/k8-fastbuild/testlogs/conformance/ct_dashboard/test.xml" - -echo "" > artifacts/junit_01.xml -echo "" >> artifacts/junit_01.xml -echo "" >> artifacts/junit_01.xml - -while IFS= read -r line -do - testline=$(echo $line | cut -c1-3) - extendline=$(echo $line | cut -c1-4) - testcase='' - close_testcase="" - - failure='" - - # This checks if the line is a test line (starts with --- but not ----) - if [ $testline = "---" ] && [ $extendline != '----' ] - then - status=$(echo $line | cut -c4-8) # The first four characters after --- are the pass/fail status of the test) - name=$(echo $line | tail -c +11 | head -c -9) # The next string excluding the time is the name of the test - echo "$testcase$name$end" >> artifacts/junit_01.xml - if [ $status = "FAIL" ] - then - read line1 - message=$(echo "$line1" | sed 's//\>/g; s/"/\"/g; s/&/\&/g') # Quote the characters in the failure message that XML might interpret - echo "$failure$message$end$close_failure" >> artifacts/junit_01.xml - else - echo $status >> artifacts/junit_01.xml - fi - echo $close_testcase >> artifacts/junit_01.xml - fi -done < "$input" -tail -2 $input >> artifacts/junit_01.xml - -comma="," -quote='"' -startdate=$(date +%s) -timestamp='"timestamp": ' -time_string="$timestamp$startdate$comma" - -result='"result": "' -test_string=$(tail -n 5 $input) - -if [[ $test_string = *"FAIL"* ]] # This operates under the assumption that the overall pass/fail message will be found in the last five lines -then - test="FAILURE" -else - test="SUCCESS" -fi -result_string="$result$test$quote" - -filedir=$(cat _DATE) -mkdir -p $filedir - -touch $filedir/build-log.txt - -echo "{" > $filedir/finished.json -echo "$time_string" >> $filedir/finished.json -echo "$result_string" >> $filedir/finished.json -echo "}" >> $filedir/finished.json - -mv started.json $filedir/started.json -mv artifacts $filedir/artifacts