From 4f715f6d4edfd141f07201e767277dc590cfe1ed Mon Sep 17 00:00:00 2001 From: Hiep Date: Sat, 18 Jul 2020 12:51:47 +0800 Subject: [PATCH] Add pluggable repository layer Part of #34 --- odin-engine/api/main.go | 88 +++++----- odin-engine/api/stats.go | 17 +- odin-engine/cmd/odin-engine/main.go | 24 ++- odin-engine/config/odin-config.yml | 3 + odin-engine/go.mod | 2 + odin-engine/go.sum | 150 ++++++++++++++++++ odin-engine/pkg/repository/all/all.go | 7 + odin-engine/pkg/repository/nosql/nosql.go | 79 +++++++++ .../pkg/repository/nosql/nosql_test.go | 101 ++++++++++++ odin-engine/pkg/repository/repository.go | 47 ++++++ odin-engine/pkg/repository/repository_test.go | 128 +++++++++++++++ odin-engine/pkg/types/main.go | 4 + 12 files changed, 596 insertions(+), 54 deletions(-) create mode 100644 odin-engine/pkg/repository/all/all.go create mode 100644 odin-engine/pkg/repository/nosql/nosql.go create mode 100644 odin-engine/pkg/repository/nosql/nosql_test.go create mode 100644 odin-engine/pkg/repository/repository.go create mode 100644 odin-engine/pkg/repository/repository_test.go diff --git a/odin-engine/api/main.go b/odin-engine/api/main.go index 41fd3c4..a087460 100644 --- a/odin-engine/api/main.go +++ b/odin-engine/api/main.go @@ -1,20 +1,20 @@ package api import ( - "os" - "syscall" + "os" + "syscall" + "github.com/theycallmemac/odin/odin-engine/pkg/fsm" + "github.com/theycallmemac/odin/odin-engine/pkg/jobs" + "github.com/theycallmemac/odin/odin-engine/pkg/repository" "github.com/valyala/fasthttp" - "github.com/theycallmemac/odin/odin-engine/pkg/fsm" - "github.com/theycallmemac/odin/odin-engine/pkg/jobs" ) var ( - // HTTPAddr contains the port used by this node - HTTPAddr string + // HTTPAddr contains the port used by this node + HTTPAddr string ) - // SetOdinEnv is used to set variables to be used by running jobs via Odin SDK // parameters: mongoURL (a string of the address for the MongoDB instance) // returns: nil @@ -31,15 +31,17 @@ func SetOdinEnv(mongoURL string) { type Service struct { addr string store fsm.Store + repo repository.Repository } // NewService is used to initialize a new service struct // parameters: addr (a string of a http address), store (a store of node details) // returns: *Service (a newly initialized service struct) -func NewService(addr string, store fsm.Store) *Service { +func NewService(addr string, store fsm.Store, repo repository.Repository) *Service { return &Service{ addr: addr, store: store, + repo: repo, } } @@ -49,44 +51,44 @@ func NewService(addr string, store fsm.Store) *Service { func (service *Service) Start() { routes := func(ctx *fasthttp.RequestCtx) { switch string(ctx.Path()) { - case "/cluster/join": - service.JoinCluster(ctx) - case "/cluster/leave": - service.LeaveCluster(ctx) - case "/execute": - Executor(ctx) - case "/execute/yaml": - ExecuteYaml(ctx) - case "/jobs/add": - AddJob(ctx) - case "/jobs/delete": - DeleteJob(ctx) - case "/jobs/info/update": - UpdateJob(ctx) - case "/jobs/info/description": - GetJobDescription(ctx) - case "/jobs/info/runs": - UpdateJobRuns(ctx) - case "/jobs/list": - ListJobs(ctx) - case "/jobs/logs": - GetJobLogs(ctx) - case "/links/add": - LinkJobs(ctx) - case "/links/delete": - UnlinkJobs(ctx) - case "/schedule": - GetJobSchedule(ctx) - case "/stats/add": - AddJobStats(ctx) - case "/stats/get": - GetJobStats(ctx) + case "/cluster/join": + service.JoinCluster(ctx) + case "/cluster/leave": + service.LeaveCluster(ctx) + case "/execute": + Executor(ctx) + case "/execute/yaml": + ExecuteYaml(ctx) + case "/jobs/add": + AddJob(ctx) + case "/jobs/delete": + DeleteJob(ctx) + case "/jobs/info/update": + UpdateJob(ctx) + case "/jobs/info/description": + GetJobDescription(ctx) + case "/jobs/info/runs": + UpdateJobRuns(ctx) + case "/jobs/list": + ListJobs(ctx) + case "/jobs/logs": + GetJobLogs(ctx) + case "/links/add": + LinkJobs(ctx) + case "/links/delete": + UnlinkJobs(ctx) + case "/schedule": + GetJobSchedule(ctx) + case "/stats/add": + AddJobStats(ctx) + case "/stats/get": + GetJobStats(service.repo, ctx) } } - // start the countdown timer for the execution until the first job - go jobs.StartTicker(service.store, service.addr) + // start the countdown timer for the execution until the first job + go jobs.StartTicker(service.store, service.addr) - HTTPAddr = service.addr + HTTPAddr = service.addr fasthttp.ListenAndServe(HTTPAddr, routes) } diff --git a/odin-engine/api/stats.go b/odin-engine/api/stats.go index 03c3f28..2c7c1b1 100644 --- a/odin-engine/api/stats.go +++ b/odin-engine/api/stats.go @@ -2,11 +2,12 @@ package api import ( "context" - "fmt" + "fmt" "strings" "github.com/theycallmemac/odin/odin-engine/pkg/jobs" - "github.com/valyala/fasthttp" + "github.com/theycallmemac/odin/odin-engine/pkg/repository" + "github.com/valyala/fasthttp" "go.mongodb.org/mongo-driver/mongo" ) @@ -29,9 +30,9 @@ func AddJobStats(ctx *fasthttp.RequestCtx) { fmt.Fprintf(ctx, "MongoDB cannot be accessed at the moment\n") } else { if InsertIntoMongo(client, typeOfValue, desc, value, id, timestamp) { - fmt.Fprintf(ctx,"200") + fmt.Fprintf(ctx, "200") } else { - fmt.Fprintf(ctx,"500") + fmt.Fprintf(ctx, "500") } } } @@ -56,13 +57,11 @@ func InsertIntoMongo(client *mongo.Client, typeOfValue string, desc string, valu } // GetJobStats is used to show stats collected by a specified job -func GetJobStats(ctx *fasthttp.RequestCtx) { - client, err := jobs.SetupClient() +func GetJobStats(repo repository.Repository, ctx *fasthttp.RequestCtx) { + statsList, err := repo.GetJobStats(ctx, string(ctx.PostBody())) if err != nil { - fmt.Fprintf(ctx, "MongoDB cannot be accessed at the moment\n") + fmt.Fprintf(ctx, "[ERROR] Cannot get job stats: %v", err) } else { - statsList := jobs.GetJobStats(client, string(ctx.PostBody())) - fmt.Fprintf(ctx, jobs.Format("ID", "DESCRIPTION", "TYPE", "VALUE")) for _, stat := range statsList { fmt.Fprintf(ctx, jobs.Format(stat.ID, stat.Description, stat.Type, stat.Value)) } diff --git a/odin-engine/cmd/odin-engine/main.go b/odin-engine/cmd/odin-engine/main.go index 5ecb242..933cdf9 100644 --- a/odin-engine/cmd/odin-engine/main.go +++ b/odin-engine/cmd/odin-engine/main.go @@ -15,7 +15,10 @@ import ( "github.com/theycallmemac/odin/odin-engine/api" "github.com/theycallmemac/odin/odin-engine/pkg/fsm" + "github.com/theycallmemac/odin/odin-engine/pkg/repository" + _ "github.com/theycallmemac/odin/odin-engine/pkg/repository/all" "github.com/theycallmemac/odin/odin-engine/pkg/resources" + "github.com/theycallmemac/odin/odin-engine/pkg/types" ) // define constsant default values to be used by the engine @@ -32,6 +35,7 @@ var ( raftAddr string joinAddr string nodeID string + config types.EngineConfig ) // init is used to define options to be used when running the engine @@ -42,6 +46,12 @@ func init() { flag.StringVar(&raftAddr, "raft", DefaultRaftAddr, "Set Raft bind address") flag.StringVar(&joinAddr, "join", "", "Set join address, if any") flag.StringVar(&nodeID, "id", "", "Node ID") + + usr, err := user.Current() + if err != nil { + log.Fatal(err) + } + config = resources.UnmarsharlYaml(resources.ReadFileBytes(usr.HomeDir + "/odin-config.yml")) } // main is used to setup the odin-engine as a single node cluster, as well as allowing for further nodes to joining it. This is all done with the use of flags when initially running the engine. @@ -67,15 +77,25 @@ func main() { log.Fatalf("%v", err) } + reg, err := repository.GetRegistration(config.Storage.Name) + if err != nil { + log.Fatal(err) + } + // TODO: options is nil for now. Pass proper storage options + repo, err := reg.OpenFunc(config.Storage.Address, nil) + if err != nil { + log.Fatal(err) + } + if httpAddr == "" { usr, _ := user.Current() config := resources.UnmarsharlYaml(resources.ReadFileBytes(usr.HomeDir + "/odin-config.yml")) api.SetOdinEnv(config.Mongo.Address) httpAddr = config.OdinVars.Master + ":" + config.OdinVars.Port } - service := api.NewService(httpAddr, *s) + service := api.NewService(httpAddr, *s, repo) go service.Start() - fmt.Println("ADDR:",httpAddr,"JOIN:",joinAddr, "RAFT:", raftAddr) + fmt.Println("ADDR:", httpAddr, "JOIN:", joinAddr, "RAFT:", raftAddr) if joinAddr != "" { if err := join(joinAddr, raftAddr, nodeID); err != nil { log.Fatalf("failed to join node at %s: %s", joinAddr, err.Error()) diff --git a/odin-engine/config/odin-config.yml b/odin-engine/config/odin-config.yml index d566e7a..4721ffa 100644 --- a/odin-engine/config/odin-config.yml +++ b/odin-engine/config/odin-config.yml @@ -4,3 +4,6 @@ odin: port: "3939" mongo: address: "mongodb://localhost:27017" +storage: + name: "mongodb" + address: "localhost:27017" diff --git a/odin-engine/go.mod b/odin-engine/go.mod index 3c6ac61..9f94482 100644 --- a/odin-engine/go.mod +++ b/odin-engine/go.mod @@ -5,7 +5,9 @@ go 1.13 require ( github.com/gorhill/cronexpr v0.0.0-20180427100037-88b0669f7d75 github.com/hashicorp/raft v1.1.2 + github.com/hidal-go/hidalgo v0.0.0-20190814174001-42e03f3b5eaa github.com/lnquy/cron v1.0.1 + github.com/stretchr/testify v1.3.0 github.com/valyala/fasthttp v1.14.0 go.mongodb.org/mongo-driver v1.3.4 gopkg.in/yaml.v2 v2.3.0 diff --git a/odin-engine/go.sum b/odin-engine/go.sum index 162e4a6..a4af58a 100644 --- a/odin-engine/go.sum +++ b/odin-engine/go.sum @@ -1,15 +1,53 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.37.4/go.mod h1:NHPJ89PdicEuT9hdPXMROBD91xc5uRDxsMtSB16k7hw= +github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= +github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= +github.com/Microsoft/go-winio v0.4.12/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= +github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= +github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= +github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/andybalholm/brotli v1.0.0 h1:7UCwP93aiSfvWpapti8g88vVVGp2qqtGyePsSuDafo4= github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= +github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 h1:EFSB7Zo9Eg91v7MJPVsifUysc/wPdN+NOnVe6bWbdBM= github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= +github.com/cenkalti/backoff v2.1.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/containerd/continuity v0.0.0-20181203112020-004b46473808/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dennwc/base v1.0.0 h1:xlBzvBNRvkQ1LFI/jom7rr0vZsvYDKtvMM6lIpjFb3M= +github.com/dennwc/base v1.0.0/go.mod h1:zaTDIiAcg2oKW9XhjIaRc1kJVteCFXSSW6jwmCedUaI= +github.com/dgraph-io/badger v1.5.4/go.mod h1:VZxzAIRPHRVNRKRo6AXrX9BJegn6il06VMTZVJYCIjQ= +github.com/dgryski/go-farm v0.0.0-20190416075124-e1214b5e05dc/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= +github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= +github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/flimzy/diff v0.1.5/go.mod h1:lFJtC7SPsK0EroDmGTSrdtWKAxOk3rO+q+e04LL05Hs= +github.com/flimzy/kivik v1.8.1/go.mod h1:S2aPycbG0eDFll4wgXt9uacSNkXISPufutnc9sv+mdA= +github.com/flimzy/testy v0.1.16/go.mod h1:3szguN8NXqgq9bt9Gu8TQVj698PJWmyx/VY1frwwKrM= +github.com/fortytw2/leaktest v1.2.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= +github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kivik/couchdb v1.8.1/go.mod h1:5XJRkAMpBlEVA4q0ktIZjUPYBjoBmRoiWvwUBzP3BOQ= +github.com/go-kivik/kivik v1.8.1/go.mod h1:nIuJ8z4ikBrVUSk3Ua8NoDqYKULPNjuddjqRvlSUyyQ= +github.com/go-kivik/kiviktest v1.1.2/go.mod h1:JdhVyzixoYhoIDUt6hRf1yAfYyaDa5/u9SDOindDkfQ= +github.com/go-kivik/pouchdb v1.3.5/go.mod h1:U+siUrqLCVxeMU3QjQTYIC3/F/e6EUKm+o5buJb7vpw= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= @@ -36,12 +74,31 @@ github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWe github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/gopherjs/gopherjs v0.0.0-20190411002643-bd77b112433e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gopherjs/jsbuiltin v0.0.0-20180426082241-50091555e127/go.mod h1:7X1acUyFRf+oVFTU6SWw9mnb57Vxn+Nbh8iPbKg95hs= github.com/gorhill/cronexpr v0.0.0-20180427100037-88b0669f7d75 h1:f0n1xnMSmBLzVfsMMvriDyA75NB/oBgILX2GcHXIQzY= github.com/gorhill/cronexpr v0.0.0-20180427100037-88b0669f7d75/go.mod h1:g2644b03hfBX9Ov0ZBDgXXens4rxSxmqFBbhvKv2yVA= +github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= +github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gotestyourself/gotestyourself v2.2.0+incompatible/go.mod h1:zZKM6oeNM8k+FRljX1mnzVYeS8wiGgQyvST1/GafPbY= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-hclog v0.9.1 h1:9PZfAcVEvez4yhLH2TBU64/h/z4xlFI80cWXRrxuKuM= github.com/hashicorp/go-hclog v0.9.1/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= @@ -53,50 +110,87 @@ github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/raft v1.1.2 h1:oxEL5DDeurYxLd3UbcY/hccgSPhLLpiBZ1YxtWEq59c= github.com/hashicorp/raft v1.1.2/go.mod h1:vPAJM8Asw6u8LxC3eJCUZmRP/E4QmUGE1R7g7k8sG/8= github.com/hashicorp/raft-boltdb v0.0.0-20171010151810-6e5ba93211ea/go.mod h1:pNv7Wc3ycL6F5oOWn+tPGo2gWD4a5X+yp/ntwdKLjRk= +github.com/hidal-go/hidalgo v0.0.0-20190814174001-42e03f3b5eaa h1:hBE4LGxApbZiV/3YoEPv7uYlUMWOogG1hwtkpiU87zQ= +github.com/hidal-go/hidalgo v0.0.0-20190814174001-42e03f3b5eaa/go.mod h1:bPkrxDlroXxigw8BMWTEPTv4W5/rQwNgg2BECXsgyX0= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/imdario/mergo v0.3.7/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.10.4 h1:jFzIFaf586tquEB5EhzQG0HwGNSlgAJpG53G6Ss11wc= github.com/klauspost/compress v1.10.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lnquy/cron v1.0.1 h1:yYcu8TBex8/Ew7FTtugYe4+canCL6HX0HhK6tq+nGM4= github.com/lnquy/cron v1.0.1/go.mod h1:hu2Y7H68/8oKk6T4+K4qdbopbnaP4rGltK3ylWiiDss= +github.com/mailru/easyjson v0.0.0-20180730094502-03f2033d19d5/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190403194419-1ea4449da983/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= +github.com/ory/dockertest v3.3.4+incompatible/go.mod h1:1vX4m9wsvi00u5bseYwXaSnhNrne+V0E6LAcBILJdPs= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= +github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= +github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= +github.com/theycallmemac/odin v1.0.0 h1:fefsCDaPp7HTD2WXCldAfazAjsTI/cK2ojg8ZbnfcIk= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= @@ -108,23 +202,50 @@ github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc h1:n+nNi93yXLkJvKwXNP9d55HC7lGK4H/SRcwB5IaUZLo= github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= +github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.mongodb.org/mongo-driver v1.0.4/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.3.4 h1:zs/dKNwX0gYUtzwrN9lLiR15hCO0nDwQj5xXx+vjCdE= go.mongodb.org/mongo-driver v1.3.4/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS3gZBapIE= +go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= +go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5 h1:8dUaAV7K4uHsF56JQWkprecIQKdPHtR9jCHF5nB8uzc= golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -134,17 +255,46 @@ golang.org/x/sys v0.0.0-20190523142557-0e01d883c5c5/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= +google.golang.org/api v0.3.2/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/olivere/elastic.v5 v5.0.80/go.mod h1:uhHoB4o3bvX5sorxBU29rPcmBQdV2Qfg0FBrx5D6pV0= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/odin-engine/pkg/repository/all/all.go b/odin-engine/pkg/repository/all/all.go new file mode 100644 index 0000000..5743b21 --- /dev/null +++ b/odin-engine/pkg/repository/all/all.go @@ -0,0 +1,7 @@ +package all + +// Importing this package will import all supported backends +// and register them in Repository registry +import ( + _ "github.com/theycallmemac/odin/odin-engine/pkg/repository/nosql" +) diff --git a/odin-engine/pkg/repository/nosql/nosql.go b/odin-engine/pkg/repository/nosql/nosql.go new file mode 100644 index 0000000..034de41 --- /dev/null +++ b/odin-engine/pkg/repository/nosql/nosql.go @@ -0,0 +1,79 @@ +package nosql + +import ( + "context" + + "github.com/hidal-go/hidalgo/legacy/nosql/mongo" + + "github.com/hidal-go/hidalgo/legacy/nosql" + base "github.com/theycallmemac/odin/odin-engine/pkg/repository" +) + +func init() { + base.AddRegistration("mongodb", &base.Registration{ + OpenFunc: func(address string, options base.Options) (base.Repository, error) { + db, err := mongo.Dial(address, base.DefaultDatabase, nosql.Options(options)) + if err != nil { + return nil, err + } + repo := &Repository{ + db: db, + } + if err := repo.ensureIndex(context.Background()); err != nil { + return nil, err + } + return repo, nil + }, + }) +} + +type Repository struct { + db nosql.Database +} + +func (repo *Repository) ensureIndex(ctx context.Context) error { + return repo.db.EnsureIndex( + ctx, + base.ObservabilityTable, + nosql.Index{ + Fields: []string{"_id"}, + Type: nosql.StringExact, + }, + []nosql.Index{ + { + Fields: []string{"id"}, + Type: nosql.StringExact, + }, + }, + ) +} + +func (repo *Repository) GetJobStats(ctx context.Context, id string) ([]base.JobStats, error) { + iter := repo.db.Query(base.ObservabilityTable).WithFields(nosql.FieldFilter{ + Path: []string{"id"}, + Filter: nosql.Equal, + Value: nosql.String(id), + }).Iterate() + + if iter.Err() != nil { + return nil, iter.Err() + } + defer iter.Close() + + results := make([]base.JobStats, 0) + for iter.Next(ctx) { + doc := iter.Doc() + jobStats := base.JobStats{ + ID: string(doc["id"].(nosql.String)), + Description: string(doc["desc"].(nosql.String)), + Type: string(doc["type"].(nosql.String)), + Value: string(doc["value"].(nosql.String)), + } + results = append(results, jobStats) + } + return results, nil +} + +func (repo *Repository) Close() error { + return repo.db.Close() +} diff --git a/odin-engine/pkg/repository/nosql/nosql_test.go b/odin-engine/pkg/repository/nosql/nosql_test.go new file mode 100644 index 0000000..01dc560 --- /dev/null +++ b/odin-engine/pkg/repository/nosql/nosql_test.go @@ -0,0 +1,101 @@ +package nosql + +import ( + "context" + "testing" + + "github.com/hidal-go/hidalgo/legacy/nosql" + "github.com/hidal-go/hidalgo/legacy/nosql/mongo" + base "github.com/theycallmemac/odin/odin-engine/pkg/repository" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + mongodriver "go.mongodb.org/mongo-driver/mongo" + mongooptions "go.mongodb.org/mongo-driver/mongo/options" +) + +// use mongo as default nosql test db +func testRepo(t *testing.T) *Repository { + // TODO: pass mongo url as env variable instead + db, err := mongo.Dial("localhost:27017", base.DefaultDatabase, nosql.Options(nil)) + require.NoError(t, err) + return &Repository{ + db: db, + } +} + +// drop all related collections +// TODO: find a better way to cleanup with hidal-go +func cleanUp(t *testing.T, ctx context.Context) { + client, err := mongodriver.NewClient(mongooptions.Client().ApplyURI("mongodb://localhost:27017")) + require.NoError(t, err) + client.Connect(ctx) + err = client.Database(base.DefaultDatabase).Collection(base.ObservabilityTable).Drop(ctx) + require.NoError(t, err) + require.NoError(t, client.Disconnect(ctx)) +} + +func TestRepository_GetJobStats(t *testing.T) { + type args struct { + ctx context.Context + id string + } + tests := []struct { + name string + existing []base.JobStats + args args + want []base.JobStats + wantErr error + }{ + { + name: "Simple success", + existing: []base.JobStats{ + { + ID: "1", + Description: "sample stats", + Type: "count", + Value: "2", + }, + }, + args: args{ + ctx: context.Background(), + id: "1", + }, + want: []base.JobStats{ + { + ID: "1", + Description: "sample stats", + Type: "count", + Value: "2", + }, + }, + wantErr: nil, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + repo := testRepo(t) + require.NoError(t, repo.ensureIndex(tt.args.ctx)) + + for _, e := range tt.existing { + _, err := repo.db.Insert(tt.args.ctx, base.ObservabilityTable, nil, nosql.Document{ + "id": nosql.String(e.ID), + "desc": nosql.String(e.Description), + "type": nosql.String(e.Type), + "value": nosql.String(e.Value), + }) + require.NoError(t, err) + } + got, err := repo.GetJobStats(tt.args.ctx, tt.args.id) + if tt.wantErr == nil { + require.NoError(t, err) + } else { + require.EqualError(t, err, tt.wantErr.Error()) + } + assert.ElementsMatch(t, tt.want, got) + + cleanUp(t, tt.args.ctx) + require.NoError(t, repo.Close()) + }) + } +} diff --git a/odin-engine/pkg/repository/repository.go b/odin-engine/pkg/repository/repository.go new file mode 100644 index 0000000..739aa41 --- /dev/null +++ b/odin-engine/pkg/repository/repository.go @@ -0,0 +1,47 @@ +package repository + +import ( + "context" + "fmt" +) + +const ( + DefaultDatabase = "odin" + ObservabilityTable = "observability" +) + +var registry = make(map[string]*Registration) + +type Options map[string]interface{} + +type Registration struct { + OpenFunc func(address string, options Options) (Repository, error) +} + +func AddRegistration(name string, reg *Registration) error { + if _, ok := registry[name]; ok { + return fmt.Errorf("%s is already registered", name) + } + registry[name] = reg + return nil +} + +func GetRegistration(name string) (*Registration, error) { + reg, ok := registry[name] + if !ok { + return nil, fmt.Errorf("cannot find %s repository", name) + } + return reg, nil +} + +type JobStats struct { + ID string + Description string + Type string + Value string +} + +type Repository interface { + GetJobStats(ctx context.Context, id string) ([]JobStats, error) + Close() error +} diff --git a/odin-engine/pkg/repository/repository_test.go b/odin-engine/pkg/repository/repository_test.go new file mode 100644 index 0000000..c350fd5 --- /dev/null +++ b/odin-engine/pkg/repository/repository_test.go @@ -0,0 +1,128 @@ +package repository + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestAddRegistration(t *testing.T) { + type args struct { + name string + reg *Registration + } + tests := []struct { + name string + existing []args + args args + wantErr error + }{ + { + name: "First registration", + existing: []args{}, + args: args{ + name: "mongodb", + reg: &Registration{}, + }, + wantErr: nil, + }, + { + name: "Second registration", + existing: []args{ + { + name: "couchdb", + reg: &Registration{}, + }, + }, + args: args{ + name: "mongodb", + reg: &Registration{}, + }, + wantErr: nil, + }, + { + name: "Duplicate registration", + existing: []args{ + { + name: "mongodb", + reg: &Registration{}, + }, + }, + args: args{ + name: "mongodb", + reg: &Registration{}, + }, + wantErr: errors.New("mongodb is already registered"), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + registry = make(map[string]*Registration) // reset registry + for _, e := range tt.existing { + registry[e.name] = e.reg + } + err := AddRegistration(tt.args.name, tt.args.reg) + if tt.wantErr == nil { + require.NoError(t, err) + } else { + require.EqualError(t, err, tt.wantErr.Error()) + } + reg, ok := registry[tt.args.name] + assert.True(t, ok) + assert.Equal(t, tt.args.reg, reg) + }) + } +} + +func TestGetRegistration(t *testing.T) { + testReg := &Registration{} + type args struct { + name string + } + tests := []struct { + name string + existingNames []string + existingRegs []*Registration + args args + want *Registration + wantErr error + }{ + { + name: "Exist", + existingNames: []string{"mongodb"}, + existingRegs: []*Registration{testReg}, + args: args{ + name: "mongodb", + }, + want: testReg, + wantErr: nil, + }, + { + name: "Not Exist", + existingNames: []string{}, + existingRegs: []*Registration{}, + args: args{ + name: "mongodb", + }, + want: nil, + wantErr: errors.New("cannot find mongodb repository"), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + registry = make(map[string]*Registration) // reset registry + for i, name := range tt.existingNames { + registry[name] = tt.existingRegs[i] + } + got, err := GetRegistration(tt.args.name) + if tt.wantErr == nil { + require.NoError(t, err) + } else { + require.EqualError(t, err, tt.wantErr.Error()) + } + assert.Equal(t, tt.want, got) + }) + } +} diff --git a/odin-engine/pkg/types/main.go b/odin-engine/pkg/types/main.go index 5f1742e..d29a04f 100644 --- a/odin-engine/pkg/types/main.go +++ b/odin-engine/pkg/types/main.go @@ -9,6 +9,10 @@ type EngineConfig struct { Mongo struct { Address string `yaml:"address"` } `yaml:"mongo"` + Storage struct { + Name string `yaml:"name"` + Address string `yaml:"address"` + } `yaml:"storage"` } // JobConfig is a type to be used for accessing job configuration file information