diff --git a/.github/workflows/pr_open.yml b/.github/workflows/pr_open.yml index 86f9a04e3..6c92ea1eb 100644 --- a/.github/workflows/pr_open.yml +++ b/.github/workflows/pr_open.yml @@ -228,3 +228,68 @@ jobs: run: go mod download - name: run vet run: go vet . + Build: + name: 'Build' + needs: + - build_control + - Test_sqlite + - Test_postgres + - Lint + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ windows-latest, ubuntu-latest ] + include: + - os: windows-latest + file: windows + - os: ubuntu-latest + file: default + steps: + - uses: actions/setup-go@v2 + with: + go-version: '1.16' + - uses: actions/checkout@v2 + - uses: actions/cache@v2 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + - name: get deps + run: go get + env: + CGO_ENABLED: 1 + - name: fetch numary control + uses: actions/download-artifact@v2 + with: + name: control-dist + path: cmd/control/ + - name: fetch docs + uses: actions/download-artifact@v2 + with: + name: docs-dist + path: docs/ + - name: OSXCross for CGO Support + if: matrix.os == 'ubuntu-latest' + run: | + mkdir ../../osxcross + git clone https://github.com/plentico/osxcross-target.git ../../osxcross/target + - name: Downgrade libssl + if: matrix.os == 'ubuntu-latest' + run: | + echo 'deb http://security.ubuntu.com/ubuntu bionic-security main' | sudo tee -a /etc/apt/sources.list + sudo apt update && apt-cache policy libssl1.0-dev + sudo apt-get install libssl1.0-dev + - name: Run GoReleaser + uses: goreleaser/goreleaser-action@v2 + with: + version: latest + args: build --parallelism 4 --rm-dist --skip-validate --snapshot --config .github/.goreleaser.${{matrix.file}}.yml + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - uses: actions/upload-artifact@v2 + with: + name: build-${{matrix.file}} + path: ./build/** \ No newline at end of file diff --git a/cmd/container.go b/cmd/container.go index c61f6e1fb..2668b68d7 100644 --- a/cmd/container.go +++ b/cmd/container.go @@ -12,7 +12,6 @@ import ( type containerConfig struct { version string - storageDriver string ledgerLister controllers.LedgerLister basicAuth string options []fx.Option @@ -28,12 +27,6 @@ func WithVersion(version string) option { } } -func WithStorageDriver(driver string) option { - return func(c *containerConfig) { - c.storageDriver = driver - } -} - func WithLedgerLister(lister controllers.LedgerLister) option { return func(c *containerConfig) { c.ledgerLister = lister @@ -66,7 +59,6 @@ func WithRememberConfig(rememberConfig bool) option { var DefaultOptions = []option{ WithVersion("latest"), - WithStorageDriver("sqlite"), WithLedgerLister(controllers.LedgerListerFn(func() []string { return []string{} })), @@ -82,7 +74,7 @@ func NewContainer(options ...option) *fx.App { providers := make([]interface{}, 0) providers = append(providers, fx.Annotate(func() string { return cfg.version }, fx.ResultTags(`name:"version"`)), - fx.Annotate(func() string { return cfg.storageDriver }, fx.ResultTags(`name:"storageDriver"`)), + fx.Annotate(func(driver storage.Driver) string { return driver.Name() }, fx.ResultTags(`name:"storageDriver"`)), fx.Annotate(func() controllers.LedgerLister { return cfg.ledgerLister }, fx.ResultTags(`name:"ledgerLister"`)), fx.Annotate(func() string { return cfg.basicAuth }, fx.ResultTags(`name:"httpBasic"`)), fx.Annotate(ledger.NewResolver, fx.ParamTags(`group:"resolverOptions"`)), @@ -92,35 +84,32 @@ func NewContainer(options ...option) *fx.App { fx.As(new(ledger.ResolverOption)), ), api.NewAPI, - func() (f storage.Factory, err error) { - f, err = storage.NewDefaultFactory(cfg.storageDriver) - if err != nil { - return nil, err - } + func(driver storage.Driver) storage.Factory { + f := storage.NewDefaultFactory(driver) if cfg.cache { f = storage.NewCachedStorageFactory(f) } if cfg.rememberConfig { f = storage.NewRememberConfigStorageFactory(f) } - return f, nil + return f }, ) - + invokes := make([]interface{}, 0) + invokes = append(invokes, func(driver storage.Driver, lifecycle fx.Lifecycle) error { + err := driver.Initialize(context.Background()) + if err != nil { + return errors.Wrap(err, "initializing driver") + } + lifecycle.Append(fx.Hook{ + OnStop: driver.Close, + }) + return nil + }) fxOptions := append( []fx.Option{ - fx.Invoke(func(lc fx.Lifecycle, h *api.API, storageFactory storage.Factory) { - lc.Append(fx.Hook{ - OnStop: func(ctx context.Context) error { - err := storageFactory.Close(ctx) - if err != nil { - return errors.Wrap(err, "closing storage factory") - } - return nil - }, - }) - }), fx.Provide(providers...), + fx.Invoke(invokes...), api.Module, }, cfg.options..., diff --git a/cmd/container_test.go b/cmd/container_test.go index b007cabea..7709f5d63 100644 --- a/cmd/container_test.go +++ b/cmd/container_test.go @@ -2,26 +2,71 @@ package cmd import ( "context" + "github.com/numary/ledger/ledgertesting" + "github.com/numary/ledger/storage" + "github.com/numary/ledger/storage/sqlstorage" + "github.com/stretchr/testify/assert" "go.uber.org/fx" "testing" ) -func TestContainer(t *testing.T) { - run := make(chan struct{}, 1) - app := NewContainer( - WithOption(fx.Invoke(func() { - run <- struct{}{} - })), - ) - err := app.Start(context.Background()) - if err != nil { - t.Fatal(err) +func TestContainers(t *testing.T) { + + type testCase struct { + name string + options []option } - defer app.Stop(context.Background()) - select { - case <-run: - default: - t.Fatal("application not started correctly") + for _, tc := range []testCase{ + { + name: "default", + options: []option{ + WithOption(fx.Provide(func() storage.Driver { + return sqlstorage.NewInMemorySQLiteDriver() + })), + }, + }, + { + name: "pg", + options: []option{ + WithRememberConfig(false), + WithOption(fx.Provide(ledgertesting.PostgresServer)), + WithOption(fx.Provide(func(t *testing.T, pgServer *ledgertesting.PGServer) storage.Driver { + return sqlstorage.NewCachedDBDriver("postgres", sqlstorage.PostgreSQL, pgServer.ConnString()) + })), + WithOption(fx.Invoke(func(t *testing.T, storageFactory storage.Factory) { + store, err := storageFactory.GetStore("testing") + assert.NoError(t, err) + assert.NoError(t, store.Close(context.Background())) + })), + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + run := make(chan struct{}, 1) + options := append(tc.options, + WithRememberConfig(false), + WithOption(fx.Invoke(func() { + run <- struct{}{} + })), + WithOption(fx.Provide(func() *testing.T { + return t + })), + ) + app := NewContainer(options...) + + err := app.Start(context.Background()) + if err != nil { + t.Fatal(err) + } + defer app.Stop(context.Background()) + + select { + case <-run: + default: + t.Fatal("application not started correctly") + } + }) } + } diff --git a/cmd/root.go b/cmd/root.go index eb338cded..66f475b39 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -5,33 +5,36 @@ import ( "context" "encoding/json" "fmt" + "github.com/gin-gonic/gin" + "github.com/numary/ledger/api" "github.com/numary/ledger/api/controllers" - "github.com/numary/ledger/storage/postgres" - "github.com/numary/ledger/storage/sqlite" + "github.com/numary/ledger/storage" + "github.com/numary/ledger/storage/sqlstorage" + "github.com/numary/machine/script/compiler" "github.com/pkg/errors" "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "github.com/spf13/viper" "go.uber.org/fx" "io/ioutil" + "net" "net/http" "os" "path" "regexp" "strings" - - "github.com/gin-gonic/gin" - "github.com/numary/ledger/api" - "github.com/numary/ledger/storage" - "github.com/numary/machine/script/compiler" - "github.com/spf13/cobra" - "github.com/spf13/viper" ) var ( Version = "develop" BuildDate = "-" Commit = "-" +) + +func NewRootCommand() *cobra.Command { + viper.SetDefault("version", Version) - root = &cobra.Command{ + root := &cobra.Command{ Use: "numary", Short: "Numary", DisableAutoGenTag: true, @@ -41,35 +44,12 @@ var ( return errors.Wrap(err, "creating storage directory") } - switch viper.GetString("storage.driver") { - case "sqlite": - storage.RegisterDriver("sqlite", sqlite.NewDriver( - viper.GetString("storage.dir"), - viper.GetString("storage.sqlite.db_name"), - )) - case "postgres": - storage.RegisterDriver("postgres", postgres.NewDriver( - viper.GetString("storage.postgres.conn_string"), - )) - default: - return fmt.Errorf("unknown storage driver %s", viper.GetString("storage.driver")) - } if viper.GetBool("debug") { logrus.StandardLogger().Level = logrus.DebugLevel } return nil }, } -) - -func PrintVersion(cmd *cobra.Command, args []string) { - fmt.Printf("Version: %s \n", Version) - fmt.Printf("Date: %s \n", BuildDate) - fmt.Printf("Commit: %s \n", Commit) -} - -func Execute() { - viper.SetDefault("version", Version) server := &cobra.Command{ Use: "server", @@ -83,26 +63,43 @@ func Execute() { start := &cobra.Command{ Use: "start", - Run: func(cmd *cobra.Command, args []string) { - app := NewContainer( - WithVersion(Version), - WithStorageDriver(viper.GetString("storage.driver")), - WithCacheStorage(viper.GetBool("storage.cache")), - WithHttpBasicAuth(viper.GetString("server.http.basic_auth")), - WithLedgerLister(controllers.LedgerListerFn(func() []string { - return viper.GetStringSlice("ledgers") - })), - WithRememberConfig(true), - WithOption(fx.Invoke(func(h *api.API) { + RunE: func(cmd *cobra.Command, args []string) error { + app, err := createContainer( + WithOption(fx.Invoke(func(h *api.API) error { + listener, err := net.Listen("tcp", viper.GetString("server.http.bind_address")) + if err != nil { + return err + } + + go http.Serve(listener, h) go func() { - err := http.ListenAndServe(viper.GetString("server.http.bind_address"), h) + select { + case <-cmd.Context().Done(): + } + err := listener.Close() if err != nil { panic(err) } }() + + return nil })), ) - app.Run() + if err != nil { + return err + } + terminated := make(chan struct{}) + go func() { + app.Run() + close(terminated) + }() + select { + case <-cmd.Context().Done(): + return app.Stop(context.Background()) + case <-terminated: + } + + return nil }, } @@ -117,7 +114,7 @@ func Execute() { Run: func(cmd *cobra.Command, args []string) { err := viper.SafeWriteConfig() if err != nil { - fmt.Println(err) + logrus.Println(err) } }, }) @@ -128,17 +125,25 @@ func Execute() { store.AddCommand(&cobra.Command{ Use: "init", - Run: func(cmd *cobra.Command, args []string) { - // TODO: Use the container? - s, err := storage.GetStore(viper.GetString("storage.driver"), "default") - if err != nil { - logrus.Fatal(err) - } + RunE: func(cmd *cobra.Command, args []string) error { + _, err := createContainer( + WithOption(fx.Invoke(func(storageFactory storage.Factory) error { + s, err := storageFactory.GetStore("default") + if err != nil { + return err + } - err = s.Initialize(context.Background()) + err = s.Initialize(context.Background()) + if err != nil { + return err + } + return nil + })), + ) if err != nil { - logrus.Fatal(err) + return err } + return nil }, }) @@ -235,6 +240,7 @@ func Execute() { root.PersistentFlags().String("storage.sqlite.db_name", "numary", "SQLite database name") root.PersistentFlags().String("storage.postgres.conn_string", "postgresql://localhost/postgres", "Postgre connection string") root.PersistentFlags().Bool("storage.cache", true, "Storage cache") + root.PersistentFlags().Bool("persist-config", true, "Persist config on disk") root.PersistentFlags().String("server.http.bind_address", "localhost:3068", "API bind address") root.PersistentFlags().String("ui.http.bind_address", "localhost:3068", "UI bind address") root.PersistentFlags().StringSlice("ledgers", []string{"quickstart"}, "Ledgers") @@ -250,7 +256,48 @@ func Execute() { viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_", "-", "_")) viper.AutomaticEnv() - if err := root.Execute(); err != nil { + return root +} + +func PrintVersion(cmd *cobra.Command, args []string) { + fmt.Printf("Version: %s \n", Version) + fmt.Printf("Date: %s \n", BuildDate) + fmt.Printf("Commit: %s \n", Commit) +} + +func createContainer(opts ...option) (*fx.App, error) { + + opts = append(opts, + WithVersion(Version), + WithOption(fx.Provide(func() (storage.Driver, error) { + switch viper.GetString("storage.driver") { + case "sqlite": + return sqlstorage.NewOpenCloseDBDriver("sqlite", sqlstorage.SQLite, func(name string) string { + return sqlstorage.SQLiteFileConnString(path.Join( + viper.GetString("storage.dir"), + fmt.Sprintf("%s_%s.db", viper.GetString("storage.sqlite.db_name"), name), + )) + }), nil + case "postgres": + return sqlstorage.NewCachedDBDriver("postgres", sqlstorage.PostgreSQL, + viper.GetString("storage.postgres.conn_string")), nil + default: + return nil, fmt.Errorf("unknown storage driver %s", viper.GetString("storage.driver")) + } + })), + WithCacheStorage(viper.GetBool("storage.cache")), + WithHttpBasicAuth(viper.GetString("server.http.basic_auth")), + WithLedgerLister(controllers.LedgerListerFn(func() []string { + return viper.GetStringSlice("ledgers") + })), + WithRememberConfig(true), + ) + + return NewContainer(opts...), nil +} + +func Execute() { + if err := NewRootCommand().Execute(); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } diff --git a/cmd/root_test.go b/cmd/root_test.go new file mode 100644 index 000000000..e2bf47a9b --- /dev/null +++ b/cmd/root_test.go @@ -0,0 +1,113 @@ +package cmd + +import ( + "bytes" + "context" + "github.com/numary/ledger/ledgertesting" + "github.com/stretchr/testify/assert" + "net/http" + "os" + "testing" + "time" +) + +func TestServer(t *testing.T) { + + pgServer, err := ledgertesting.PostgresServer() + assert.NoError(t, err) + defer pgServer.Close() + + type env struct { + key string + value string + } + + type testCase struct { + name string + args []string + env []env + } + + for _, tc := range []testCase{ + { + name: "default", + }, + { + name: "pg", + args: []string{"--storage.driver", "postgres", "--storage.postgres.conn_string", pgServer.ConnString()}, + }, + { + name: "pg-with-env-var", + env: []env{ + { + key: "NUMARY_STORAGE_DRIVER", + value: "postgres", + }, + { + key: "NUMARY_STORAGE_POSTGRES_CONN_STRING", + value: pgServer.ConnString(), + }, + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + for _, e := range tc.env { + oldValue := os.Getenv(e.key) + os.Setenv(e.key, e.value) + defer os.Setenv(e.key, oldValue) + } + args := []string{"server", "start", "--persist-config=false"} + args = append(args, tc.args...) + root := NewRootCommand() + root.SetArgs(args) + root.SetOut(os.Stdout) + root.SetIn(os.Stdin) + root.SetErr(os.Stdout) + + terminated := make(chan struct{}) + + defer func() { + select { + case <-terminated: + } + }() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + go func() { + err := root.ExecuteContext(ctx) + assert.NoError(t, err) + close(terminated) + }() + + counter := time.Duration(0) + timeout := 5 * time.Second + delay := 200 * time.Millisecond + for { + _, err := http.DefaultClient.Get("http://localhost:3068/_info") + if err != nil { + if counter*delay < timeout { + counter++ + <-time.After(delay) + continue + } + assert.FailNow(t, err.Error()) + } + break + } + + res, err := http.DefaultClient.Post("http://localhost:3068/testing/transactions", "application/json", bytes.NewBufferString(`{ + "postings": [{ + "source": "world", + "destination": "central_bank", + "asset": "USD", + "amount": 100 + }] + }`)) + assert.NoError(t, err) + assert.Equal(t, http.StatusOK, res.StatusCode) + }) + } + +} diff --git a/go.mod b/go.mod index 6d9cd5705..28c1bfd93 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.16 require ( github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/gin-contrib/cors v1.3.1 github.com/gin-contrib/logger v0.2.0 github.com/gin-gonic/gin v1.7.4 @@ -14,10 +15,13 @@ require ( github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-sqlite3 v1.14.7 github.com/numary/machine v0.0.0-20210831114934-e54c99840e08 + github.com/ory/dockertest/v3 v3.8.1 + github.com/pborman/uuid v1.2.1 github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.8.1 github.com/spf13/cobra v1.1.3 github.com/spf13/viper v1.8.1 + github.com/stretchr/testify v1.7.0 github.com/swaggo/swag v1.7.4 go.uber.org/atomic v1.9.0 // indirect go.uber.org/dig v1.13.0 // indirect diff --git a/go.sum b/go.sum index e345c622c..7c23fcbdc 100644 --- a/go.sum +++ b/go.sum @@ -37,12 +37,18 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8= +github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= +github.com/Microsoft/go-winio v0.5.1 h1:aPJp2QD7OOrhO5tQXqQoGSJc+DjDtWTGLOmNyAm6FgY= +github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= +github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw= +github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= @@ -74,18 +80,24 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= +github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= +github.com/cenkalti/backoff/v4 v4.1.2 h1:6Yo7N8UP2K6LWZnW94DLVSSrbobcWdVzAYOisuDPIFo= +github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/prettybench v0.1.0 h1:6H/vgT6ZVysiOSYZXd0Cl3bD7U3XtVbmZycfsCFnHgk= github.com/cespare/prettybench v0.1.0/go.mod h1:ZXdFCkgnnk2/4dXGxEzvTuKF2uKsVvNriRCyMRmYJQc= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= @@ -95,6 +107,9 @@ github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= +github.com/containerd/console v1.0.2/go.mod h1:ytZPjGgY2oeTkAONYafi2kSj0aYggsf8acV1PGKCbzQ= +github.com/containerd/continuity v0.0.0-20190827140505-75bee3e2ccb6 h1:NmTXa/uVnDyp0TY5MKi197+3HWcnYWfnHGyaFthlnGw= +github.com/containerd/continuity v0.0.0-20190827140505-75bee3e2ccb6/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -111,6 +126,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.1 h1:r/myEWzV9lfsM1tFLgDyu0atFtJ1fXn261LKY github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -118,6 +135,14 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dnephin/pflag v1.0.7 h1:oxONGlWxhmUct0YzKTgrpQv9AUA1wtPBn7zuSjJqptk= github.com/dnephin/pflag v1.0.7/go.mod h1:uxE91IoWURlOiTUIA8Mq5ZZkAv3dPUfZNaT80Zm7OQE= +github.com/docker/cli v20.10.11+incompatible h1:tXU1ezXcruZQRrMP8RN2z9N91h+6egZTS1gsPsKantc= +github.com/docker/cli v20.10.11+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/docker v20.10.7+incompatible h1:Z6O9Nhsjv+ayUEeI1IojKbYcsGdgYSNqxe1s2MYzUhQ= +github.com/docker/docker v20.10.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= +github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= +github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= @@ -136,6 +161,7 @@ github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= +github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -197,6 +223,7 @@ github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFG github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -265,6 +292,7 @@ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm4 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= @@ -315,6 +343,8 @@ github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= +github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= @@ -396,6 +426,7 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxv github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -404,6 +435,7 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw= github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= +github.com/lib/pq v0.0.0-20180327071824-d34b9ff171c2/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= @@ -452,12 +484,16 @@ github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:F github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= +github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 h1:rzf0wL0CHVc8CEsgyygG0Mn9CNCCPZqOPaz8RiiHYQk= +github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= @@ -480,6 +516,14 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= +github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= +github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM= +github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/runc v1.0.2 h1:opHZMaswlyxz1OuGpBE53Dwe4/xF7EZTY0A2L/FpCOg= +github.com/opencontainers/runc v1.0.2/go.mod h1:aTaHFFwQXuA71CiyxOdFFIorAoemI04suvGRQFzWTD0= +github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8= github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= @@ -488,9 +532,13 @@ github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxS github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= +github.com/ory/dockertest/v3 v3.8.1 h1:vU/8d1We4qIad2YM0kOwRVtnyue7ExvacPiw1yDm17g= +github.com/ory/dockertest/v3 v3.8.1/go.mod h1:wSRQ3wmkz+uSARYMk7kVJFDBGm8x5gSxIhI7NDc+BAQ= github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw= +github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= @@ -543,6 +591,7 @@ github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= @@ -552,6 +601,7 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeV github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= @@ -597,6 +647,7 @@ github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/swaggo/swag v1.7.4 h1:up+ixy8yOqJKiFcuhMgkuYuF4xnevuhnFAXXF8OSfNg= github.com/swaggo/swag v1.7.4/go.mod h1:zD8h6h4SPv7t3l+4BKdRquqW1ASWjKZgT6Qv9z3kNqI= +github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= @@ -608,6 +659,14 @@ github.com/urfave/cli v1.22.1 h1:+mkCCcOFKPnCmVYVcURKps1Xe+3zP90gSYGNfRkjoIY= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M= github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= +github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= +github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -757,6 +816,7 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= @@ -807,6 +867,7 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -814,6 +875,7 @@ golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -831,12 +893,15 @@ golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -845,6 +910,7 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1078,6 +1144,7 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= @@ -1086,6 +1153,7 @@ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/gotestsum v1.7.0 h1:RwpqwwFKBAa2h+F6pMEGpE707Edld0etUD3GhqqhDNc= gotest.tools/gotestsum v1.7.0/go.mod h1:V1m4Jw3eBerhI/A6qCxUE07RnCg7ACkKj9BYcAm09V8= +gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/ledger/ledger_test.go b/ledger/ledger_test.go index 276067bcf..487b3b15f 100644 --- a/ledger/ledger_test.go +++ b/ledger/ledger_test.go @@ -6,39 +6,44 @@ import ( "errors" "flag" "fmt" - "github.com/jackc/pgx/v4/pgxpool" - "github.com/numary/ledger/storage/sqlite" + "github.com/numary/ledger/ledgertesting" + "github.com/numary/ledger/storage" + "github.com/numary/ledger/storage/sqlstorage" "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" "math/rand" "os" - "path" "reflect" "testing" - "github.com/numary/ledger/storage" - "github.com/google/go-cmp/cmp" "github.com/numary/ledger/core" "github.com/numary/ledger/ledger/query" - "github.com/numary/ledger/storage/postgres" "go.uber.org/fx" - - // Register sql drivers - _ "github.com/numary/ledger/storage/postgres" - _ "github.com/numary/ledger/storage/sqlite" ) +var driver storage.Driver + func with(f func(l *Ledger)) { - fx.New( - fx.Option( - fx.NopLogger, - ), + app := fx.New( + fx.NopLogger, + fx.Provide(func() storage.Driver { + return driver + }), + fx.Invoke(func(d storage.Driver) error { + return d.Initialize(context.Background()) + }), + fx.Provide(storage.NewDefaultFactory), fx.Provide( func(storageFactory storage.Factory) (*Ledger, error) { store, err := storageFactory.GetStore("test") if err != nil { return nil, err } + err = store.Initialize(context.Background()) + if err != nil { + return nil, err + } l, err := NewLedger("test", store, NewInMemoryLocker()) if err != nil { panic(err) @@ -53,45 +58,56 @@ func with(f func(l *Ledger)) { logrus.Error(err) } }), + // Closing the driver after each test cause a test to fail + // Tests seems not independent + //fx.Invoke(func(d storage.Driver) error { + // return d.Close(context.Background()) + //}), ) + if app.Err() != nil { + panic(app.Err()) + } } func TestMain(m *testing.M) { + var ( + code int + ) + defer func() { + os.Exit(code) // os.Exit don't care about defer so defer the os.Exit allow us to execute other defer + }() + flag.Parse() if testing.Verbose() { logrus.StandardLogger().Level = logrus.DebugLevel } switch os.Getenv("NUMARY_STORAGE_DRIVER") { - case "sqlite": - storage.RegisterDriver("sqlite", sqlite.NewDriver(os.TempDir(), "ledger")) - err := os.Remove(path.Join(os.TempDir(), "ledger_test.db")) - if err != nil { - panic(err) - } + case "sqlite", "": + driver = sqlstorage.NewInMemorySQLiteDriver() case "postgres": - connString := os.Getenv("NUMARY_STORAGE_POSTGRES_CONN_STRING") - storage.RegisterDriver("postgres", postgres.NewDriver(connString)) - - // @gfyrag: Why this test? - pool, err := pgxpool.Connect(context.Background(), connString) - if err != nil { - panic(err) - } - store, err := postgres.NewStore("test", pool) + pgServer, err := ledgertesting.PostgresServer() if err != nil { - panic(err) + fmt.Fprintln(os.Stderr, err) + os.Exit(1) } - store.DropTest() + defer pgServer.Close() + + driver = sqlstorage.NewOpenCloseDBDriver( + "postgres", + sqlstorage.PostgreSQL, + func(name string) string { + return pgServer.ConnString() + }, + ) } - m.Run() + code = m.Run() } func TestTransaction(t *testing.T) { with(func(l *Ledger) { - testsize := 1e4 total := 0 batch := []core.Transaction{} @@ -215,12 +231,15 @@ func TestLast(t *testing.T) { func TestAccountMetadata(t *testing.T) { with(func(l *Ledger) { - l.SaveMeta(context.Background(), "account", "users:001", core.Metadata{ + err := l.SaveMeta(context.Background(), "account", "users:001", core.Metadata{ "a random metadata": json.RawMessage(`"old value"`), }) - l.SaveMeta(context.Background(), "account", "users:001", core.Metadata{ + assert.NoError(t, err) + + err = l.SaveMeta(context.Background(), "account", "users:001", core.Metadata{ "a random metadata": json.RawMessage(`"new value"`), }) + assert.NoError(t, err) { acc, err := l.GetAccount(context.Background(), "users:001") @@ -242,7 +261,6 @@ func TestAccountMetadata(t *testing.T) { { cursor, err := l.FindAccounts(context.Background(), query.Account("users:001")) - if err != nil { t.Fatal(err) } @@ -271,7 +289,6 @@ func TestAccountMetadata(t *testing.T) { func TestTransactionMetadata(t *testing.T) { with(func(l *Ledger) { - l.Commit(context.Background(), []core.Transaction{{ Postings: []core.Posting{ { diff --git a/ledger/resolver.go b/ledger/resolver.go index a8ae6ae33..6f2d846d5 100644 --- a/ledger/resolver.go +++ b/ledger/resolver.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "github.com/numary/ledger/storage" + "github.com/numary/ledger/storage/sqlstorage" "github.com/pkg/errors" "github.com/sirupsen/logrus" "sync" @@ -33,7 +34,7 @@ func WithLocker(locker Locker) ResolveOptionFn { } var DefaultResolverOptions = []ResolverOption{ - WithStorageFactory(&storage.BuiltInFactory{Driver: "sqlite"}), + WithStorageFactory(storage.NewDefaultFactory(sqlstorage.NewInMemorySQLiteDriver())), WithLocker(NewInMemoryLocker()), } @@ -74,6 +75,8 @@ func (r *Resolver) GetLedger(ctx context.Context, name string) (*Ledger, error) } r.lock.Lock() + defer r.lock.Unlock() + _, ok = r.initializedStores[name] if !ok { err = store.Initialize(ctx) @@ -84,7 +87,6 @@ func (r *Resolver) GetLedger(ctx context.Context, name string) (*Ledger, error) } r.initializedStores[name] = struct{}{} } - r.lock.Unlock() ret: return NewLedger(name, store, r.locker) diff --git a/ledgertesting/testing.go b/ledgertesting/testing.go new file mode 100644 index 000000000..eab734324 --- /dev/null +++ b/ledgertesting/testing.go @@ -0,0 +1,71 @@ +package ledgertesting + +import ( + "context" + "github.com/jackc/pgx/v4" + "github.com/ory/dockertest/v3" + "os" + "time" +) + +type PGServer struct { + url string + close func() error +} + +func (s *PGServer) ConnString() string { + return s.url +} + +func (s *PGServer) Close() error { + return s.close() +} + +func PostgresServer() (*PGServer, error) { + + externalConnectionString := os.Getenv("NUMARY_STORAGE_POSTGRES_CONN_STRING") + if externalConnectionString != "" { + return &PGServer{ + url: externalConnectionString, + close: func() error { + return nil + }, + }, nil + } + + pool, err := dockertest.NewPool("") + if err != nil { + return nil, err + } + + resource, err := pool.Run("postgres", "13-alpine", []string{ + "POSTGRES_USER=root", + "POSTGRES_PASSWORD=root", + "POSTGRES_DB=ledger", + }) + if err != nil { + return nil, err + } + + connString := "postgresql://root:root@localhost:" + resource.GetPort("5432/tcp") + "/ledger" + try := time.Duration(0) + delay := 200 * time.Millisecond + for try*delay < 5*time.Second { + conn, err := pgx.Connect(context.Background(), connString) + if err != nil { + try++ + <-time.After(delay) + continue + } + _ = conn.Close(context.Background()) + break + } + + return &PGServer{ + url: "postgresql://root:root@localhost:" + resource.GetPort("5432/tcp") + "/ledger", + close: func() error { + return pool.Purge(resource) + }, + }, nil + +} diff --git a/storage/driver.go b/storage/driver.go index ab7981553..e8ffa0ab5 100644 --- a/storage/driver.go +++ b/storage/driver.go @@ -1,15 +1,12 @@ package storage -import "context" +import ( + "context" +) type Driver interface { Initialize(ctx context.Context) error NewStore(name string) (Store, error) Close(ctx context.Context) error -} - -var drivers = make(map[string]Driver) - -func RegisterDriver(name string, driver Driver) { - drivers[name] = driver + Name() string } diff --git a/storage/factory.go b/storage/factory.go index 72c985ae9..65ec26ec1 100644 --- a/storage/factory.go +++ b/storage/factory.go @@ -2,58 +2,25 @@ package storage import ( "context" - "fmt" ) type Factory interface { GetStore(name string) (Store, error) Close(ctx context.Context) error } -type FactoryFn func(string) (Store, error) - -func (f FactoryFn) GetStore(name string) (Store, error) { - return f(name) -} - -type closeError struct { - errs map[string]error -} - -func (e *closeError) Error() string { - buf := "" - if len(e.errs) == 0 { - return "" - } - for driver, err := range e.errs { - buf += fmt.Sprintf("%s: %s,", driver, err) - } - return buf[:len(buf)-1] -} type BuiltInFactory struct { - Driver string + Driver Driver } func (f *BuiltInFactory) GetStore(name string) (Store, error) { - return GetStore(f.Driver, name) + return f.Driver.NewStore(name) } func (f *BuiltInFactory) Close(ctx context.Context) error { - closeErr := &closeError{ - errs: map[string]error{}, - } - for name, driver := range drivers { - err := driver.Close(ctx) - if err != nil { - closeErr.errs[name] = err - } - } - if len(closeErr.errs) > 0 { - return closeErr - } - return nil + return f.Driver.Close(ctx) } -func NewDefaultFactory(driver string) (Factory, error) { - return &BuiltInFactory{Driver: driver}, nil +func NewDefaultFactory(driver Driver) Factory { + return &BuiltInFactory{Driver: driver} } diff --git a/storage/postgres/accounts.go b/storage/postgres/accounts.go deleted file mode 100644 index 6cf449006..000000000 --- a/storage/postgres/accounts.go +++ /dev/null @@ -1,107 +0,0 @@ -package postgres - -import ( - "context" - - "github.com/huandu/go-sqlbuilder" - "github.com/numary/ledger/core" - "github.com/numary/ledger/ledger/query" -) - -func (s *PGStore) CountAccounts(ctx context.Context) (int64, error) { - var count int64 - - sqlq, _ := sqlbuilder. - Select("count(*)"). - From(s.table("addresses")). - BuildWithFlavor(sqlbuilder.PostgreSQL) - - err := s.Conn().QueryRow( - ctx, - sqlq, - ).Scan(&count) - - return count, err -} - -func (s *PGStore) FindAccounts(ctx context.Context, q query.Query) (query.Cursor, error) { - c := query.Cursor{} - results := []core.Account{} - - queryRem := sqlbuilder.Select("count(*)") - queryRem.From(s.table("addresses")) - - if q.After != "" { - queryRem.Where(queryRem.LessThan("address", q.After)) - } - - sqlRem, args := queryRem.BuildWithFlavor(sqlbuilder.PostgreSQL) - - var remaining int - - err := s.Conn().QueryRow( - ctx, - sqlRem, - args..., - ).Scan(&remaining) - - if err != nil { - return c, err - } - - queryAcc := sqlbuilder. - Select("address"). - From(s.table("addresses")). - GroupBy("address"). - OrderBy("address desc"). - Limit(q.Limit) - - if q.After != "" { - queryAcc.Where(queryAcc.LessThan("address", q.After)) - } - - sqlAcc, args := queryAcc.BuildWithFlavor(sqlbuilder.PostgreSQL) - - rows, err := s.Conn().Query( - ctx, - sqlAcc, - args..., - ) - - if err != nil { - return c, err - } - - for rows.Next() { - var address string - - err := rows.Scan(&address) - - if err != nil { - return c, err - } - - account := core.Account{ - Address: address, - Contract: "default", - } - - meta, err := s.GetMeta(ctx, "account", account.Address) - if err != nil { - return c, err - } - account.Metadata = meta - - results = append(results, account) - } - - total, _ := s.CountAccounts(ctx) - - c.PageSize = q.Limit - c.HasMore = len(results) < remaining - c.Remaining = remaining - len(results) - c.Total = total - c.Data = results - - return c, nil -} diff --git a/storage/postgres/aggregations.go b/storage/postgres/aggregations.go deleted file mode 100644 index 1ac2c43bf..000000000 --- a/storage/postgres/aggregations.go +++ /dev/null @@ -1,99 +0,0 @@ -package postgres - -import ( - "context" - - "github.com/huandu/go-sqlbuilder" -) - -func (s *PGStore) CountMeta(ctx context.Context) (int64, error) { - var count int64 - - sb := sqlbuilder.NewSelectBuilder() - - sb. - Select("count(*)"). - From(s.table("metadata")) - - sqlq, args := sb.BuildWithFlavor(sqlbuilder.PostgreSQL) - - err := s.Conn().QueryRow(ctx, sqlq, args...).Scan(&count) - - return count, err -} - -func (s *PGStore) AggregateBalances(ctx context.Context, address string) (map[string]int64, error) { - balances := map[string]int64{} - - volumes, err := s.AggregateVolumes(ctx, address) - - if err != nil { - return balances, err - } - - for asset := range volumes { - balances[asset] = volumes[asset]["input"] - volumes[asset]["output"] - } - - return balances, nil -} - -func (s *PGStore) AggregateVolumes(ctx context.Context, address string) (map[string]map[string]int64, error) { - volumes := map[string]map[string]int64{} - - agg1 := sqlbuilder.NewSelectBuilder() - agg1. - Select("asset", "'_out'", "sum(amount)"). - From(s.table("postings")).Where(agg1.Equal("source", address)). - GroupBy("asset") - - agg2 := sqlbuilder.NewSelectBuilder() - agg2. - Select("asset", "'_in'", "sum(amount)"). - From(s.table("postings")).Where(agg2.Equal("destination", address)). - GroupBy("asset") - - union := sqlbuilder.Union(agg1, agg2) - - sb := sqlbuilder.NewSelectBuilder() - sb.Select("*") - sb.From(sb.BuilderAs(union, "assets")) - - sqlq, args := sb.BuildWithFlavor(sqlbuilder.PostgreSQL) - - rows, err := s.Conn().Query( - ctx, - sqlq, - args..., - ) - - if err != nil { - return volumes, err - } - - for rows.Next() { - var row = struct { - asset string - t string - amount int64 - }{} - - err := rows.Scan(&row.asset, &row.t, &row.amount) - - if err != nil { - return volumes, err - } - - if _, ok := volumes[row.asset]; !ok { - volumes[row.asset] = map[string]int64{} - } - - if row.t == "_out" { - volumes[row.asset]["output"] += row.amount - } else { - volumes[row.asset]["input"] += row.amount - } - } - - return volumes, nil -} diff --git a/storage/postgres/driver.go b/storage/postgres/driver.go deleted file mode 100644 index ae2f0c202..000000000 --- a/storage/postgres/driver.go +++ /dev/null @@ -1,52 +0,0 @@ -package postgres - -import ( - "context" - "github.com/jackc/pgx/v4/pgxpool" - "github.com/numary/ledger/storage" - "github.com/sirupsen/logrus" - "sync" -) - -type Driver struct { - once sync.Once - pool *pgxpool.Pool - connString string -} - -func (d *Driver) Initialize(ctx context.Context) error { - errCh := make(chan error, 1) - d.once.Do(func() { - logrus.Debugln("initiating postgres pool") - - pool, err := pgxpool.Connect(ctx, d.connString) - if err != nil { - errCh <- err - } - d.pool = pool - errCh <- nil - }) - select { - case err := <-errCh: - return err - default: - return nil - } -} - -func (d *Driver) NewStore(name string) (storage.Store, error) { - return NewStore(name, d.pool) -} - -func (d *Driver) Close(ctx context.Context) error { - if d.pool != nil { - d.pool.Close() - } - return nil -} - -func NewDriver(connString string) *Driver { - return &Driver{ - connString: connString, - } -} diff --git a/storage/postgres/metadata.go b/storage/postgres/metadata.go deleted file mode 100644 index 1e0b041ab..000000000 --- a/storage/postgres/metadata.go +++ /dev/null @@ -1,112 +0,0 @@ -package postgres - -import ( - "context" - "encoding/json" - "github.com/sirupsen/logrus" - - "github.com/huandu/go-sqlbuilder" - "github.com/numary/ledger/core" -) - -func (s *PGStore) GetMeta(ctx context.Context, ty string, id string) (core.Metadata, error) { - sb := sqlbuilder.NewSelectBuilder() - sb.Select( - "meta_key", - "meta_value", - ) - sb.From(s.table("metadata")) - sb.Where( - sb.And( - sb.Equal("meta_target_type", ty), - sb.Equal("meta_target_id", id), - ), - ) - - sqlq, args := sb.BuildWithFlavor(sqlbuilder.PostgreSQL) - logrus.Debugln(sqlq, args) - - rows, err := s.Conn().Query( - ctx, - sqlq, - args..., - ) - - if err != nil { - return nil, err - } - - meta := core.Metadata{} - - for rows.Next() { - var metaKey string - var metaValue string - - err := rows.Scan( - &metaKey, - &metaValue, - ) - - if err != nil { - return nil, err - } - - var value json.RawMessage - - err = json.Unmarshal([]byte(metaValue), &value) - - if err != nil { - return nil, err - } - - meta[metaKey] = value - } - - return meta, nil -} - -func (s *PGStore) SaveMeta(ctx context.Context, id int64, timestamp, targetType, targetID, key, value string) error { - tx, _ := s.Conn().Begin(ctx) - - ib := sqlbuilder.NewInsertBuilder() - ib.InsertInto(s.table("metadata")) - ib.Cols( - "meta_id", - "meta_target_type", - "meta_target_id", - "meta_key", - "meta_value", - "timestamp", - ) - ib.Values(id, targetType, targetID, key, string(value), timestamp) - - sqlq, args := ib.BuildWithFlavor(sqlbuilder.PostgreSQL) - - _, err := tx.Exec( - ctx, - sqlq, - args..., - ) - - if err != nil { - tx.Rollback(ctx) - - return err - } - - err = tx.Commit(ctx) - - if err != nil { - return err - } - - return nil -} - -func (s *PGStore) LastMetaID(ctx context.Context) (int64, error) { - count, err := s.CountMeta(ctx) - if err != nil { - return 0, err - } - return count - 1, nil -} diff --git a/storage/postgres/store.go b/storage/postgres/store.go deleted file mode 100644 index d36dba012..000000000 --- a/storage/postgres/store.go +++ /dev/null @@ -1,92 +0,0 @@ -package postgres - -import ( - "context" - "embed" - "fmt" - "github.com/sirupsen/logrus" - "path" - "strings" - - "github.com/jackc/pgx/v4/pgxpool" -) - -//go:embed migration -var migrations embed.FS - -type PGStore struct { - ledger string - pool *pgxpool.Pool -} - -func (s *PGStore) Conn() *pgxpool.Pool { - return s.pool -} - -func NewStore(name string, pool *pgxpool.Pool) (*PGStore, error) { - return &PGStore{ - ledger: name, - pool: pool, - }, nil -} - -func (s *PGStore) Name() string { - return s.ledger -} - -func (s *PGStore) Initialize(ctx context.Context) error { - statements := []string{} - - entries, err := migrations.ReadDir("migration") - - if err != nil { - return err - } - - for _, m := range entries { - logrus.Debugf("running migration %s\n", m.Name()) - - b, err := migrations.ReadFile(path.Join("migration", m.Name())) - - if err != nil { - return err - } - - plain := strings.ReplaceAll(string(b), "VAR_LEDGER_NAME", s.ledger) - - statements = append( - statements, - strings.Split(plain, "--statement")..., - ) - } - - for i, statement := range statements { - _, err = s.Conn().Exec( - ctx, - statement, - ) - - if err != nil { - err = fmt.Errorf("failed to run statement %d: %w", i, err) - logrus.Errorf("error running statement: %s", err) - return err - } - } - - return nil -} - -func (s *PGStore) table(name string) string { - return fmt.Sprintf(`"%s"."%s"`, s.ledger, name) -} - -func (s *PGStore) Close(ctx context.Context) error { - return nil -} - -func (s *PGStore) DropTest() { - s.Conn().Exec( - context.Background(), - "DROP SCHEMA test CASCADE", - ) -} diff --git a/storage/postgres/transactions.go b/storage/postgres/transactions.go deleted file mode 100644 index 3ecfa223b..000000000 --- a/storage/postgres/transactions.go +++ /dev/null @@ -1,338 +0,0 @@ -package postgres - -import ( - "context" - "database/sql" - "fmt" - "github.com/sirupsen/logrus" - "math" - "sort" - - "github.com/huandu/go-sqlbuilder" - "github.com/numary/ledger/core" - "github.com/numary/ledger/ledger/query" -) - -func (s *PGStore) SaveTransactions(ctx context.Context, ts []core.Transaction) error { - tx, _ := s.Conn().Begin(context.Background()) - - for _, t := range ts { - var ref *string - - if t.Reference != "" { - ref = &t.Reference - } - - ib := sqlbuilder.NewInsertBuilder() - ib.InsertInto(s.table("transactions")) - ib.Cols("id", "reference", "timestamp", "hash") - ib.Values(t.ID, ref, t.Timestamp, t.Hash) - - sqlq, args := ib.BuildWithFlavor(sqlbuilder.PostgreSQL) - - _, err := tx.Exec( - ctx, - sqlq, - args..., - ) - - if err != nil { - tx.Rollback(ctx) - - return err - } - - for i, p := range t.Postings { - ib := sqlbuilder.NewInsertBuilder() - ib.InsertInto(s.table("postings")) - ib.Cols("id", "txid", "source", "destination", "amount", "asset") - ib.Values(i, t.ID, p.Source, p.Destination, p.Amount, p.Asset) - - sqlq, args := ib.BuildWithFlavor(sqlbuilder.PostgreSQL) - - _, err := tx.Exec( - ctx, - // `INSERT INTO "postings" - // ("id", "txid", "source", "destination", "amount", "asset") - // VALUES - // ($1, $2, $3, $4, $5, $6)`, - // i, - // t.ID, - // p.Source, - // p.Destination, - // p.Amount, - // p.Asset, - sqlq, - args..., - ) - - if err != nil { - tx.Rollback(ctx) - - return err - } - } - - nextID, err := s.CountMeta(ctx) - if err != nil { - tx.Rollback(ctx) - - return err - } - - for key, value := range t.Metadata { - ib := sqlbuilder.NewInsertBuilder() - ib.InsertInto(s.table("metadata")) - ib.Cols( - "meta_id", - "meta_target_type", - "meta_target_id", - "meta_key", - "meta_value", - "timestamp", - ) - ib.Values( - int(nextID), - "transaction", - fmt.Sprintf("%d", t.ID), - key, - string(value), - t.Timestamp, - ) - - sqlq, args := ib.BuildWithFlavor(sqlbuilder.PostgreSQL) - - _, err = tx.Exec(ctx, sqlq, args...) - - if err != nil { - tx.Rollback(ctx) - - return err - } - - nextID++ - } - } - - return tx.Commit(ctx) -} - -func (s *PGStore) CountTransactions(ctx context.Context) (int64, error) { - var count int64 - - sb := sqlbuilder.NewSelectBuilder() - sb.Select("count(*)") - sb.From(s.table("transactions")) - - sqlq, args := sb.BuildWithFlavor(sqlbuilder.PostgreSQL) - - err := s.Conn().QueryRow( - ctx, - sqlq, - args..., - ).Scan(&count) - - return count, err -} - -func (s *PGStore) FindTransactions(ctx context.Context, q query.Query) (query.Cursor, error) { - q.Limit = int(math.Max(-1, math.Min(float64(q.Limit), 100))) - - c := query.Cursor{} - results := []core.Transaction{} - - in := sqlbuilder.NewSelectBuilder() - in.Select("txid").From(s.table("postings")) - in.GroupBy("txid") - in.OrderBy("txid desc") - in.Limit(q.Limit) - - if q.After != "" { - in.Where(in.LessThan("txid", q.After)) - } - - if q.HasParam("account") { - in.Where(in.Or( - in.Equal("source", q.Params["account"]), - in.Equal("destination", q.Params["account"]), - )) - } - - if q.HasParam("reference") { - in.Where( - in.Equal("reference", q.Params["reference"]), - ) - } - - sb := sqlbuilder.NewSelectBuilder() - sb.Select( - "t.id", - "t.timestamp", - "t.hash", - "t.reference", - "p.source", - "p.destination", - "p.amount", - "p.asset", - ) - sb.From(sb.As(s.table("transactions"), "t")) - sb.Where(sb.In("t.id", in)) - sb.JoinWithOption(sqlbuilder.LeftJoin, sb.As(s.table("postings"), "p"), "p.txid = t.id") - sb.OrderBy("t.id desc, p.id asc") - - sqlq, args := sb.BuildWithFlavor(sqlbuilder.PostgreSQL) - - rows, err := s.Conn().Query( - ctx, - sqlq, - args..., - ) - - if err != nil { - return c, err - } - - transactions := map[int64]core.Transaction{} - - for rows.Next() { - var txid int64 - var ts string - var thash string - var tref sql.NullString - - posting := core.Posting{} - - rows.Scan( - &txid, - &ts, - &thash, - &tref, - &posting.Source, - &posting.Destination, - &posting.Amount, - &posting.Asset, - ) - - if _, ok := transactions[txid]; !ok { - transactions[txid] = core.Transaction{ - ID: txid, - Postings: []core.Posting{}, - Timestamp: ts, - Hash: thash, - Reference: tref.String, - Metadata: core.Metadata{}, - } - } - - t := transactions[txid] - t.AppendPosting(posting) - transactions[txid] = t - } - - for _, t := range transactions { - meta, err := s.GetMeta(ctx, "transaction", fmt.Sprintf("%d", t.ID)) - if err != nil { - return c, err - } - t.Metadata = meta - - results = append(results, t) - } - - sort.Slice(results, func(i, j int) bool { - return results[i].ID > results[j].ID - }) - - c.Data = results - - return c, nil -} - -func (s *PGStore) GetTransaction(ctx context.Context, txid string) (tx core.Transaction, err error) { - sb := sqlbuilder.NewSelectBuilder() - sb.Select( - "t.id", - "t.timestamp", - "t.hash", - "t.reference", - "p.source", - "p.destination", - "p.amount", - "p.asset", - ) - sb.From(sb.As(s.table("transactions"), "t")) - sb.Where(sb.Equal("t.id", txid)) - sb.JoinWithOption(sqlbuilder.LeftJoin, sb.As(s.table("postings"), "p"), "p.txid = t.id") - sb.OrderBy("p.id asc") - - sqlq, args := sb.BuildWithFlavor(sqlbuilder.PostgreSQL) - logrus.Debugln(sqlq, args) - - rows, err := s.Conn().Query( - ctx, - sqlq, - args..., - ) - - if err != nil { - return tx, err - } - - for rows.Next() { - var txid int64 - var ts string - var thash string - var tref sql.NullString - - posting := core.Posting{} - - err := rows.Scan( - &txid, - &ts, - &thash, - &tref, - &posting.Source, - &posting.Destination, - &posting.Amount, - &posting.Asset, - ) - if err != nil { - return tx, err - } - - tx.ID = txid - tx.Timestamp = ts - tx.Hash = thash - tx.Metadata = core.Metadata{} - tx.Reference = tref.String - - tx.AppendPosting(posting) - } - - meta, err := s.GetMeta(ctx, "transaction", fmt.Sprintf("%d", tx.ID)) - if err != nil { - return tx, err - } - tx.Metadata = meta - - return tx, nil -} - -func (s *PGStore) LastTransaction(ctx context.Context) (*core.Transaction, error) { - var lastTransaction core.Transaction - - q := query.New() - q.Modify(query.Limit(1)) - - c, err := s.FindTransactions(ctx, q) - if err != nil { - return nil, err - } - - txs := (c.Data).([]core.Transaction) - if len(txs) > 0 { - lastTransaction = txs[0] - return &lastTransaction, nil - } - return nil, nil -} diff --git a/storage/sqlite/driver.go b/storage/sqlite/driver.go deleted file mode 100644 index cdee68fad..000000000 --- a/storage/sqlite/driver.go +++ /dev/null @@ -1,30 +0,0 @@ -package sqlite - -import ( - "context" - "github.com/numary/ledger/storage" -) - -type Driver struct { - storageDir string - dbName string -} - -func (d *Driver) Initialize(ctx context.Context) error { - return nil -} - -func (d *Driver) NewStore(name string) (storage.Store, error) { - return NewStore(d.storageDir, d.dbName, name) -} - -func (d *Driver) Close(ctx context.Context) error { - return nil -} - -func NewDriver(storageDir, dbName string) *Driver { - return &Driver{ - storageDir: storageDir, - dbName: dbName, - } -} diff --git a/storage/sqlite/store.go b/storage/sqlite/store.go deleted file mode 100644 index 3dbccf4fb..000000000 --- a/storage/sqlite/store.go +++ /dev/null @@ -1,98 +0,0 @@ -package sqlite - -import ( - "context" - "database/sql" - "embed" - "fmt" - "github.com/sirupsen/logrus" - "path" - "strings" - - _ "github.com/mattn/go-sqlite3" -) - -//go:embed migration -var migrations embed.FS - -type SQLiteStore struct { - ledger string - db *sql.DB -} - -func NewStore(storageDir, dbName, name string) (*SQLiteStore, error) { - dbpath := fmt.Sprintf( - "file:%s?_journal=WAL", - path.Join( - storageDir, - fmt.Sprintf("%s_%s.db", dbName, name), - ), - ) - - logrus.Debugf("opening %s\n", dbpath) - - db, err := sql.Open("sqlite3", dbpath) - - if err != nil { - return nil, err - } - - return &SQLiteStore{ - ledger: name, - db: db, - }, nil -} - -func (s *SQLiteStore) Name() string { - return s.ledger -} - -func (s *SQLiteStore) Initialize(ctx context.Context) error { - logrus.Debugln("initializing sqlite db") - - statements := []string{} - - entries, err := migrations.ReadDir("migration") - - if err != nil { - return err - } - - for _, m := range entries { - logrus.Debugf("running migration %s\n", m.Name()) - - b, err := migrations.ReadFile(path.Join("migration", m.Name())) - - if err != nil { - return err - } - - plain := strings.ReplaceAll(string(b), "VAR_LEDGER_NAME", s.ledger) - - statements = append( - statements, - strings.Split(plain, "--statement")..., - ) - } - - for i, statement := range statements { - _, err = s.db.ExecContext(ctx, statement) - - if err != nil { - err = fmt.Errorf("failed to run statement %d: %w", i, err) - logrus.Errorln(err) - return err - } - } - - return nil -} - -func (s *SQLiteStore) Close(ctx context.Context) error { - logrus.Debugln("sqlite db closed") - err := s.db.Close() - if err != nil { - return err - } - return nil -} diff --git a/storage/sqlite/accounts.go b/storage/sqlstorage/accounts.go similarity index 80% rename from storage/sqlite/accounts.go rename to storage/sqlstorage/accounts.go index 8484d8371..cb02bd6f4 100644 --- a/storage/sqlite/accounts.go +++ b/storage/sqlstorage/accounts.go @@ -1,4 +1,4 @@ -package sqlite +package sqlstorage import ( "context" @@ -10,16 +10,17 @@ import ( "github.com/numary/ledger/ledger/query" ) -func (s *SQLiteStore) FindAccounts(ctx context.Context, q query.Query) (query.Cursor, error) { +func (s *Store) FindAccounts(ctx context.Context, q query.Query) (query.Cursor, error) { + // We fetch an additional account to know if we have more documents q.Limit = int(math.Max(-1, math.Min(float64(q.Limit), 100))) + 1 c := query.Cursor{} - results := []core.Account{} + results := make([]core.Account, 0) sb := sqlbuilder.NewSelectBuilder() sb. Select("address"). - From("addresses"). + From(s.table("addresses")). GroupBy("address"). OrderBy("address desc"). Limit(q.Limit) @@ -28,7 +29,7 @@ func (s *SQLiteStore) FindAccounts(ctx context.Context, q query.Query) (query.Cu sb.Where(sb.LessThan("address", q.After)) } - sqlq, args := sb.BuildWithFlavor(sqlbuilder.SQLite) + sqlq, args := sb.BuildWithFlavor(s.flavor) logrus.Debugln(sqlq, args) rows, err := s.db.QueryContext( diff --git a/storage/sqlite/aggregations.go b/storage/sqlstorage/aggregations.go similarity index 68% rename from storage/sqlite/aggregations.go rename to storage/sqlstorage/aggregations.go index dd501e900..a288f820f 100644 --- a/storage/sqlite/aggregations.go +++ b/storage/sqlstorage/aggregations.go @@ -1,16 +1,17 @@ -package sqlite +package sqlstorage import ( "context" "github.com/huandu/go-sqlbuilder" + "github.com/sirupsen/logrus" ) -func (s *SQLiteStore) CountTransactions(ctx context.Context) (int64, error) { +func (s *Store) CountTransactions(ctx context.Context) (int64, error) { var count int64 sb := sqlbuilder.NewSelectBuilder() sb.Select("count(*)") - sb.From("transactions") + sb.From(s.table("transactions")) sqlq, args := sb.Build() @@ -19,15 +20,15 @@ func (s *SQLiteStore) CountTransactions(ctx context.Context) (int64, error) { return count, err } -func (s *SQLiteStore) CountAccounts(ctx context.Context) (int64, error) { +func (s *Store) CountAccounts(ctx context.Context) (int64, error) { var count int64 sb := sqlbuilder.NewSelectBuilder() sb. Select("count(*)"). - From("addresses"). - BuildWithFlavor(sqlbuilder.SQLite) + From(s.table("addresses")). + BuildWithFlavor(s.flavor) sqlq, args := sb.Build() @@ -36,16 +37,17 @@ func (s *SQLiteStore) CountAccounts(ctx context.Context) (int64, error) { return count, err } -func (s *SQLiteStore) CountMeta(ctx context.Context) (int64, error) { +func (s *Store) CountMeta(ctx context.Context) (int64, error) { var count int64 sb := sqlbuilder.NewSelectBuilder() sb. Select("count(*)"). - From("metadata") + From(s.table("metadata")) - sqlq, args := sb.BuildWithFlavor(sqlbuilder.SQLite) + sqlq, args := sb.BuildWithFlavor(s.flavor) + logrus.StandardLogger().Debugln(sqlq) q := s.db.QueryRowContext(ctx, sqlq, args...) err := q.Scan(&count) @@ -53,7 +55,7 @@ func (s *SQLiteStore) CountMeta(ctx context.Context) (int64, error) { return count, err } -func (s *SQLiteStore) AggregateBalances(ctx context.Context, address string) (map[string]int64, error) { +func (s *Store) AggregateBalances(ctx context.Context, address string) (map[string]int64, error) { balances := map[string]int64{} volumes, err := s.AggregateVolumes(ctx, address) @@ -69,19 +71,19 @@ func (s *SQLiteStore) AggregateBalances(ctx context.Context, address string) (ma return balances, nil } -func (s *SQLiteStore) AggregateVolumes(ctx context.Context, address string) (map[string]map[string]int64, error) { +func (s *Store) AggregateVolumes(ctx context.Context, address string) (map[string]map[string]int64, error) { volumes := map[string]map[string]int64{} agg1 := sqlbuilder.NewSelectBuilder() agg1. Select("asset", "'_out'", "sum(amount)"). - From("postings").Where(agg1.Equal("source", address)). + From(s.table("postings")).Where(agg1.Equal("source", address)). GroupBy("asset") agg2 := sqlbuilder.NewSelectBuilder() agg2. Select("asset", "'_in'", "sum(amount)"). - From("postings").Where(agg2.Equal("destination", address)). + From(s.table("postings")).Where(agg2.Equal("destination", address)). GroupBy("asset") union := sqlbuilder.Union(agg1, agg2) @@ -90,7 +92,7 @@ func (s *SQLiteStore) AggregateVolumes(ctx context.Context, address string) (map sb.Select("*") sb.From(sb.BuilderAs(union, "assets")) - sqlq, args := sb.BuildWithFlavor(sqlbuilder.SQLite) + sqlq, args := sb.BuildWithFlavor(s.flavor) rows, err := s.db.QueryContext(ctx, sqlq, args...) diff --git a/storage/sqlstorage/driver.go b/storage/sqlstorage/driver.go new file mode 100644 index 000000000..09f996e5b --- /dev/null +++ b/storage/sqlstorage/driver.go @@ -0,0 +1,140 @@ +package sqlstorage + +import ( + "context" + "database/sql" + "errors" + "fmt" + "github.com/huandu/go-sqlbuilder" + "github.com/numary/ledger/storage" +) + +type Flavor = sqlbuilder.Flavor + +var ( + SQLite = sqlbuilder.SQLite + PostgreSQL = sqlbuilder.PostgreSQL +) + +var sqlDrivers = map[Flavor]struct { + driverName string +}{ + SQLite: { + driverName: "sqlite3", + }, + PostgreSQL: { + driverName: "pgx", + }, +} + +type ConnStringResolver func(name string) string + +// openCloseDBDriver is a driver which connect on the database each time the NewStore() method is called. +// Therefore, the provided store is configured to close the *sql.DB instance when the Close() method of the store is called. +// It is suitable for databases engines like SQLite +type openCloseDBDriver struct { + name string + connString ConnStringResolver + flavor Flavor +} + +func (d *openCloseDBDriver) Name() string { + return d.name +} + +func (d *openCloseDBDriver) Initialize(ctx context.Context) error { + return nil +} + +func (d *openCloseDBDriver) NewStore(name string) (storage.Store, error) { + cfg, ok := sqlDrivers[d.flavor] + if !ok { + return nil, fmt.Errorf("unsupported flavor %s", d.flavor) + } + db, err := sql.Open(cfg.driverName, d.connString(name)) + if err != nil { + return nil, err + } + return NewStore(name, d.flavor, db, func(ctx context.Context) error { + return db.Close() + }) +} + +func (d *openCloseDBDriver) Close(ctx context.Context) error { + return nil +} + +func NewOpenCloseDBDriver(name string, flavor Flavor, connString ConnStringResolver) *openCloseDBDriver { + return &openCloseDBDriver{ + flavor: flavor, + connString: connString, + name: name, + } +} + +// cachedDBDriver is a driver which connect on a database and keep the connection open until closed +// it suitable for databases engines like PostgreSQL or MySQL +// Therefore, the NewStore() method return stores backed with the same underlying *sql.DB instance. +type cachedDBDriver struct { + name string + where string + db *sql.DB + flavor Flavor +} + +func (s *cachedDBDriver) Name() string { + return s.name +} + +func (s *cachedDBDriver) Initialize(ctx context.Context) error { + + cfg, ok := sqlDrivers[s.flavor] + if !ok { + return errors.New("unknown flavor") + } + + db, err := sql.Open(cfg.driverName, s.where) + if err != nil { + return err + } + s.db = db + return nil +} + +func (s *cachedDBDriver) NewStore(name string) (storage.Store, error) { + return NewStore(name, s.flavor, s.db, func(ctx context.Context) error { + return nil + }) +} + +func (d *cachedDBDriver) Close(ctx context.Context) error { + if d.db == nil { + return nil + } + return d.db.Close() +} + +const SQLiteMemoryConnString = "file::memory:?cache=shared" + +func SQLiteFileConnString(path string) string { + return fmt.Sprintf( + "file:%s?_journal=WAL", + path, + ) +} + +func NewCachedDBDriver(name string, flavor Flavor, where string) *cachedDBDriver { + return &cachedDBDriver{ + where: where, + name: name, + flavor: flavor, + } +} + +func NewInMemorySQLiteDriver() *cachedDBDriver { + return &cachedDBDriver{ + where: SQLiteMemoryConnString, + flavor: SQLite, + name: "sqlite", + } +} diff --git a/storage/sqlstorage/driver_test.go b/storage/sqlstorage/driver_test.go new file mode 100644 index 000000000..00611e063 --- /dev/null +++ b/storage/sqlstorage/driver_test.go @@ -0,0 +1,45 @@ +package sqlstorage + +import ( + "context" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestNewOpenCloseDBDriver(t *testing.T) { + d := NewOpenCloseDBDriver("sqlite", SQLite, func(name string) string { + return SQLiteMemoryConnString + }) + err := d.Initialize(context.Background()) + assert.NoError(t, err) + defer d.Close(context.Background()) + + store, err := d.NewStore("foo") + assert.NoError(t, err) + + err = store.Initialize(context.Background()) + assert.NoError(t, err) + + store.Close(context.Background()) + + _, err = store.(*Store).db.Query("select * from transactions") + assert.NotNil(t, err) + assert.Equal(t, "sql: database is closed", err.Error()) +} + +func TestNewCachedDBDriver(t *testing.T) { + d := NewCachedDBDriver("sqlite", SQLite, SQLiteMemoryConnString) + err := d.Initialize(context.Background()) + assert.NoError(t, err) + defer d.Close(context.Background()) + + store, err := d.NewStore("foo") + assert.NoError(t, err) + store.Close(context.Background()) + + err = store.Initialize(context.Background()) + assert.NoError(t, err) + + _, err = store.(*Store).db.Query("select * from transactions") + assert.NoError(t, err, "database should have been closed") +} diff --git a/storage/sqlite/metadata.go b/storage/sqlstorage/metadata.go similarity index 70% rename from storage/sqlite/metadata.go rename to storage/sqlstorage/metadata.go index 1ad75c41a..a52deef5f 100644 --- a/storage/sqlite/metadata.go +++ b/storage/sqlstorage/metadata.go @@ -1,4 +1,4 @@ -package sqlite +package sqlstorage import ( "context" @@ -8,7 +8,7 @@ import ( "github.com/sirupsen/logrus" ) -func (s *SQLiteStore) LastMetaID(ctx context.Context) (int64, error) { +func (s *Store) LastMetaID(ctx context.Context) (int64, error) { count, err := s.CountMeta(ctx) if err != nil { return 0, err @@ -16,13 +16,13 @@ func (s *SQLiteStore) LastMetaID(ctx context.Context) (int64, error) { return count - 1, nil } -func (s *SQLiteStore) GetMeta(ctx context.Context, ty string, id string) (core.Metadata, error) { +func (s *Store) GetMeta(ctx context.Context, ty string, id string) (core.Metadata, error) { sb := sqlbuilder.NewSelectBuilder() sb.Select( "meta_key", "meta_value", ) - sb.From("metadata") + sb.From(s.table("metadata")) sb.Where( sb.And( sb.Equal("meta_target_type", ty), @@ -30,7 +30,7 @@ func (s *SQLiteStore) GetMeta(ctx context.Context, ty string, id string) (core.M ), ) - sqlq, args := sb.BuildWithFlavor(sqlbuilder.SQLite) + sqlq, args := sb.BuildWithFlavor(s.flavor) logrus.Debugln(sqlq, args) rows, err := s.db.QueryContext(ctx, sqlq, args...) @@ -68,11 +68,14 @@ func (s *SQLiteStore) GetMeta(ctx context.Context, ty string, id string) (core.M return meta, nil } -func (s *SQLiteStore) SaveMeta(ctx context.Context, id int64, timestamp, targetType, targetID, key, value string) error { - tx, _ := s.db.Begin() +func (s *Store) SaveMeta(ctx context.Context, id int64, timestamp, targetType, targetID, key, value string) error { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return err + } ib := sqlbuilder.NewInsertBuilder() - ib.InsertInto("metadata") + ib.InsertInto(s.table("metadata")) ib.Cols( "meta_id", "meta_target_type", @@ -90,11 +93,10 @@ func (s *SQLiteStore) SaveMeta(ctx context.Context, id int64, timestamp, targetT timestamp, ) - sqlq, args := ib.BuildWithFlavor(sqlbuilder.SQLite) + sqlq, args := ib.BuildWithFlavor(s.flavor) logrus.Debugln(sqlq, args) - _, err := tx.ExecContext(ctx, sqlq, args...) - + _, err = tx.ExecContext(ctx, sqlq, args...) if err != nil { logrus.Debugln("failed to save metadata", err) tx.Rollback() diff --git a/storage/postgres/migration/v001.sql b/storage/sqlstorage/migrations/postgresql/v001.sql similarity index 100% rename from storage/postgres/migration/v001.sql rename to storage/sqlstorage/migrations/postgresql/v001.sql diff --git a/storage/sqlite/migration/v001.sql b/storage/sqlstorage/migrations/sqlite/v001.sql similarity index 100% rename from storage/sqlite/migration/v001.sql rename to storage/sqlstorage/migrations/sqlite/v001.sql diff --git a/storage/sqlstorage/store.go b/storage/sqlstorage/store.go new file mode 100644 index 000000000..ea6481deb --- /dev/null +++ b/storage/sqlstorage/store.go @@ -0,0 +1,98 @@ +package sqlstorage + +import ( + "context" + "database/sql" + "embed" + "fmt" + "github.com/huandu/go-sqlbuilder" + "github.com/sirupsen/logrus" + "path" + "strings" + + _ "github.com/jackc/pgx/v4/stdlib" + _ "github.com/mattn/go-sqlite3" +) + +//go:embed migrations +var migrations embed.FS + +type Store struct { + flavor sqlbuilder.Flavor + ledger string + db *sql.DB + onClose func(ctx context.Context) error +} + +func (s *Store) table(name string) string { + switch s.flavor { + case sqlbuilder.PostgreSQL: + return fmt.Sprintf(`"%s"."%s"`, s.ledger, name) + default: + return name + } +} + +func NewStore(name string, flavor sqlbuilder.Flavor, db *sql.DB, onClose func(ctx context.Context) error) (*Store, error) { + return &Store{ + ledger: name, + db: db, + flavor: flavor, + onClose: onClose, + }, nil +} + +func (s *Store) Name() string { + return s.ledger +} + +func (s *Store) Initialize(ctx context.Context) error { + logrus.Debugln("initializing sqlite db") + + statements := make([]string, 0) + + migrationsDir := fmt.Sprintf("migrations/%s", strings.ToLower(s.flavor.String())) + + entries, err := migrations.ReadDir(migrationsDir) + + if err != nil { + return err + } + + for _, m := range entries { + logrus.Debugf("running migrations %s\n", m.Name()) + + b, err := migrations.ReadFile(path.Join(migrationsDir, m.Name())) + if err != nil { + return err + } + + plain := strings.ReplaceAll(string(b), "VAR_LEDGER_NAME", s.ledger) + + statements = append( + statements, + strings.Split(plain, "--statement")..., + ) + } + + for i, statement := range statements { + logrus.Debugf("running statement: %s", statement) + _, err = s.db.ExecContext(ctx, statement) + + if err != nil { + err = fmt.Errorf("failed to run statement %d: %w", i, err) + logrus.Errorln(err) + return err + } + } + + return nil +} + +func (s *Store) Close(ctx context.Context) error { + err := s.onClose(ctx) + if err != nil { + return err + } + return nil +} diff --git a/storage/sqlstorage/store_test.go b/storage/sqlstorage/store_test.go new file mode 100644 index 000000000..513bc29d7 --- /dev/null +++ b/storage/sqlstorage/store_test.go @@ -0,0 +1,498 @@ +package sqlstorage + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + "github.com/huandu/go-sqlbuilder" + "github.com/numary/ledger/core" + "github.com/numary/ledger/ledger/query" + "github.com/numary/ledger/ledgertesting" + "github.com/numary/ledger/storage" + "github.com/pborman/uuid" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "os" + "path" + "testing" + "time" +) + +func TestStore(t *testing.T) { + + if testing.Verbose() { + logrus.StandardLogger().Level = logrus.DebugLevel + } + + pgServer, err := ledgertesting.PostgresServer() + assert.NoError(t, err) + defer pgServer.Close() + + type driverConfig struct { + driver string + connString ConnStringResolver + flavor sqlbuilder.Flavor + } + var drivers = []driverConfig{ + { + driver: "sqlite3", + connString: func(name string) string { + return SQLiteFileConnString(path.Join(os.TempDir(), name)) + }, + flavor: sqlbuilder.SQLite, + }, + { + driver: "pgx", + connString: func(name string) string { + return pgServer.ConnString() + }, + flavor: sqlbuilder.PostgreSQL, + }, + } + + type testingFunction struct { + name string + fn func(t *testing.T, store storage.Store) + } + + for _, driver := range drivers { + for _, tf := range []testingFunction{ + { + name: "SaveTransactions", + fn: testSaveTransaction, + }, + { + name: "SaveMeta", + fn: testSaveMeta, + }, + { + name: "LastTransaction", + fn: testLastTransaction, + }, + { + name: "LastMetaID", + fn: testLastMetaID, + }, + { + name: "CountAccounts", + fn: testCountAccounts, + }, + { + name: "AggregateBalances", + fn: testAggregateBalances, + }, + { + name: "AggregateVolumes", + fn: testAggregateVolumes, + }, + { + name: "CountMeta", + fn: testCountMeta, + }, + { + name: "FindAccounts", + fn: testFindAccounts, + }, + { + name: "CountTransactions", + fn: testCountTransactions, + }, + { + name: "FindTransactions", + fn: testFindTransactions, + }, + { + name: "GetMeta", + fn: testGetMeta, + }, + { + name: "GetTransaction", + fn: testGetTransaction, + }, + } { + t.Run(fmt.Sprintf("%s/%s", driver.driver, tf.name), func(t *testing.T) { + ledger := uuid.New() + + db, err := sql.Open(driver.driver, driver.connString(ledger)) + assert.NoError(t, err) + + counter := 0 + for { + err = db.Ping() + if err != nil { + if counter < 5 { + counter++ + <-time.After(time.Second) + continue + } + assert.Fail(t, "timeout waiting database: %s", err) + return + } + break + } + + store, err := NewStore(ledger, driver.flavor, db, func(ctx context.Context) error { + + return db.Close() + }) + assert.NoError(t, err) + defer store.Close(context.Background()) + + err = store.Initialize(context.Background()) + assert.NoError(t, err) + + tf.fn(t, store) + }) + } + } +} + +func testSaveTransaction(t *testing.T, store storage.Store) { + txs := []core.Transaction{ + { + ID: 1, + Postings: []core.Posting{ + { + Source: "world", + Destination: "central_bank", + Amount: 100, + Asset: "USD", + }, + }, + Timestamp: time.Now().Format(time.RFC3339), + }, + } + err := store.SaveTransactions(context.Background(), txs) + assert.NoError(t, err) +} + +func testSaveMeta(t *testing.T, store storage.Store) { + err := store.SaveMeta(context.Background(), 1, time.Now().Format(time.RFC3339), + "transaction", "1", "firstname", "\"YYY\"") + assert.NoError(t, err) +} + +func testGetMeta(t *testing.T, store storage.Store) { + var ( + firstname = "\"John\"" + lastname = "\"Doe\"" + ) + err := store.SaveMeta(context.Background(), 1, time.Now().Format(time.RFC3339), "transaction", "1", "firstname", firstname) + assert.NoError(t, err) + + err = store.SaveMeta(context.Background(), 2, time.Now().Format(time.RFC3339), "transaction", "1", "lastname", lastname) + assert.NoError(t, err) + + meta, err := store.GetMeta(context.TODO(), "transaction", "1") + assert.NoError(t, err) + + assert.Equal(t, core.Metadata{ + "firstname": json.RawMessage(firstname), + "lastname": json.RawMessage(lastname), + }, meta) +} + +func testLastTransaction(t *testing.T, store storage.Store) { + txs := []core.Transaction{ + { + ID: 1, + Postings: []core.Posting{ + { + Source: "world", + Destination: "central_bank", + Amount: 100, + Asset: "USD", + }, + }, + Timestamp: time.Now().Format(time.RFC3339), + }, + } + err := store.SaveTransactions(context.Background(), txs) + assert.NoError(t, err) + + lastTx, err := store.LastTransaction(context.Background()) + assert.NoError(t, err) + assert.NotNil(t, lastTx) + assert.Equal(t, core.Transaction{ + ID: txs[0].ID, + Postings: txs[0].Postings, + Timestamp: txs[0].Timestamp, + Metadata: map[string]json.RawMessage{}, + }, *lastTx) + +} + +func testLastMetaID(t *testing.T, store storage.Store) { + err := store.SaveMeta(context.Background(), 0, time.Now().Format(time.RFC3339), + "transaction", "1", "firstname", "\"YYY\"") + assert.NoError(t, err) + + lastMetaId, err := store.LastMetaID(context.Background()) + assert.NoError(t, err) + assert.EqualValues(t, 0, lastMetaId) +} + +func testCountAccounts(t *testing.T, store storage.Store) { + txs := []core.Transaction{ + { + ID: 1, + Postings: []core.Posting{ + { + Source: "world", + Destination: "central_bank", + Amount: 100, + Asset: "USD", + }, + }, + Timestamp: time.Now().Format(time.RFC3339), + }, + } + err := store.SaveTransactions(context.Background(), txs) + assert.NoError(t, err) + + countAccounts, err := store.CountAccounts(context.Background()) + assert.EqualValues(t, 2, countAccounts) // world + central_bank + +} + +func testAggregateBalances(t *testing.T, store storage.Store) { + txs := []core.Transaction{ + { + ID: 1, + Postings: []core.Posting{ + { + Source: "world", + Destination: "central_bank", + Amount: 100, + Asset: "USD", + }, + }, + Timestamp: time.Now().Format(time.RFC3339), + }, + } + err := store.SaveTransactions(context.Background(), txs) + assert.NoError(t, err) + + balances, err := store.AggregateBalances(context.Background(), "central_bank") + assert.NoError(t, err) + assert.Len(t, balances, 1) + assert.EqualValues(t, 100, balances["USD"]) +} + +func testAggregateVolumes(t *testing.T, store storage.Store) { + txs := []core.Transaction{ + { + ID: 1, + Postings: []core.Posting{ + { + Source: "world", + Destination: "central_bank", + Amount: 100, + Asset: "USD", + }, + }, + Timestamp: time.Now().Format(time.RFC3339), + }, + } + err := store.SaveTransactions(context.Background(), txs) + assert.NoError(t, err) + + volumes, err := store.AggregateVolumes(context.Background(), "central_bank") + assert.NoError(t, err) + assert.Len(t, volumes, 1) + assert.Len(t, volumes["USD"], 1) + assert.EqualValues(t, 100, volumes["USD"]["input"]) +} + +func testFindAccounts(t *testing.T, store storage.Store) { + txs := []core.Transaction{ + { + ID: 1, + Postings: []core.Posting{ + { + Source: "world", + Destination: "central_bank", + Amount: 100, + Asset: "USD", + }, + }, + Timestamp: time.Now().Format(time.RFC3339), + }, + } + err := store.SaveTransactions(context.Background(), txs) + assert.NoError(t, err) + + accounts, err := store.FindAccounts(context.Background(), query.Query{ + Limit: 1, + }) + assert.NoError(t, err) + assert.EqualValues(t, 2, accounts.Total) + assert.True(t, accounts.HasMore) + assert.Equal(t, 1, accounts.PageSize) + + accounts, err = store.FindAccounts(context.Background(), query.Query{ + Limit: 1, + After: accounts.Data.([]core.Account)[0].Address, + }) + assert.NoError(t, err) + assert.EqualValues(t, 2, accounts.Total) + assert.False(t, accounts.HasMore) + assert.Equal(t, 1, accounts.PageSize) +} + +func testCountMeta(t *testing.T, store storage.Store) { + txs := []core.Transaction{ + { + ID: 1, + Postings: []core.Posting{ + { + Source: "world", + Destination: "central_bank", + Amount: 100, + Asset: "USD", + }, + }, + Metadata: map[string]json.RawMessage{ + "lastname": json.RawMessage(`"XXX"`), + }, + Timestamp: time.Now().Format(time.RFC3339), + }, + } + err := store.SaveTransactions(context.Background(), txs) + assert.NoError(t, err) + + err = store.SaveMeta(context.Background(), 1, time.Now().Format(time.RFC3339), + "transaction", "1", "firstname", "\"YYY\"") + assert.NoError(t, err) + + countMeta, err := store.CountMeta(context.Background()) + assert.NoError(t, err) + assert.EqualValues(t, 2, countMeta) +} + +func testCountTransactions(t *testing.T, store storage.Store) { + txs := []core.Transaction{ + { + ID: 1, + Postings: []core.Posting{ + { + Source: "world", + Destination: "central_bank", + Amount: 100, + Asset: "USD", + }, + }, + Metadata: map[string]json.RawMessage{ + "lastname": json.RawMessage(`"XXX"`), + }, + Timestamp: time.Now().Format(time.RFC3339), + }, + } + err := store.SaveTransactions(context.Background(), txs) + assert.NoError(t, err) + + countTransactions, err := store.CountTransactions(context.Background()) + assert.NoError(t, err) + assert.EqualValues(t, 1, countTransactions) +} + +func testFindTransactions(t *testing.T, store storage.Store) { + txs := []core.Transaction{ + { + ID: 0, + Postings: []core.Posting{ + { + Source: "world", + Destination: "central_bank", + Amount: 100, + Asset: "USD", + }, + }, + Timestamp: time.Now().Format(time.RFC3339), + Reference: "tx1", + }, + { + ID: 1, + Postings: []core.Posting{ + { + Source: "world", + Destination: "central_bank", + Amount: 100, + Asset: "USD", + }, + }, + Timestamp: time.Now().Format(time.RFC3339), + Reference: "tx2", + }, + } + err := store.SaveTransactions(context.Background(), txs) + assert.NoError(t, err) + + cursor, err := store.FindTransactions(context.Background(), query.Query{ + Limit: 1, + }) + assert.NoError(t, err) + assert.Equal(t, 1, cursor.PageSize) + assert.True(t, cursor.HasMore) + + cursor, err = store.FindTransactions(context.Background(), query.Query{ + After: fmt.Sprint(cursor.Data.([]core.Transaction)[0].ID), + Limit: 1, + }) + assert.NoError(t, err) + assert.Equal(t, 1, cursor.PageSize) + assert.False(t, cursor.HasMore) + + cursor, err = store.FindTransactions(context.Background(), query.Query{ + Params: map[string]interface{}{ + "account": "world", + "reference": "tx1", + }, + Limit: 1, + }) + assert.NoError(t, err) + assert.Equal(t, 1, cursor.PageSize) + assert.False(t, cursor.HasMore) + +} + +func testGetTransaction(t *testing.T, store storage.Store) { + txs := []core.Transaction{ + { + ID: 0, + Postings: []core.Posting{ + { + Source: "world", + Destination: "central_bank", + Amount: 100, + Asset: "USD", + }, + }, + Timestamp: time.Now().Format(time.RFC3339), + Reference: "tx1", + Metadata: core.Metadata{}, + }, + { + ID: 1, + Postings: []core.Posting{ + { + Source: "world", + Destination: "central_bank", + Amount: 100, + Asset: "USD", + }, + }, + Timestamp: time.Now().Format(time.RFC3339), + Reference: "tx2", + Metadata: core.Metadata{}, + }, + } + err := store.SaveTransactions(context.Background(), txs) + assert.NoError(t, err) + + tx, err := store.GetTransaction(context.Background(), "1") + assert.NoError(t, err) + assert.Equal(t, txs[1], tx) + +} diff --git a/storage/sqlite/transactions.go b/storage/sqlstorage/transactions.go similarity index 80% rename from storage/sqlite/transactions.go rename to storage/sqlstorage/transactions.go index 4ba982f45..476ce3272 100644 --- a/storage/sqlite/transactions.go +++ b/storage/sqlstorage/transactions.go @@ -1,4 +1,4 @@ -package sqlite +package sqlstorage import ( "context" @@ -13,14 +13,14 @@ import ( "github.com/numary/ledger/ledger/query" ) -func (s *SQLiteStore) FindTransactions(ctx context.Context, q query.Query) (query.Cursor, error) { +func (s *Store) FindTransactions(ctx context.Context, q query.Query) (query.Cursor, error) { q.Limit = int(math.Max(-1, math.Min(float64(q.Limit), 100))) + 1 c := query.Cursor{} - results := []core.Transaction{} + results := make([]core.Transaction, 0) in := sqlbuilder.NewSelectBuilder() - in.Select("txid").From("postings") + in.Select("txid").From(s.table("postings")) in.GroupBy("txid") in.OrderBy("txid desc") in.Limit(q.Limit) @@ -53,12 +53,12 @@ func (s *SQLiteStore) FindTransactions(ctx context.Context, q query.Query) (quer "p.amount", "p.asset", ) - sb.From(sb.As("transactions", "t")) + sb.From(sb.As(s.table("transactions"), "t")) sb.Where(sb.In("t.id", in)) - sb.JoinWithOption(sqlbuilder.LeftJoin, sb.As("postings", "p"), "p.txid = t.id") + sb.JoinWithOption(sqlbuilder.LeftJoin, sb.As(s.table("postings"), "p"), "p.txid = t.id") sb.OrderBy("t.id desc, p.id asc") - sqlq, args := sb.BuildWithFlavor(sqlbuilder.SQLite) + sqlq, args := sb.BuildWithFlavor(s.flavor) logrus.Debugln(sqlq, args) rows, err := s.db.QueryContext( @@ -139,8 +139,12 @@ func (s *SQLiteStore) FindTransactions(ctx context.Context, q query.Query) (quer return c, nil } -func (s *SQLiteStore) SaveTransactions(ctx context.Context, ts []core.Transaction) error { - tx, _ := s.db.BeginTx(ctx, nil) +func (s *Store) SaveTransactions(ctx context.Context, ts []core.Transaction) error { + + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return err + } for _, t := range ts { var ref *string @@ -150,14 +154,12 @@ func (s *SQLiteStore) SaveTransactions(ctx context.Context, ts []core.Transactio } ib := sqlbuilder.NewInsertBuilder() - ib.InsertInto("transactions") + ib.InsertInto(s.table("transactions")) ib.Cols("id", "reference", "timestamp", "hash") ib.Values(t.ID, ref, t.Timestamp, t.Hash) - sqlq, args := ib.BuildWithFlavor(sqlbuilder.SQLite) - + sqlq, args := ib.BuildWithFlavor(s.flavor) _, err := tx.ExecContext(ctx, sqlq, args...) - if err != nil { tx.Rollback() @@ -166,11 +168,11 @@ func (s *SQLiteStore) SaveTransactions(ctx context.Context, ts []core.Transactio for i, p := range t.Postings { ib := sqlbuilder.NewInsertBuilder() - ib.InsertInto("postings") + ib.InsertInto(s.table("postings")) ib.Cols("id", "txid", "source", "destination", "amount", "asset") ib.Values(i, t.ID, p.Source, p.Destination, p.Amount, p.Asset) - sqlq, args := ib.BuildWithFlavor(sqlbuilder.SQLite) + sqlq, args := ib.BuildWithFlavor(s.flavor) _, err := tx.ExecContext(ctx, sqlq, args...) @@ -190,7 +192,7 @@ func (s *SQLiteStore) SaveTransactions(ctx context.Context, ts []core.Transactio for key, value := range t.Metadata { ib := sqlbuilder.NewInsertBuilder() - ib.InsertInto("metadata") + ib.InsertInto(s.table("metadata")) ib.Cols( "meta_id", "meta_target_type", @@ -208,11 +210,10 @@ func (s *SQLiteStore) SaveTransactions(ctx context.Context, ts []core.Transactio t.Timestamp, ) - sqlq, args := ib.BuildWithFlavor(sqlbuilder.SQLite) + sqlq, args := ib.BuildWithFlavor(s.flavor) logrus.Debugln(sqlq, args) _, err = tx.ExecContext(ctx, sqlq, args...) - if err != nil { tx.Rollback() @@ -226,7 +227,7 @@ func (s *SQLiteStore) SaveTransactions(ctx context.Context, ts []core.Transactio return tx.Commit() } -func (s *SQLiteStore) GetTransaction(ctx context.Context, txid string) (tx core.Transaction, err error) { +func (s *Store) GetTransaction(ctx context.Context, txid string) (tx core.Transaction, err error) { sb := sqlbuilder.NewSelectBuilder() sb.Select( "t.id", @@ -238,12 +239,12 @@ func (s *SQLiteStore) GetTransaction(ctx context.Context, txid string) (tx core. "p.amount", "p.asset", ) - sb.From(sb.As("transactions", "t")) + sb.From(sb.As(s.table("transactions"), "t")) sb.Where(sb.Equal("t.id", txid)) - sb.JoinWithOption(sqlbuilder.LeftJoin, sb.As("postings", "p"), "p.txid = t.id") + sb.JoinWithOption(sqlbuilder.LeftJoin, sb.As(s.table("postings"), "p"), "p.txid = t.id") sb.OrderBy("p.id asc") - sqlq, args := sb.BuildWithFlavor(sqlbuilder.SQLite) + sqlq, args := sb.BuildWithFlavor(s.flavor) logrus.Debugln(sqlq, args) rows, err := s.db.QueryContext( @@ -296,7 +297,7 @@ func (s *SQLiteStore) GetTransaction(ctx context.Context, txid string) (tx core. return tx, nil } -func (s *SQLiteStore) LastTransaction(ctx context.Context) (*core.Transaction, error) { +func (s *Store) LastTransaction(ctx context.Context) (*core.Transaction, error) { var lastTransaction core.Transaction q := query.New() diff --git a/storage/storage.go b/storage/storage.go index ff12ee259..c3255356d 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -4,7 +4,6 @@ import ( "context" "github.com/numary/ledger/core" "github.com/numary/ledger/ledger/query" - "github.com/pkg/errors" ) type Store interface { @@ -25,19 +24,3 @@ type Store interface { Name() string Close(context.Context) error } - -func GetStore(driverStr, name string) (Store, error) { - driver, ok := drivers[driverStr] - if !ok { - panic(errors.Errorf( - "unsupported store: %s", - driver, - )) - } - err := driver.Initialize(context.Background()) - if err != nil { - return nil, errors.Wrap(err, "initializing driver") - } - - return driver.NewStore(name) -}