From 09d0546be262b7e0cb262aec36aa4f77fc02d81f Mon Sep 17 00:00:00 2001 From: Geoffrey Ragot Date: Tue, 22 Oct 2024 12:16:02 +0200 Subject: [PATCH] fix: tests --- go.mod | 10 +++++++-- go.sum | 13 +++++------- internal/storage/bucket/migrations.go | 2 +- internal/storage/ledger/transactions.go | 20 ------------------ internal/storage/module.go | 26 ++++++++++++++++-------- test/e2e/api_balances_aggregated_test.go | 4 ++-- test/e2e/lifecycle_test.go | 4 +++- test/e2e/suite_test.go | 22 ++++++++++---------- 8 files changed, 48 insertions(+), 53 deletions(-) diff --git a/go.mod b/go.mod index 827eebc33..7dc24df35 100644 --- a/go.mod +++ b/go.mod @@ -21,6 +21,7 @@ require ( github.com/google/go-cmp v0.6.0 github.com/google/uuid v1.6.0 github.com/invopop/jsonschema v0.12.0 + github.com/jackc/pgx/v5 v5.7.1 github.com/jamiealquiza/tachymeter v2.0.0+incompatible github.com/logrusorgru/aurora v2.0.3+incompatible github.com/nats-io/nats.go v1.37.0 @@ -28,12 +29,14 @@ require ( github.com/onsi/gomega v1.34.2 github.com/ory/dockertest/v3 v3.11.0 github.com/pborman/uuid v1.2.1 + github.com/pkg/errors v0.9.1 github.com/shomali11/xsql v0.0.0-20190608141458-bf76292144df github.com/spf13/cobra v1.8.1 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.9.0 github.com/uptrace/bun v1.2.3 github.com/uptrace/bun/dialect/pgdialect v1.2.3 + github.com/uptrace/bun/extra/bundebug v1.2.3 github.com/xeipuuv/gojsonschema v1.2.0 github.com/xo/dburl v0.23.2 go.opentelemetry.io/otel v1.31.0 @@ -44,6 +47,7 @@ require ( go.uber.org/mock v0.4.0 golang.org/x/oauth2 v0.23.0 golang.org/x/sync v0.8.0 + gopkg.in/yaml.v3 v3.0.1 ) require ( @@ -89,6 +93,7 @@ require ( github.com/eapache/queue v1.1.0 // indirect github.com/ebitengine/purego v0.8.0 // indirect github.com/ericlagergren/decimal v0.0.0-20240411145413-00de7ca16731 // indirect + github.com/fatih/color v1.17.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/formancehq/numscript v0.0.9-0.20241009144012-1150c14a1417 github.com/go-chi/chi v4.1.2+incompatible // indirect @@ -116,7 +121,7 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect - github.com/jackc/pgx/v5 v5.7.1 + github.com/jackc/pgx/v5 v5.7.1 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect @@ -128,6 +133,8 @@ require ( github.com/lithammer/shortuuid/v3 v3.0.7 // indirect github.com/lufia/plan9stats v0.0.0-20240909124753-873cd0166683 // indirect github.com/mailru/easyjson v0.7.7 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect github.com/minio/highwayhash v1.0.3 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/term v0.5.0 // indirect @@ -199,5 +206,4 @@ require ( google.golang.org/protobuf v1.35.1 // indirect gopkg.in/go-jose/go-jose.v2 v2.6.3 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 238d91b57..f7620f9fc 100644 --- a/go.sum +++ b/go.sum @@ -95,14 +95,8 @@ github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/formancehq/go-libs/v2 v2.0.1-0.20241017152835-2c30f563ab46 h1:8wZtnWSIYNV7DwD0Jr4HsbcRgezOrgDJ2Q0w9ABieKc= -github.com/formancehq/go-libs/v2 v2.0.1-0.20241017152835-2c30f563ab46/go.mod h1:LgxayMN6wgAQbkB3ioBDTHOVMKp1rC6Q55M1CvG44xY= -github.com/formancehq/go-libs/v2 v2.0.1-0.20241017153232-1a62cecf1a61 h1:GSIhsdo/YXuZXI4q8xA8IrdOkkjfFp6O+DiNywk8s8U= -github.com/formancehq/go-libs/v2 v2.0.1-0.20241017153232-1a62cecf1a61/go.mod h1:LgxayMN6wgAQbkB3ioBDTHOVMKp1rC6Q55M1CvG44xY= -github.com/formancehq/go-libs/v2 v2.0.1-0.20241021110602-fbd3b37b93f8 h1:q9mP1jT2q2/QxGdEg9mWOcsf/P7NNvC1vMmyDvj5EEY= -github.com/formancehq/go-libs/v2 v2.0.1-0.20241021110602-fbd3b37b93f8/go.mod h1:KO+eOrTVQ5tR3TZUAHapoQ+d7y2+Ie5Tg0QwfZHAK4k= -github.com/formancehq/go-libs/v2 v2.0.1-0.20241021183239-813f4dc647a1 h1:PnDpgyTBicMbSC/c7PTdYaNZCCHlnKyVXURo4VTJyCc= -github.com/formancehq/go-libs/v2 v2.0.1-0.20241021183239-813f4dc647a1/go.mod h1:LgxayMN6wgAQbkB3ioBDTHOVMKp1rC6Q55M1CvG44xY= +github.com/formancehq/go-libs/v2 v2.0.1-0.20241022171306-dcd601f231ec h1:50Ojig/hsng4K8FgoezONfVljawi9+sYQkjj+vf7/Fw= +github.com/formancehq/go-libs/v2 v2.0.1-0.20241022171306-dcd601f231ec/go.mod h1:LgxayMN6wgAQbkB3ioBDTHOVMKp1rC6Q55M1CvG44xY= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/go-chi/chi v4.1.2+incompatible h1:fGFk2Gmi/YKXk0OmGfBh0WgmN3XB8lVnEyNz34tQRec= @@ -220,6 +214,7 @@ github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/minio/highwayhash v1.0.3 h1:kbnuUMoHYyVl7szWjSxJnxw11k2U709jqFPPmIUyD6Q= @@ -437,8 +432,10 @@ golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= diff --git a/internal/storage/bucket/migrations.go b/internal/storage/bucket/migrations.go index f8937e0dc..d853c38a9 100644 --- a/internal/storage/bucket/migrations.go +++ b/internal/storage/bucket/migrations.go @@ -6,9 +6,9 @@ import ( "embed" "fmt" "github.com/formancehq/go-libs/v2/migrations" - "github.com/ghodss/yaml" "github.com/uptrace/bun" "go.opentelemetry.io/otel/trace" + "gopkg.in/yaml.v3" "io/fs" "path/filepath" "slices" diff --git a/internal/storage/ledger/transactions.go b/internal/storage/ledger/transactions.go index 3ad30d173..5f001ff11 100644 --- a/internal/storage/ledger/transactions.go +++ b/internal/storage/ledger/transactions.go @@ -160,26 +160,6 @@ func (s *Store) selectTransactions(date *time.Time, expandVolumes, expandEffecti Column("transactions_id"). ColumnExpr("aggregate_objects(post_commit_effective_volumes::jsonb) as post_commit_effective_volumes"). Group("transactions_id"), - //s.db.NewSelect(). - // Column("transactions_id"). - // ColumnExpr("aggregate_objects(pcev::jsonb) as post_commit_effective_volumes"). - // TableExpr( - // "(?) data", - // s.db.NewSelect(). - // DistinctOn("transactions_id, accounts_address, asset"). - // ModelTableExpr(s.GetPrefixedRelationName("moves")). - // Column("transactions_id"). - // ColumnExpr(` - // json_build_object( - // moves.accounts_address, - // json_build_object( - // moves.asset, - // first_value(moves.post_commit_effective_volumes) over (partition by (transactions_id, accounts_address, asset) order by seq desc) - // ) - // ) as pcev - // `), - // ). - // Group("transactions_id"), ). ColumnExpr("pcev.*") } diff --git a/internal/storage/module.go b/internal/storage/module.go index 16a45915e..093eda91b 100644 --- a/internal/storage/module.go +++ b/internal/storage/module.go @@ -29,17 +29,27 @@ func NewFXModule(autoUpgrade bool) fx.Option { go func() { defer close(upgradeStopped) - if err := driver.UpgradeAllBuckets(upgradeContext); err != nil { - // Long migrations can be cancelled (app rescheduled for example) - // before fully terminated, handle this gracefully, don't panic, - // the next start will try again. - if errors.Is(err, context.DeadlineExceeded) || - errors.Is(err, context.Canceled) { + for { + select { + case <-ctx.Done(): + return + default: + logging.FromContext(ctx).Infof("Upgrading buckets...") + if err := driver.UpgradeAllBuckets(upgradeContext); err != nil { + // Long migrations can be cancelled (app rescheduled for example) + // before fully terminated, handle this gracefully, don't panic, + // the next start will try again. + if errors.Is(err, context.DeadlineExceeded) || + errors.Is(err, context.Canceled) { + return + } + logging.FromContext(ctx).Errorf("Upgrading buckets: %s", err) + continue + } return } - - panic(err) } + }() return nil }, diff --git a/test/e2e/api_balances_aggregated_test.go b/test/e2e/api_balances_aggregated_test.go index 0cc59f4fc..375444698 100644 --- a/test/e2e/api_balances_aggregated_test.go +++ b/test/e2e/api_balances_aggregated_test.go @@ -38,7 +38,7 @@ var _ = Context("Ledger engine tests", func() { }) Expect(err).To(BeNil()) - _, err = CreateBulk(ctx, testServer.GetValue(), operations.V2CreateBulkRequest{ + ret, err := CreateBulk(ctx, testServer.GetValue(), operations.V2CreateBulkRequest{ RequestBody: []components.V2BulkElement{ components.CreateV2BulkElementCreateTransaction(components.V2BulkElementCreateTransaction{ Data: &components.V2PostTransaction{ @@ -99,7 +99,7 @@ var _ = Context("Ledger engine tests", func() { }) Expect(err).To(Succeed()) - firstTransactionsInsertedAt = time.Now() + firstTransactionsInsertedAt = ret[2].V2BulkElementResultCreateTransaction.Data.InsertedAt _, err = CreateBulk(ctx, testServer.GetValue(), operations.V2CreateBulkRequest{ RequestBody: []components.V2BulkElement{ diff --git a/test/e2e/lifecycle_test.go b/test/e2e/lifecycle_test.go index 10af57c8d..1255a3d6d 100644 --- a/test/e2e/lifecycle_test.go +++ b/test/e2e/lifecycle_test.go @@ -8,6 +8,7 @@ import ( "github.com/formancehq/go-libs/v2/bun/bunconnect" "github.com/formancehq/go-libs/v2/logging" "github.com/formancehq/go-libs/v2/pointer" + "github.com/formancehq/go-libs/v2/testing/platform/pgtesting" "github.com/formancehq/go-libs/v2/time" ledger "github.com/formancehq/ledger/internal" "github.com/formancehq/ledger/pkg/client/models/components" @@ -26,11 +27,11 @@ import ( var _ = Context("Ledger application lifecycle tests", func() { var ( - db = UseTemplatedDatabase() ctx = logging.TestingContext() ) Context("Pending transaction should be fully processed before stopping or restarting the server", func() { + db := UseTemplatedDatabase() testServer := NewTestServer(func() Configuration { return Configuration{ PostgresConfiguration: db.GetValue().ConnectionOptions(), @@ -172,6 +173,7 @@ var _ = Context("Ledger application lifecycle tests", func() { Context("Ledger should respond correctly as well as the minimal schema version is respected", func() { var ( ledgerName = "default" + db = pgtesting.UsePostgresDatabase(pgServer) ) BeforeEach(func() { bunDB, err := bunconnect.OpenSQLDB(ctx, db.GetValue().ConnectionOptions()) diff --git a/test/e2e/suite_test.go b/test/e2e/suite_test.go index 558df1bd1..069ff41bf 100644 --- a/test/e2e/suite_test.go +++ b/test/e2e/suite_test.go @@ -10,8 +10,6 @@ import ( ledger "github.com/formancehq/ledger/internal" "github.com/formancehq/ledger/internal/storage/bucket" "github.com/formancehq/ledger/internal/storage/driver" - "github.com/stretchr/testify/require" - "go.opentelemetry.io/otel/trace/noop" "os" "testing" @@ -35,7 +33,7 @@ var ( debug = os.Getenv("DEBUG") == "true" logger = logging.NewDefaultLogger(GinkgoWriter, debug, false) - DBTemplate = "template1" + DBTemplate = "dbtemplate" ) type ParallelExecutionContext struct { @@ -57,18 +55,20 @@ var _ = SynchronizedBeforeSuite(func() []byte { ) By("Postgres address: " + ret.GetDSN()) - db, err := bunconnect.OpenSQLDB(context.Background(), bunconnect.ConnectionOptions{ - DatabaseSourceName: ret.GetDatabaseDSN(DBTemplate), - }) - require.NoError(GinkgoT(), err) + templateDatabase := ret.NewDatabase(GinkgoT(), WithName(DBTemplate)) - err = driver.Migrate(context.Background(), db) - require.NoError(GinkgoT(), err) + bunDB, err := bunconnect.OpenSQLDB(context.Background(), templateDatabase.ConnectionOptions()) + Expect(err).To(BeNil()) + + err = driver.Migrate(context.Background(), bunDB) + Expect(err).To(BeNil()) // Initialize the _default bucket on the default database // This way, we will be able to clone this database to speed up the tests - err = bucket.Migrate(context.Background(), noop.Tracer{}, db, ledger.DefaultBucket) - require.NoError(GinkgoT(), err) + err = bucket.GetMigrator(ledger.DefaultBucket).Up(context.Background(), bunDB) + Expect(err).To(BeNil()) + + Expect(bunDB.Close()).To(BeNil()) return ret })