From aa32239d17ff6fe81214657a311dbe271ec5eab4 Mon Sep 17 00:00:00 2001 From: Mykhailo Bobrovskyi Date: Sun, 12 Feb 2023 23:06:30 +0200 Subject: [PATCH 1/3] Using option.WithEndpoint to connect to bigquery-emulator --- database/bigquery/README.md | 2 +- database/bigquery/bigquery.go | 9 +++++-- database/bigquery/bigquery_test.go | 39 +++++++++++++++++++++--------- 3 files changed, 35 insertions(+), 15 deletions(-) diff --git a/database/bigquery/README.md b/database/bigquery/README.md index 4410353bb..d64661161 100644 --- a/database/bigquery/README.md +++ b/database/bigquery/README.md @@ -4,7 +4,7 @@ * [Examples](./examples) ### Usage -`bigquery://https://www.googleapis.com/bigquery/v2:443?x-migrations-table=schema_migrations&x-statement-timeout=0&credentials_filename=./myproject-XXXXXXXXXXXXX-XXXXXXXXXXXX.json&project_id=myproject-XXXXXXXXXXXXX&dataset_id=mydataset` +`bigquery://https://bigquery.googleapis.com/bigquery/v2/?x-migrations-table=schema_migrations&x-statement-timeout=0&credentials_filename=./myproject-XXXXXXXXXXXXX-XXXXXXXXXXXX.json&project_id=myproject-XXXXXXXXXXXXX&dataset_id=mydataset` ### System variables reference diff --git a/database/bigquery/bigquery.go b/database/bigquery/bigquery.go index bee06c3c6..dcc6391f0 100644 --- a/database/bigquery/bigquery.go +++ b/database/bigquery/bigquery.go @@ -14,6 +14,7 @@ import ( "google.golang.org/api/option" "io" nurl "net/url" + "os" "strconv" "strings" "time" @@ -148,12 +149,14 @@ func (b *BigQuery) Open(url string) (database.Driver, error) { } } - //By default, try to get from environment variable GOOGLE_APPLICATION_CREDENTIALS if q.Has("credentials_filename") { opts = append(opts, option.WithCredentialsFile(q.Get("credentials_filename"))) + } else if os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") != "" { + opts = append(opts, option.WithCredentialsFile(os.Getenv("GOOGLE_APPLICATION_CREDENTIALS"))) + } else { + opts = append(opts, option.WithoutAuthentication()) } - // By default, try to get from credentials file projectID := bigquery.DetectProjectID if q.Has("project_id") { projectID = q.Get("project_id") @@ -175,6 +178,8 @@ func (b *BigQuery) Open(url string) (database.Driver, error) { config.TimeZone = q.Get("time_zone") } + opts = append(opts, option.WithEndpoint(fmt.Sprintf("%s%s", purl.Host, purl.Path))) + client, err := bigquery.NewClient(ctx, projectID, opts...) if err != nil { return nil, err diff --git a/database/bigquery/bigquery_test.go b/database/bigquery/bigquery_test.go index 09bc643fa..9842c65b5 100644 --- a/database/bigquery/bigquery_test.go +++ b/database/bigquery/bigquery_test.go @@ -7,7 +7,8 @@ import ( ) const ( - connectionUrl = "bigquery://https://www.googleapis.com/bigquery/v2:443?x-migrations-table=schema_migrations&x-statement-timeout=0&credentials_filename=./tmp/myproject-XXXXXXXXXXXXX-XXXXXXXXXXXX.json&dataset_id=mydataset" + //connectionUrl = "bigquery://https://bigquery.googleapis.com/bigquery/v2/?x-migrations-table=schema_migrations&x-statement-timeout=0&credentials_filename=./tmp/myproject-XXXXXXXXXXXXX-XXXXXXXXXXXX.json&dataset_id=mydataset" + connectionUrl = "bigquery://http://0.0.0.0:9050/?x-migrations-table=schema_migrations&project_id=myproject&dataset_id=mydataset" ) func openConnection() (database.Driver, error) { @@ -22,65 +23,75 @@ func openConnection() (database.Driver, error) { } func TestOpen(t *testing.T) { - _, err := openConnection() + driver, err := openConnection() if err != nil { t.Error(err) return } + + defer driver.Close() } -func TestVersion(t *testing.T) { +func TestClose(t *testing.T) { driver, err := openConnection() if err != nil { t.Error(err) return } - version, dirty, err := driver.Version() + defer driver.Close() + + err = driver.Close() if err != nil { t.Error(err) return } - - t.Log(version, dirty) } -func TestDrop(t *testing.T) { +func TestVersion(t *testing.T) { driver, err := openConnection() if err != nil { t.Error(err) return } - err = driver.Drop() + defer driver.Close() + + version, dirty, err := driver.Version() if err != nil { t.Error(err) return } + + t.Log(version, dirty) } -func TestClose(t *testing.T) { +func TestSetVersion(t *testing.T) { driver, err := openConnection() if err != nil { t.Error(err) return } - err = driver.Close() + defer driver.Close() + + err = driver.SetVersion(-1, false) if err != nil { t.Error(err) return } } -func TestSetVersion(t *testing.T) { +func TestDrop(t *testing.T) { driver, err := openConnection() if err != nil { t.Error(err) return } - err = driver.SetVersion(-1, false) + defer driver.Close() + + err = driver.Drop() if err != nil { t.Error(err) return @@ -94,6 +105,8 @@ func TestRun(t *testing.T) { return } + defer driver.Close() + err = driver.Run(strings.NewReader(` CREATE TABLE IF NOT EXISTS users ( first_name STRING, @@ -112,6 +125,8 @@ func TestRunWithError(t *testing.T) { return } + defer driver.Close() + err = driver.Run(strings.NewReader(` CREATE TABLE IF NOT EXISTS users ( first_name STRINGa, From 2d917509f7688672796abc1980ca68d07f9ac15b Mon Sep 17 00:00:00 2001 From: Mykhailo Bobrovskyi Date: Sun, 12 Feb 2023 23:15:15 +0200 Subject: [PATCH 2/3] Remove system variable from SetVersion method --- database/bigquery/bigquery.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/database/bigquery/bigquery.go b/database/bigquery/bigquery.go index dcc6391f0..cf9208fd4 100644 --- a/database/bigquery/bigquery.go +++ b/database/bigquery/bigquery.go @@ -272,10 +272,9 @@ func (b *BigQuery) SetVersion(version int, dirty bool) error { ctx := context.Background() query := fmt.Sprintf(` - SET @@dataset_id = '%[1]s'; BEGIN TRANSACTION; - DELETE FROM %[2]s WHERE true; - INSERT INTO %[2]s (version, dirty) VALUES (%[3]d, %[4]t); + DELETE FROM `+"`%[1]s.%[2]s`"+` WHERE true; + INSERT INTO `+"`%[1]s.%[2]s`"+` (version, dirty) VALUES (%[3]d, %[4]t); COMMIT TRANSACTION; `, b.config.DatasetID, b.config.MigrationsTable, version, dirty) From e764e1d096598c41fd342c9d7adbb5cfc963f24b Mon Sep 17 00:00:00 2001 From: Mykhailo Bobrovskyi Date: Sun, 12 Feb 2023 23:33:42 +0200 Subject: [PATCH 3/3] Added TestWithInstanceWithoutClient --- database/bigquery/bigquery_test.go | 75 +++++++++++++++++++++++++----- 1 file changed, 63 insertions(+), 12 deletions(-) diff --git a/database/bigquery/bigquery_test.go b/database/bigquery/bigquery_test.go index 9842c65b5..d4b6e69d2 100644 --- a/database/bigquery/bigquery_test.go +++ b/database/bigquery/bigquery_test.go @@ -1,6 +1,8 @@ package bigquery import ( + "context" + "errors" "github.com/golang-migrate/migrate/v4/database" "strings" "testing" @@ -22,6 +24,22 @@ func openConnection() (database.Driver, error) { return driver, nil } +func TestWithInstanceWithoutClient(t *testing.T) { + driver, err := WithInstance(context.Background(), nil, &Config{}) + if err == nil { + t.Errorf("expected `no client`, got nil") + return + } + if !errors.Is(err, ErrNoClient) { + t.Errorf("expected `no client`, got %s", err.Error()) + return + } + if driver != nil { + t.Errorf("driver should be nil") + return + } +} + func TestOpen(t *testing.T) { driver, err := openConnection() if err != nil { @@ -29,7 +47,12 @@ func TestOpen(t *testing.T) { return } - defer driver.Close() + defer func() { + err := driver.Close() + if err != nil { + t.Error(err) + } + }() } func TestClose(t *testing.T) { @@ -39,7 +62,12 @@ func TestClose(t *testing.T) { return } - defer driver.Close() + defer func() { + err := driver.Close() + if err != nil { + t.Error(err) + } + }() err = driver.Close() if err != nil { @@ -55,7 +83,12 @@ func TestVersion(t *testing.T) { return } - defer driver.Close() + defer func() { + err := driver.Close() + if err != nil { + t.Error(err) + } + }() version, dirty, err := driver.Version() if err != nil { @@ -73,7 +106,12 @@ func TestSetVersion(t *testing.T) { return } - defer driver.Close() + defer func() { + err := driver.Close() + if err != nil { + t.Error(err) + } + }() err = driver.SetVersion(-1, false) if err != nil { @@ -89,7 +127,12 @@ func TestDrop(t *testing.T) { return } - defer driver.Close() + defer func() { + err := driver.Close() + if err != nil { + t.Error(err) + } + }() err = driver.Drop() if err != nil { @@ -105,7 +148,12 @@ func TestRun(t *testing.T) { return } - defer driver.Close() + defer func() { + err := driver.Close() + if err != nil { + t.Error(err) + } + }() err = driver.Run(strings.NewReader(` CREATE TABLE IF NOT EXISTS users ( @@ -125,18 +173,21 @@ func TestRunWithError(t *testing.T) { return } - defer driver.Close() + defer func() { + err := driver.Close() + if err != nil { + t.Error(err) + } + }() err = driver.Run(strings.NewReader(` CREATE TABLE IF NOT EXISTS users ( first_name STRINGa, last_name STRING )`)) - if err != nil { - t.Log(err) - return + if err == nil { + t.Error("expected 'googleapi: Error 400: Query error: Type not found: STRINGa at [4:36], invalidQuery' got nil") } - t.Error("error is nil, should be 'googleapi: Error 400: Query error: Type not found: STRINGa at [4:36], invalidQuery'") - return + t.Log(err) }