Merge pull request #172 from snowplow/release/0.9.2

Release/0.9.2

.travis.yml

@@ -3,7 +3,7 @@ language: go
 services:
   - docker
 go:
-  - "1.13.8"
+  - "1.15.5"
 
 before_script:
   - make setup-up
@@ -22,7 +22,7 @@ deploy:
   provider: script
   script: make release
   on:
-    condition: '"${TRAVIS_GO_VERSION}" == "1.13.8"'
+    condition: '"${TRAVIS_GO_VERSION}" == "1.15.5"'
     tags: true
 
 env:

CHANGELOG

@@ -1,3 +1,9 @@
+Version 0.9.2 (2020-12-15)
+--------------------------
+BigQuery: Run multiple-step queries as single script (closes #169)
+BigQuery: Explicitly set Client.Location from the region playbook variable (closes #173)
+Bump Travis Golang version to 1.15 (closes #174)
+
 Version 0.9.1 (2020-11-17)
 --------------------------
 Add ability to template YAML playbook with custom functions (#167)

README.md

@@ -53,14 +53,14 @@ guest> make format
 
 First either compile the binary from source using the above `make` command or download the published Binary directly from Bintray:
 
-* [Darwin (macOS)](https://dl.bintray.com/snowplow/snowplow-generic/sql_runner_0.9.1_darwin_amd64.zip)
-* [Linux](https://dl.bintray.com/snowplow/snowplow-generic/sql_runner_0.9.1_linux_amd64.zip)
-* [Windows](https://dl.bintray.com/snowplow/snowplow-generic/sql_runner_0.9.1_windows_amd64.zip)
+* [Darwin (macOS)](https://dl.bintray.com/snowplow/snowplow-generic/sql_runner_0.9.2_darwin_amd64.zip)
+* [Linux](https://dl.bintray.com/snowplow/snowplow-generic/sql_runner_0.9.2_linux_amd64.zip)
+* [Windows](https://dl.bintray.com/snowplow/snowplow-generic/sql_runner_0.9.2_windows_amd64.zip)
 
 ### CLI Output
 
 ```bash
-sql-runner version: 0.9.0
+sql-runner version: 0.9.2
 Run playbooks of SQL scripts in series and parallel on Redshift and Postgres
 Usage:
   -checkLock string
@@ -117,7 +117,7 @@ limitations under the License.
 [travis]: https://travis-ci.org/snowplow/sql-runner
 [travis-image]: https://travis-ci.org/snowplow/sql-runner.png?branch=master
 
-[release-image]: http://img.shields.io/badge/release-0.9.1-6ad7e5.svg?style=flat
+[release-image]: http://img.shields.io/badge/release-0.9.2-6ad7e5.svg?style=flat
 [releases]: https://github.com/snowplow/sql-runner/releases
 
 [license-image]: http://img.shields.io/badge/license-Apache--2-blue.svg?style=flat
@@ -135,6 +135,6 @@ limitations under the License.
 [developers-guide]: https://github.com/snowplow/sql-runner/wiki/Guide-for-developers
 [devops-guide]: https://github.com/snowplow/sql-runner/wiki/Guide-for-devops
 
-[devops-image]:  http://sauna-github-static.s3-website-us-east-1.amazonaws.com/devops.svg
-[analysts-image]: http://sauna-github-static.s3-website-us-east-1.amazonaws.com/analyst.svg
-[developers-image]:  http://sauna-github-static.s3-website-us-east-1.amazonaws.com/developer.svg
+[devops-image]:  https://d3i6fms1cm1j0i.cloudfront.net/github/images/setup.png
+[analysts-image]: https://d3i6fms1cm1j0i.cloudfront.net/github/images/techdocs.png
+[developers-image]:  https://d3i6fms1cm1j0i.cloudfront.net/github/images/setup.png

VERSION

@@ -1 +1 @@
-0.9.1
+0.9.2

go.mod

@@ -1,14 +1,13 @@
 module github.com/snowplow/sql-runner
 
-go 1.13
+go 1.15
 
 require (
-	cloud.google.com/go/bigquery v1.9.0
+	cloud.google.com/go/bigquery v1.13.0
 	github.com/armon/go-metrics v0.3.3 // indirect
 	github.com/aws/aws-sdk-go v1.15.27
 	github.com/go-ini/ini v1.38.2 // indirect
 	github.com/go-pg/pg v6.14.5+incompatible
-	github.com/google/uuid v1.0.0 // indirect
 	github.com/hashicorp/consul v1.2.2
 	github.com/hashicorp/go-msgpack v1.1.5 // indirect
 	github.com/hashicorp/go-rootcerts v0.0.0-20160503143440-6bb64b370b90 // indirect
@@ -30,9 +29,8 @@ require (
 	github.com/smartystreets/goconvey v1.6.4 // indirect
 	github.com/snowflakedb/gosnowflake v1.1.10
 	github.com/stretchr/testify v1.4.0
-	golang.org/x/net v0.0.0-20200707034311-ab3426394381
-	golang.org/x/tools v0.0.0-20200714190737-9048b464a08d // indirect
-	google.golang.org/api v0.29.0
+	golang.org/x/net v0.0.0-20201026091529-146b70c837a4
+	google.golang.org/api v0.34.0
 	gopkg.in/ini.v1 v1.57.0 // indirect
 	gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0
 )

sql_runner/bigquery_target.go

@@ -48,6 +48,8 @@ func NewBigQueryTarget(target Target) *BigQueryTarget {
 		log.Fatalf("ERROR: Failed to create client: %v", err)
 	}
 
+	client.Location = target.Region
+
 	return &BigQueryTarget{target, client}
 }
 
@@ -56,7 +58,6 @@ func (bqt BigQueryTarget) GetTarget() Target {
 }
 
 // Run a query against the target
-// One statement per API call
 func (bqt BigQueryTarget) RunQuery(query ReadyQuery, dryRun bool, showQueryOutput bool) QueryStatus {
 	var affected int64 = 0
 	var err error = nil
@@ -72,48 +73,56 @@ func (bqt BigQueryTarget) RunQuery(query ReadyQuery, dryRun bool, showQueryOutpu
 		return QueryStatus{query, query.Path, 0, nil}
 	}
 
-	scripts := strings.Split(query.Script, ";")
-
-	for _, script := range scripts {
-		if len(strings.TrimSpace(script)) > 0 {
-			// If showing query output, perform a dry run to get column metadata
-			if showQueryOutput {
-				dq := bqt.Client.Query(script)
-				dq.DryRun = true
-				dqJob, err := dq.Run(ctx)
-				if err != nil {
-					log.Printf("ERROR: Failed to dry run job: %s.", err)
-					return QueryStatus{query, query.Path, int(affected), err}
-				}
-
-				schema = dqJob.LastStatus().Statistics.Details.(*bq.QueryStatistics).Schema
-			}
-
-			q := bqt.Client.Query(script)
+	script := query.Script
 
-			job, err := q.Run(ctx)
+	if len(strings.TrimSpace(script)) > 0 {
+		// If showing query output, perform a dry run to get column metadata
+		if showQueryOutput {
+			dq := bqt.Client.Query(script)
+			dq.DryRun = true
+			dqJob, err := dq.Run(ctx)
 			if err != nil {
-				log.Printf("ERROR: Failed to run job: %s.", err)
+				log.Printf("ERROR: Failed to dry run job: %s.", err)
 				return QueryStatus{query, query.Path, int(affected), err}
 			}
 
-			it, err := job.Read(ctx)
+			schema = dqJob.LastStatus().Statistics.Details.(*bq.QueryStatistics).Schema
+		}
+
+		q := bqt.Client.Query(script)
+
+		job, err := q.Run(ctx)
+		if err != nil {
+			log.Printf("ERROR: Failed to run job: %s.", err)
+			return QueryStatus{query, query.Path, int(affected), err}
+		}
+
+		it, err := job.Read(ctx)
+		if err != nil {
+			log.Printf("ERROR: Failed to read job results: %s.", err)
+			return QueryStatus{query, query.Path, int(affected), err}
+		}
+
+		status, err := job.Status(ctx)
+		if err != nil {
+			log.Printf("ERROR: Failed to read job results: %s.", err)
+			return QueryStatus{query, query.Path, int(affected), err}
+		}
+		if err := status.Err(); err != nil {
+			log.Printf("ERROR: Error running job: %s.", err)
+			return QueryStatus{query, query.Path, int(affected), err}
+		}
+
+		if showQueryOutput {
+			err = printBqTable(it, schema)
 			if err != nil {
-				log.Printf("ERROR: Failed to read job results: %s.", err)
+				log.Printf("ERROR: Failed to print output: %s.", err)
 				return QueryStatus{query, query.Path, int(affected), err}
 			}
-
-			if showQueryOutput {
-				err = printBqTable(it, schema)
-				if err != nil {
-					log.Printf("ERROR: Failed to print output: %s.", err)
-					return QueryStatus{query, query.Path, int(affected), err}
-				}
-			} else {
-				queryStats := job.LastStatus().Statistics.Details.(*bq.QueryStatistics)
-				aff := queryStats.NumDMLAffectedRows
-				affected += aff
-			}
+		} else {
+			queryStats := job.LastStatus().Statistics.Details.(*bq.QueryStatistics)
+			aff := queryStats.NumDMLAffectedRows
+			affected += aff
 		}
 	}
 

sql_runner/main.go

@@ -25,7 +25,7 @@ import (
 const (
 	CLI_NAME        = "sql-runner"
 	CLI_DESCRIPTION = `Run playbooks of SQL scripts in series and parallel on Redshift and Postgres`
-	CLI_VERSION     = "0.9.1"
+	CLI_VERSION     = "0.9.2"
 
 	SQLROOT_BINARY         = "BINARY"
 	SQLROOT_PLAYBOOK       = "PLAYBOOK"