run migrate and schema against managed emulator instance

.github/workflows/test.yml

@@ -9,17 +9,22 @@ name: Tests with spanner emulator
 jobs:
   tests:
     runs-on: ubuntu-latest
-    container: node:10.18-jessie
     services:
       spanner:
         env:
           SPANNER_INSTANCE_ID: ${{env.SPANNER_INSTANCE_ID}}
           SPANNER_PROJECT_ID: ${{env.SPANNER_PROJECT_ID}}
         image: roryq/spanner-emulator:1.4.0
+        ports:
+          - 9010:9010
+      dind:
+        image: docker:23.0-rc-dind-rootless
+        ports:
+          - 2375:2375
     env:
       SPANNER_INSTANCE_ID: inst
       SPANNER_PROJECT_ID: proj
-      SPANNER_EMULATOR_HOST: spanner:9010
+      SPANNER_EMULATOR_HOST: localhost:9010
     steps:
       - name: Install Go
         uses: actions/setup-go@v2

README.md

@@ -65,6 +65,7 @@ Available Commands:
   reset         Equivalent to drop and then create
   load          Load schema from server to file
   load-discrete Load schema from server to discrete files per object
+  schema        Runs the migrations against a dockerised spanner emulator, then loads the schema and static data to disk. (Requires docker)
   apply         Apply DDL file to database
   migrate       Migrate database
   truncate      Truncate all tables without deleting a database

cmd/cmd.go

@@ -23,8 +23,9 @@ import (
 	"context"
 	"path/filepath"
 
-	"github.com/roryq/wrench/pkg/spanner"
 	"github.com/spf13/cobra"
+
+	"github.com/roryq/wrench/pkg/spanner"
 )
 
 const (
@@ -42,6 +43,7 @@ const (
 	flagDDLFile                 = "ddl"
 	flagDMLFile                 = "dml"
 	flagPartitioned             = "partitioned"
+	flagSpannerEmulatorImage    = "spanner-emulator-image"
 	defaultSchemaFileName       = "schema.sql"
 	defaultStaticDataTablesFile = "{wrench.json|static_data_tables.txt}"
 )

cmd/migrate.go

@@ -101,7 +101,6 @@ func init() {
 	migrateUpCmd.Flags().SetNormalizeFunc(underscoreToDashes)
 
 	migrateCreateCmd.Flags().Bool(flagNameCreateNoPrompt, false, "Don't prompt for a migration file description")
-	migrateCmd.PersistentFlags().String(flagNameDirectory, "", "Directory that migration files placed (required)")
 	migrateUpCmd.Flags().UintSlice(flagSkipVersions, []uint{}, "Versions to skip during migration")
 }
 

cmd/root.go

@@ -70,10 +70,12 @@ func init() {
 	rootCmd.AddCommand(resetCmd)
 	rootCmd.AddCommand(loadCmd)
 	rootCmd.AddCommand(loadDiscreteCmd)
+	rootCmd.AddCommand(schemaCmd)
 	rootCmd.AddCommand(applyCmd)
 	rootCmd.AddCommand(migrateCmd)
 	rootCmd.AddCommand(truncateCmd)
 
+	// global flags
 	rootCmd.PersistentFlags().StringVar(&project, flagNameProject, spannerProjectID(), "GCP project id (optional. if not set, will use $SPANNER_PROJECT_ID or $GOOGLE_CLOUD_PROJECT value)")
 	rootCmd.PersistentFlags().StringVar(&instance, flagNameInstance, spannerInstanceID(), "Cloud Spanner instance name (optional. if not set, will use $SPANNER_INSTANCE_ID value)")
 	rootCmd.PersistentFlags().StringVar(&database, flagNameDatabase, spannerDatabaseID(), "Cloud Spanner database name (optional. if not set, will use $SPANNER_DATABASE_ID value)")

cmd/schema.go

@@ -0,0 +1,154 @@
+package cmd
+
+import (
+	"fmt"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/ory/dockertest/v3"
+	"github.com/ory/dockertest/v3/docker"
+	"github.com/spf13/cobra"
+
+	"github.com/roryq/wrench/internal/graceful"
+)
+
+var gracefulSchemaTasks = graceful.OnShutdown{}
+
+var schemaCmd = &cobra.Command{
+	Use:   "schema",
+	Short: "Runs the migrations against a dockerised spanner emulator, then loads the schema and static data to disk. (Requires docker)",
+	RunE:  schema,
+}
+
+func init() {
+	// schema flags
+	schemaCmd.Flags().String(flagSpannerEmulatorImage, "roryq/spanner-emulator:latest", "Spanner emulator image to use. Override this to pin version or change registry.")
+
+	// copy migrate up flags
+	schemaCmd.Flags().AddFlagSet(findCommand("up").LocalFlags())
+}
+
+func findCommand(name string) *cobra.Command {
+	for _, command := range migrateCmd.Commands() {
+		if command.Name() == name {
+			return command
+		}
+	}
+	return nil
+}
+
+func schema(c *cobra.Command, args []string) error {
+	defer gracefulSchemaTasks.Exit()
+
+	f := c.Flag(flagSpannerEmulatorImage)
+	_, err := runSpannerEmulator(f.Value.String())
+	if err != nil {
+		return err
+	}
+
+	c.Flag(flagNameProject).Value.Set("schema")
+	c.Flag(flagNameInstance).Value.Set("schema")
+	c.Flag(flagNameDatabase).Value.Set("schema")
+
+	// run migrations
+	if err := migrateUp(c, args); err != nil {
+		return err
+	}
+
+	// load schema
+	if err := load(c, args); err != nil {
+		return err
+	}
+
+	// load discrete
+	if err := loadDiscrete(c, args); err != nil {
+		return err
+	}
+
+	return err
+}
+
+func connectDockerPool() (*dockertest.Pool, error) {
+	pool, err := dockertest.NewPool("")
+	if err != nil {
+		return nil, err
+	}
+
+	return pool, pool.Client.Ping()
+}
+
+func runSpannerEmulator(image string) (*spannerEmulator, error) {
+	pool, err := connectDockerPool()
+	if err != nil {
+		return nil, err
+	}
+
+	repo, tag, _ := strings.Cut(image, ":")
+	container, err := pool.RunWithOptions(&dockertest.RunOptions{
+		Repository: repo,
+		Tag:        tag,
+		Env: []string{
+			"SPANNER_PROJECT_ID=schema",
+			"SPANNER_INSTANCE_ID=schema",
+			"SPANNER_DATABASE_ID=schema",
+		},
+	},
+		func(config *docker.HostConfig) {
+			config.AutoRemove = true
+			config.RestartPolicy = docker.NeverRestart()
+		},
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	gracefulSchemaTasks.Do(func() {
+		if err = container.Close(); err != nil {
+			println("error during spanner container shutdown", err.Error())
+		}
+	})
+
+	pool.MaxWait = time.Minute
+	err = pool.Retry(func() error {
+		hcheck := fmt.Sprintf("http://localhost:%s/v1/projects/test-project/instanceConfigs", container.GetPort("9020/tcp"))
+		_, err := http.Get(hcheck)
+		return err
+	})
+	if err != nil {
+		return nil, err
+	}
+	unset, err := setenv("SPANNER_EMULATOR_HOST", container.GetHostPort("9010/tcp"))
+	if err != nil {
+		return nil, err
+	}
+	gracefulSchemaTasks.Do(unset)
+
+	return &spannerEmulator{container: container, pool: pool}, nil
+}
+
+type spannerEmulator struct {
+	container *dockertest.Resource
+	pool      *dockertest.Pool
+}
+
+func (s *spannerEmulator) SpannerEmulatorHost() string {
+	return s.container.GetHostPort("9010/tcp")
+}
+
+func setenv(key, value string) (func(), error) {
+	prevValue, ok := os.LookupEnv(key)
+
+	if err := os.Setenv(key, value); err != nil {
+		return func() {}, err
+	}
+
+	return func() {
+		if ok {
+			os.Setenv(key, prevValue)
+		} else {
+			os.Unsetenv(key)
+		}
+	}, nil
+}

cmd/schema_test.go

@@ -0,0 +1,39 @@
+package cmd
+
+import (
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// Test_schema runs the schema command which requires docker
+func Test_schema(t *testing.T) {
+	// clean before and after
+	t.Cleanup(func() { cleanup(t) })
+	cleanup(t)
+
+	// execute schema command
+	cmd := schemaCmd
+	cmd.Flag(flagNameDirectory).Value.Set("testdata/schema_test")
+	err := schema(cmd, []string{})
+	assert.NoError(t, err)
+
+	assert.DirExists(t, "testdata/schema_test/table")
+	assert.FileExists(t, "testdata/schema_test/schema.sql")
+
+	contentSchema, err := os.ReadFile("testdata/schema_test/schema.sql")
+	assert.NoError(t, err)
+	contentMigration, err := os.ReadFile("testdata/schema_test/migrations/000001_create_singers.sql")
+	assert.NoError(t, err)
+
+	assert.Contains(t, string(contentSchema), string(contentMigration))
+}
+
+func cleanup(t *testing.T) {
+	os.RemoveAll("testdata/schema_test/table")
+	os.RemoveAll("testdata/schema_test/schema.sql")
+	require.NoDirExists(t, "testdata/schema_test/table")
+	require.NoFileExists(t, "testdata/schema_test/schema.sql")
+}

cmd/testdata/schema_test/.gitignore

@@ -0,0 +1,2 @@
+/table/
+/schema.sql

cmd/testdata/schema_test/migrations/000001_create_singers.sql

@@ -0,0 +1,4 @@
+CREATE TABLE Singers (
+  SingerID STRING(36) NOT NULL,
+  FirstName STRING(1024),
+) PRIMARY KEY(SingerID);

go.mod

@@ -1,42 +1,75 @@
 module github.com/roryq/wrench
 
 require (
-	cloud.google.com/go v0.104.0
-	cloud.google.com/go/spanner v1.39.0
+	cloud.google.com/go v0.110.2
+	cloud.google.com/go/spanner v1.47.0
 	github.com/google/go-cmp v0.5.9
 	github.com/google/uuid v1.3.0
 	github.com/kennygrant/sanitize v1.2.4
-	github.com/spf13/cobra v1.6.0
+	github.com/ory/dockertest/v3 v3.10.0
+	github.com/spf13/cobra v1.7.0
 	github.com/spf13/pflag v1.0.5
-	github.com/stretchr/testify v1.7.0
-	google.golang.org/api v0.99.0
-	google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a
-	google.golang.org/grpc v1.50.1
+	github.com/stretchr/testify v1.8.3
+	google.golang.org/api v0.128.0
+	google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc
+	google.golang.org/grpc v1.56.0
 )
 
 require (
-	cloud.google.com/go/compute v1.10.0 // indirect
-	github.com/census-instrumentation/opencensus-proto v0.3.0 // indirect
-	github.com/cespare/xxhash/v2 v2.1.2 // indirect
-	github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4 // indirect
-	github.com/cncf/xds/go v0.0.0-20211216145620-d92e9ce0af51 // indirect
-	github.com/davecgh/go-spew v1.1.0 // indirect
-	github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1 // indirect
-	github.com/envoyproxy/protoc-gen-validate v0.6.2 // indirect
+	cloud.google.com/go/compute v1.20.1 // indirect
+	cloud.google.com/go/compute/metadata v0.2.3 // indirect
+	cloud.google.com/go/iam v1.1.1 // indirect
+	cloud.google.com/go/longrunning v0.5.1 // indirect
+	github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect
+	github.com/Microsoft/go-winio v0.6.1 // indirect
+	github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect
+	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
+	github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe // indirect
+	github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 // indirect
+	github.com/containerd/continuity v0.4.1 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/docker/cli v24.0.2+incompatible // indirect
+	github.com/docker/docker v24.0.2+incompatible // indirect
+	github.com/docker/go-connections v0.4.0 // indirect
+	github.com/docker/go-units v0.5.0 // indirect
+	github.com/envoyproxy/go-control-plane v0.11.1 // indirect
+	github.com/envoyproxy/protoc-gen-validate v1.0.1 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
-	github.com/golang/protobuf v1.5.2 // indirect
-	github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect
-	github.com/googleapis/gax-go/v2 v2.5.1 // indirect
-	github.com/inconshreveable/mousetrap v1.0.1 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/google/s2a-go v0.1.4 // indirect
+	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect
+	github.com/googleapis/gax-go/v2 v2.11.0 // indirect
+	github.com/imdario/mergo v0.3.16 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/moby/term v0.5.0 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.0.2 // indirect
+	github.com/opencontainers/runc v1.1.7 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	go.opencensus.io v0.23.0 // indirect
-	golang.org/x/net v0.0.0-20221012135044-0b7e1fb9d458 // indirect
-	golang.org/x/oauth2 v0.0.0-20221006150949-b44042a4b9c1 // indirect
-	golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 // indirect
-	golang.org/x/text v0.3.7 // indirect
+	github.com/sirupsen/logrus v1.9.3 // indirect
+	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
+	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
+	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
+	go.opencensus.io v0.24.0 // indirect
+	golang.org/x/crypto v0.10.0 // indirect
+	golang.org/x/mod v0.11.0 // indirect
+	golang.org/x/net v0.11.0 // indirect
+	golang.org/x/oauth2 v0.9.0 // indirect
+	golang.org/x/sys v0.9.0 // indirect
+	golang.org/x/text v0.10.0 // indirect
+	golang.org/x/tools v0.10.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/protobuf v1.28.1 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect
+	google.golang.org/protobuf v1.30.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )