Merge pull request #78 from dhui/mysql_dsn

Allow DB connection URLs to contain encoded reserved URL characters

Makefile

@@ -38,6 +38,7 @@ test-with-flags:
 
 	@go test $(TEST_FLAGS) .
 	@go test $(TEST_FLAGS) ./cli/...
+	@go test $(TEST_FLAGS) ./database
 	@go test $(TEST_FLAGS) ./testing/...
 
 	@echo -n '$(SOURCE)' | tr -s ' ' '\n' | xargs -I{} go test $(TEST_FLAGS) ./source/{}

README.md

@@ -39,6 +39,25 @@ Database drivers run migrations. [Add a new database?](database/driver.go)
   * [CockroachDB](database/cockroachdb)
   * [ClickHouse](database/clickhouse)
 
+### Database URLs
+
+Database connection strings are specified via URLs. The URL format is driver dependent but generally has the form: `dbdriver://username:password@host:port/dbname?option1=true&option2=false`
+
+Any [reserved URL characters](https://en.wikipedia.org/wiki/Percent-encoding#Percent-encoding_reserved_characters) need to be escaped. Note, the `%` character also [needs to be escaped](https://en.wikipedia.org/wiki/Percent-encoding#Percent-encoding_the_percent_character)
+
+Explicitly, the following characters need to be escaped:
+`!`, `#`, `$`, `%`, `&`, `'`, `(`, `)`, `*`, `+`, `,`, `/`, `:`, `;`, `=`, `?`, `@`, `[`, `]`
+
+It's easiest to always run the URL parts of your DB connection URL (e.g. username, password, etc) through an URL encoder. See the example Python helpers below:
+```bash
+$ python3 -c 'import urllib.parse; print(urllib.parse.quote(input("String to encode: "), ""))'
+String to encode: FAKEpassword!#$%&'()*+,/:;=?@[]
+FAKEpassword%21%23%24%25%26%27%28%29%2A%2B%2C%2F%3A%3B%3D%3F%40%5B%5D
+$ python2 -c 'import urllib; print urllib.quote(raw_input("String to encode: "), "")'
+String to encode: FAKEpassword!#$%&'()*+,/:;=?@[]
+FAKEpassword%21%23%24%25%26%27%28%29%2A%2B%2C%2F%3A%3B%3D%3F%40%5B%5D
+$
+```
 
 ## Migration Sources
 

database/driver.go

@@ -81,7 +81,8 @@ type Driver interface {
 func Open(url string) (Driver, error) {
 	u, err := nurl.Parse(url)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("Unable to parse URL. Did you escape all reserved URL characters? "+
+			"See: https://github.com/golang-migrate/migrate#database-urls Error: %v", err)
 	}
 
 	if u.Scheme == "" {

database/mysql/mysql.go

@@ -13,8 +13,13 @@ import (
 	nurl "net/url"
 	"strconv"
 	"strings"
+)
 
+import (
 	"github.com/go-sql-driver/mysql"
+)
+
+import (
 	"github.com/golang-migrate/migrate"
 	"github.com/golang-migrate/migrate/database"
 )
@@ -89,8 +94,26 @@ func WithInstance(instance *sql.DB, config *Config) (database.Driver, error) {
 	return mx, nil
 }
 
+// urlToMySQLConfig takes a net/url URL and returns a go-sql-driver/mysql Config.
+// Manually sets username and password to avoid net/url from url-encoding the reserved URL characters
+func urlToMySQLConfig(u nurl.URL) (*mysql.Config, error) {
+	origUserInfo := u.User
+	u.User = nil
+
+	c, err := mysql.ParseDSN(strings.TrimPrefix(u.String(), "mysql://"))
+	if err != nil {
+		return nil, err
+	}
+	if origUserInfo != nil {
+		c.User = origUserInfo.Username()
+		if p, ok := origUserInfo.Password(); ok {
+			c.Passwd = p
+		}
+	}
+	return c, nil
+}
+
 func (m *Mysql) Open(url string) (database.Driver, error) {
-	url = strings.TrimPrefix(url, "mysql://")
 	purl, err := nurl.Parse(url)
 	if err != nil {
 		return nil, err
@@ -100,7 +123,11 @@ func (m *Mysql) Open(url string) (database.Driver, error) {
 	q.Set("multiStatements", "true")
 	purl.RawQuery = q.Encode()
 
-	db, err := sql.Open("mysql", migrate.FilterCustomQuery(purl).String())
+	c, err := urlToMySQLConfig(*migrate.FilterCustomQuery(purl))
+	if err != nil {
+		return nil, err
+	}
+	db, err := sql.Open("mysql", c.FormatDSN())
 	if err != nil {
 		return nil, err
 	}

database/mysql/mysql_test.go

@@ -4,11 +4,15 @@ import (
 	"database/sql"
 	sqldriver "database/sql/driver"
 	"fmt"
-	// "io/ioutil"
-	// "log"
+	"net/url"
 	"testing"
+)
 
+import (
 	"github.com/go-sql-driver/mysql"
+)
+
+import (
 	dt "github.com/golang-migrate/migrate/database/testing"
 	mt "github.com/golang-migrate/migrate/testing"
 )
@@ -97,3 +101,55 @@ func TestLockWorks(t *testing.T) {
 			}
 		})
 }
+
+func TestURLToMySQLConfig(t *testing.T) {
+	testcases := []struct {
+		name        string
+		urlStr      string
+		expectedDSN string // empty string signifies that an error is expected
+	}{
+		{name: "no user/password", urlStr: "mysql://tcp(127.0.0.1:3306)/myDB?multiStatements=true",
+			expectedDSN: "tcp(127.0.0.1:3306)/myDB?multiStatements=true"},
+		{name: "only user", urlStr: "mysql://username@tcp(127.0.0.1:3306)/myDB?multiStatements=true",
+			expectedDSN: "username@tcp(127.0.0.1:3306)/myDB?multiStatements=true"},
+		{name: "only user - with encoded :",
+			urlStr:      "mysql://username%3A@tcp(127.0.0.1:3306)/myDB?multiStatements=true",
+			expectedDSN: "username:@tcp(127.0.0.1:3306)/myDB?multiStatements=true"},
+		{name: "only user - with encoded @",
+			urlStr:      "mysql://username%40@tcp(127.0.0.1:3306)/myDB?multiStatements=true",
+			expectedDSN: "username@@tcp(127.0.0.1:3306)/myDB?multiStatements=true"},
+		{name: "user/password", urlStr: "mysql://username:password@tcp(127.0.0.1:3306)/myDB?multiStatements=true",
+			expectedDSN: "username:password@tcp(127.0.0.1:3306)/myDB?multiStatements=true"},
+		// Not supported yet: https://github.com/go-sql-driver/mysql/issues/591
+		// {name: "user/password - user with encoded :",
+		// 	urlStr:      "mysql://username%3A:password@tcp(127.0.0.1:3306)/myDB?multiStatements=true",
+		// 	expectedDSN: "username::pasword@tcp(127.0.0.1:3306)/myDB?multiStatements=true"},
+		{name: "user/password - user with encoded @",
+			urlStr:      "mysql://username%40:password@tcp(127.0.0.1:3306)/myDB?multiStatements=true",
+			expectedDSN: "username@:password@tcp(127.0.0.1:3306)/myDB?multiStatements=true"},
+		{name: "user/password - password with encoded :",
+			urlStr:      "mysql://username:password%3A@tcp(127.0.0.1:3306)/myDB?multiStatements=true",
+			expectedDSN: "username:password:@tcp(127.0.0.1:3306)/myDB?multiStatements=true"},
+		{name: "user/password - password with encoded @",
+			urlStr:      "mysql://username:password%40@tcp(127.0.0.1:3306)/myDB?multiStatements=true",
+			expectedDSN: "username:password@@tcp(127.0.0.1:3306)/myDB?multiStatements=true"},
+	}
+	for _, tc := range testcases {
+		t.Run(tc.name, func(t *testing.T) {
+			u, err := url.Parse(tc.urlStr)
+			if err != nil {
+				t.Fatal("Failed to parse url string:", tc.urlStr, "error:", err)
+			}
+			if config, err := urlToMySQLConfig(*u); err == nil {
+				dsn := config.FormatDSN()
+				if dsn != tc.expectedDSN {
+					t.Error("Got unexpected DSN:", dsn, "!=", tc.expectedDSN)
+				}
+			} else {
+				if tc.expectedDSN != "" {
+					t.Error("Got unexpected error:", err, "urlStr:", tc.urlStr)
+				}
+			}
+		})
+	}
+}