Skip to content

Commit

Permalink
feat: import from MySQL to serverless cluster (#99)
Browse files Browse the repository at this point in the history 
## What is the purpose of the change

Support feature import from MySQL to serverless cluster. It depends on 'mysql' command-line tool that the user should install.

## Brief change log

- Check dependency  'mysql' command-line tool.
- Download CA file for Windows and Linux.
- Use `mysqldump` to dump SQL files.
- Use `mysql` to import SQL files.
  • Loading branch information
@xuanyu66
xuanyu66 committed May 19, 2023
1 parent a90b380 commit c148916
Show file tree
Hide file tree
Showing 28 changed files with 1,745 additions and 298 deletions.
1 change: 1 addition & 0 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ generate-mocks: ## Generate mock objects
go install github.com/vektra/mockery/v2@latest
mockery --name TiDBCloudClient --recursive --output=internal/mock --outpkg mock --filename api_client.go
mockery --name EventsSender --recursive --output=internal/mock --outpkg mock --filename sender.go
mockery --name MySQLHelper --recursive --output=internal/mock --outpkg mock --filename mysql_helper.go

# Required to install go-swagger https://goswagger.io/install.html
.PHONY: generate-import-client
Expand Down
1 change: 1 addition & 0 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ require (
github.com/juju/errors v1.0.0
github.com/mattn/go-isatty v0.0.17
github.com/pelletier/go-toml v1.9.5
github.com/pingcap/errors v0.11.0
github.com/pingcap/log v1.1.0
github.com/rodaine/table v1.0.1
github.com/sahilm/fuzzy v0.1.0
Expand Down
291 changes: 90 additions & 201 deletions internal/cli/cluster/connectinfo.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,85 +24,14 @@ import (
"tidbcloud-cli/internal/config"
"tidbcloud-cli/internal/flag"
"tidbcloud-cli/internal/service/cloud"
connectInfoModel "tidbcloud-cli/pkg/tidbcloud/connect_info/models"
"tidbcloud-cli/internal/util"

clusterApi "github.com/c4pt0r/go-tidbcloud-sdk-v1/client/cluster"

"github.com/juju/errors"
"github.com/spf13/cobra"
)

// xxxDisplayName is used to display in interactive mode
// xxxInputName is used to input in non-interactive mode and display in help message
const (
GeneralParameterID string = "general"
GeneralParameterDisplayName string = "General"
GeneralParameterInputName string = "general"
MysqlCliID string = "mysql_cli"
MysqlCliDisplayName string = "MySQL CLI"
MysqlCliInputName string = "mysql_cli"
MyCliID string = "mycli"
MyCliDisplayName string = "MyCLI"
MyCliInputName string = "mycli"
LibMysqlClientID string = "libmysqlclient"
LibMysqlClientDisplayName string = "libmysqlclient"
LibMysqlClientInputName string = "libmysqlclient"
MysqlClientID string = "mysqlclient"
MysqlClientDisplayName string = "mysqlclient (Python)"
MysqlClientInputName string = "python_mysqlclient"
PyMysqlID string = "pymysql"
PyMysqlDisplayName string = "PyMySQL"
PyMysqlInputName string = "pymysql"
MysqlConnectorPythonID string = "mysql_connector_python"
MysqlConnectorPythonDisplayName string = "MySQL Connector/Python"
MysqlConnectorPythonInputName string = "mysql_connector_python"
MysqlConnectorJavaID string = "mysql_connector_java"
MysqlConnectorJavaDisplayName string = "MySQL Connector/Java"
MysqlConnectorJavaInputName string = "mysql_connector_java"
GoMysqlDriverID string = "go_mysql_driver"
GoMysqlDriverDisplayName string = "Go MySQL Driver"
GoMysqlDriverInputName string = "go_mysql_driver"
NodeMysql2ID string = "node_mysql2"
NodeMysql2DisplayName string = "Node MySQL 2"
NodeMysql2InputName string = "node_mysql2"
Mysql2RubyID string = "mysql2_ruby"
Mysql2RubyDisplayName string = "Mysql2 (Ruby)"
Mysql2RubyInputName string = "ruby_mysql2"
MysqliID string = "mysqli"
MysqliDisplayName string = "MySQLi (PHP)"
MysqliInputName string = "php_mysqli"
MysqlRustID string = "mysql_rust"
MysqlRustDisplayName string = "mysql (Rust)"
MysqlRustInputName string = "rust_mysql"
MybatisID string = "mybatis"
MybatisDisplayName string = "MyBatis"
MybatisInputName string = "mybatis"
HibernateID string = "hibernate"
HibernateDisplayName string = "Hibernate"
HibernateInputName string = "hibernate"
SpringBootID string = "spring_boot"
SpringBootDisplayName string = "Spring Boot"
SpringBootInputName string = "spring_boot"
GormID string = "gorm"
GormDisplayName string = "GORM"
GormInputName string = "gorm"
PrismaID string = "prisma"
PrismaDisplayName string = "Prisma"
PrismaInputName string = "prisma"
SequelizeID string = "sequelize_mysql2"
SequelizeDisplayName string = "Sequelize (mysql2)"
SequelizeInputName string = "sequelize_mysql2"
DjangoID string = "django_tidb"
DjangoDisplayName string = "Django (django_tidb)"
DjangoInputName string = "django_tidb"
SQLAlchemyID string = "sqlalchemy_mysqlclient"
SqlAlchemyDisplayName string = "SQLAlchemy (mysqlclient)"
SqlAlchemyInputName string = "sqlalchemy_mysqlclient"
ActiveRecordID string = "active_record"
ActiveRecordDisplayName string = "Active Record"
ActiveRecordInputName string = "active_record"
)

type connectInfoOpts struct {
interactive bool
}
Expand All @@ -119,131 +48,131 @@ func (c connectInfoOpts) NonInteractiveFlags() []string {
// Display clients name orderly in interactive mode
var connectClientsList = []string{
// pure parameter
GeneralParameterDisplayName,
util.GeneralParameterDisplayName,

// CLI
MysqlCliDisplayName,
MyCliDisplayName,
util.MysqlCliDisplayName,
util.MyCliDisplayName,

// driver
LibMysqlClientDisplayName,
MysqlClientDisplayName,
PyMysqlDisplayName,
MysqlConnectorPythonDisplayName,
MysqlConnectorJavaDisplayName,
GoMysqlDriverDisplayName,
NodeMysql2DisplayName,
Mysql2RubyDisplayName,
MysqliDisplayName,
MysqlRustDisplayName,
util.LibMysqlClientDisplayName,
util.MysqlClientDisplayName,
util.PyMysqlDisplayName,
util.MysqlConnectorPythonDisplayName,
util.MysqlConnectorJavaDisplayName,
util.GoMysqlDriverDisplayName,
util.NodeMysql2DisplayName,
util.Mysql2RubyDisplayName,
util.MysqliDisplayName,
util.MysqlRustDisplayName,

// ORM
MybatisDisplayName,
HibernateDisplayName,
SpringBootDisplayName,
GormDisplayName,
PrismaDisplayName,
SequelizeDisplayName,
DjangoDisplayName,
SqlAlchemyDisplayName,
ActiveRecordDisplayName,
util.MybatisDisplayName,
util.HibernateDisplayName,
util.SpringBootDisplayName,
util.GormDisplayName,
util.PrismaDisplayName,
util.SequelizeDisplayName,
util.DjangoDisplayName,
util.SqlAlchemyDisplayName,
util.ActiveRecordDisplayName,
}

// Display clients name orderly in help message
var connectClientsListForHelp = []string{
// pure parameter
GeneralParameterInputName,
util.GeneralParameterInputName,

// CLI
MysqlCliInputName,
MyCliInputName,
util.MysqlCliInputName,
util.MyCliInputName,

// driver
LibMysqlClientInputName,
MysqlClientInputName,
PyMysqlInputName,
MysqlConnectorPythonInputName,
MysqlConnectorJavaInputName,
GoMysqlDriverInputName,
NodeMysql2InputName,
Mysql2RubyInputName,
MysqliInputName,
MysqlRustInputName,
util.LibMysqlClientInputName,
util.MysqlClientInputName,
util.PyMysqlInputName,
util.MysqlConnectorPythonInputName,
util.MysqlConnectorJavaInputName,
util.GoMysqlDriverInputName,
util.NodeMysql2InputName,
util.Mysql2RubyInputName,
util.MysqliInputName,
util.MysqlRustInputName,

// ORM
MybatisInputName,
HibernateInputName,
SpringBootInputName,
GormInputName,
PrismaInputName,
SequelizeInputName,
DjangoInputName,
SqlAlchemyInputName,
ActiveRecordInputName,
util.MybatisInputName,
util.HibernateInputName,
util.SpringBootInputName,
util.GormInputName,
util.PrismaInputName,
util.SequelizeInputName,
util.DjangoInputName,
util.SqlAlchemyInputName,
util.ActiveRecordInputName,
}

var clientsForInteractiveMap = map[string]string{
// pure parameter
GeneralParameterDisplayName: GeneralParameterID,
util.GeneralParameterDisplayName: util.GeneralParameterID,

// CLI
MysqlCliDisplayName: MysqlCliID,
MyCliDisplayName: MyCliID,
util.MysqlCliDisplayName: util.MysqlCliID,
util.MyCliDisplayName: util.MyCliID,

// driver
LibMysqlClientDisplayName: LibMysqlClientID,
MysqlClientDisplayName: MysqlClientID,
PyMysqlDisplayName: PyMysqlID,
MysqlConnectorPythonDisplayName: MysqlConnectorPythonID,
MysqlConnectorJavaDisplayName: MysqlConnectorJavaID,
GoMysqlDriverDisplayName: GoMysqlDriverID,
NodeMysql2DisplayName: NodeMysql2ID,
Mysql2RubyDisplayName: Mysql2RubyID,
MysqliDisplayName: MysqliID,
MysqlRustDisplayName: MysqlRustID,
util.LibMysqlClientDisplayName: util.LibMysqlClientID,
util.MysqlClientDisplayName: util.MysqlClientID,
util.PyMysqlDisplayName: util.PyMysqlID,
util.MysqlConnectorPythonDisplayName: util.MysqlConnectorPythonID,
util.MysqlConnectorJavaDisplayName: util.MysqlConnectorJavaID,
util.GoMysqlDriverDisplayName: util.GoMysqlDriverID,
util.NodeMysql2DisplayName: util.NodeMysql2ID,
util.Mysql2RubyDisplayName: util.Mysql2RubyID,
util.MysqliDisplayName: util.MysqliID,
util.MysqlRustDisplayName: util.MysqlRustID,

// ORM
MybatisDisplayName: MybatisID,
HibernateDisplayName: HibernateID,
SpringBootDisplayName: SpringBootID,
GormDisplayName: GormID,
PrismaDisplayName: PrismaID,
SequelizeDisplayName: SequelizeID,
DjangoDisplayName: DjangoID,
SqlAlchemyDisplayName: SQLAlchemyID,
ActiveRecordDisplayName: ActiveRecordID,
util.MybatisDisplayName: util.MybatisID,
util.HibernateDisplayName: util.HibernateID,
util.SpringBootDisplayName: util.SpringBootID,
util.GormDisplayName: util.GormID,
util.PrismaDisplayName: util.PrismaID,
util.SequelizeDisplayName: util.SequelizeID,
util.DjangoDisplayName: util.DjangoID,
util.SqlAlchemyDisplayName: util.SQLAlchemyID,
util.ActiveRecordDisplayName: util.ActiveRecordID,
}

var clientsForHelpMap = map[string]string{
// pure parameter
GeneralParameterInputName: GeneralParameterID,
util.GeneralParameterInputName: util.GeneralParameterID,

// CLI
MysqlCliInputName: MysqlCliID,
MyCliInputName: MyCliID,
util.MysqlCliInputName: util.MysqlCliID,
util.MyCliInputName: util.MyCliID,

// driver
LibMysqlClientInputName: LibMysqlClientID,
MysqlClientInputName: MysqlClientID,
PyMysqlInputName: PyMysqlID,
MysqlConnectorPythonInputName: MysqlConnectorPythonID,
MysqlConnectorJavaInputName: MysqlConnectorJavaID,
GoMysqlDriverInputName: GoMysqlDriverID,
NodeMysql2InputName: NodeMysql2ID,
Mysql2RubyInputName: Mysql2RubyID,
MysqliInputName: MysqliID,
MysqlRustInputName: MysqlRustID,
util.LibMysqlClientInputName: util.LibMysqlClientID,
util.MysqlClientInputName: util.MysqlClientID,
util.PyMysqlInputName: util.PyMysqlID,
util.MysqlConnectorPythonInputName: util.MysqlConnectorPythonID,
util.MysqlConnectorJavaInputName: util.MysqlConnectorJavaID,
util.GoMysqlDriverInputName: util.GoMysqlDriverID,
util.NodeMysql2InputName: util.NodeMysql2ID,
util.Mysql2RubyInputName: util.Mysql2RubyID,
util.MysqliInputName: util.MysqliID,
util.MysqlRustInputName: util.MysqlRustID,

// ORM
MybatisInputName: MybatisID,
HibernateInputName: HibernateID,
SpringBootInputName: SpringBootID,
GormInputName: GormID,
PrismaInputName: PrismaID,
SequelizeInputName: SequelizeID,
DjangoInputName: DjangoID,
SqlAlchemyInputName: SQLAlchemyID,
ActiveRecordInputName: ActiveRecordID,
util.MybatisInputName: util.MybatisID,
util.HibernateInputName: util.HibernateID,
util.SpringBootInputName: util.SpringBootID,
util.GormInputName: util.GormID,
util.PrismaInputName: util.PrismaID,
util.SequelizeInputName: util.SequelizeID,
util.DjangoInputName: util.DjangoID,
util.SqlAlchemyInputName: util.SQLAlchemyID,
util.ActiveRecordInputName: util.ActiveRecordID,
}

// Display operating system orderly in interactive mode
Expand Down Expand Up @@ -271,20 +200,6 @@ var operatingSystemListForHelp = []string{
"Others",
}

var caPath = map[string]string{
"macos": "/etc/ssl/cert.pem",
"alpine": "/etc/ssl/cert.pem",
"centos": "/etc/pki/tls/certs/ca-bundle.crt",
"redhat": "/etc/pki/tls/certs/ca-bundle.crt",
"fedora": "/etc/pki/tls/certs/ca-bundle.crt",
"debian": "/etc/ssl/certs/ca-certificates.crt",
"ubuntu": "/etc/ssl/certs/ca-certificates.crt",
"arch": "/etc/ssl/certs/ca-certificates.crt",
"windows": "<path_to_ca_cert>",
"opensuse": "/etc/ssl/ca-bundle.pem",
"others": "<path_to_ca_cert>",
}

// Cluster type
const (
SERVERLESS = "SERVERLESS"
Expand Down Expand Up @@ -439,7 +354,7 @@ func ConnectInfoCmd(h *internal.Helper) *cobra.Command {
if err != nil {
return err
}
connectionString, err := generateConnectionString(connectInfo, client, host, defaultUser, port, clusterType, operatingSystem)
connectionString, err := util.GenerateConnectionString(connectInfo, client, host, defaultUser, port, clusterType, operatingSystem, util.Shell)
if err != nil {
return err
}
Expand All @@ -458,32 +373,6 @@ func ConnectInfoCmd(h *internal.Helper) *cobra.Command {
return cmd
}

func generateConnectionString(connectInfo *connectInfoModel.ConnectInfo, client string, host string, user string, port string, clusterType string, operatingSystem string) (string, error) {
if client == GeneralParameterID {
return fmt.Sprintf(`Host: %s
Port: %s
User: %s`,
host, port, user), nil
}

for _, clientData := range connectInfo.ClientData {
// find user chose client
if strings.EqualFold(clientData.ID, client) {
for _, content := range clientData.Content {
if strings.EqualFold(clusterType, content.Type) {
connectionString := content.ConnectionString
connectionString = strings.Replace(connectionString, "${host}", host, -1)
connectionString = strings.Replace(connectionString, "${username}", user, -1)
connectionString = strings.Replace(connectionString, "${port}", port, -1)
connectionString = strings.Replace(connectionString, "${ca_root_path}", caPath[strings.ToLower(operatingSystem)], -1)
return connectionString, nil
}
}
}
}
return "", errors.New("failed to generate connection string")
}

func contains(str string, vec []string) bool {
for _, v := range vec {
if strings.EqualFold(str, v) {
Expand Down
Loading

0 comments on commit c148916

Please sign in to comment.