refactor: Update models to use DatabaseV2 instead of DataSourceV2

crawlab-team · Aug 5, 2024 · e57655c · e57655c
1 parent e2cb99e
commit e57655c
Show file tree

Hide file tree

Showing 21 changed files with 44 additions and 54 deletions.
diff --git a/core/constants/ds.go b/core/constants/ds.go
@@ -16,13 +16,13 @@ const (
 )
 
 const (
-	DefaultMongoPort         = "27017"
-	DefaultMysqlPort         = "3306"
-	DefaultPostgresqlPort    = "5432"
-	DefaultMssqlPort         = "1433"
-	DefaultCockroachdbPort   = "26257"
-	DefaultElasticsearchPort = "9200"
-	DefaultKafkaPort         = "9092"
+	DefaultMongoPort         = 27017
+	DefaultMysqlPort         = 3306
+	DefaultPostgresqlPort    = 5432
+	DefaultMssqlPort         = 1433
+	DefaultCockroachdbPort   = 26257
+	DefaultElasticsearchPort = 9200
+	DefaultKafkaPort         = 9092
 )
 
 const (

diff --git a/core/controllers/database_v2.go b/core/controllers/database_v2.go
@@ -16,7 +16,7 @@ func PostDatabase(c *gin.Context) {
 		Type        string            `json:"type"`
 		Description string            `json:"description"`
 		Host        string            `json:"host"`
-		Port        string            `json:"port"`
+		Port        int               `json:"port"`
 		Url         string            `json:"url"`
 		Hosts       []string          `json:"hosts"`
 		Database    string            `json:"database"`
@@ -42,7 +42,6 @@ func PostDatabase(c *gin.Context) {
 		Host:        payload.Host,
 		Port:        payload.Port,
 		Url:         payload.Url,
-		Hosts:       payload.Hosts,
 		Database:    payload.Database,
 		Username:    payload.Username,
 		Password:    payload.Password,

diff --git a/core/controllers/router_v2.go b/core/controllers/router_v2.go
@@ -56,18 +56,6 @@ func InitRoutes(app *gin.Engine) (err error) {
 	groups := NewRouterGroups(app)
 
 	RegisterController(groups.AuthGroup, "/data/collections", NewControllerV2[models2.DataCollectionV2]())
-	RegisterController(groups.AuthGroup, "/databases", NewControllerV2[models2.DatabaseV2]([]Action{
-		{
-			Method:      http.MethodPost,
-			Path:        "",
-			HandlerFunc: PostDatabase,
-		},
-		{
-			Method:      http.MethodPut,
-			Path:        "/:id",
-			HandlerFunc: PutDatabaseById,
-		},
-	}...))
 	RegisterController(groups.AuthGroup, "/environments", NewControllerV2[models2.EnvironmentV2]())
 	RegisterController(groups.AuthGroup, "/nodes", NewControllerV2[models2.NodeV2]())
 	RegisterController(groups.AuthGroup, "/projects", NewControllerV2[models2.ProjectV2]([]Action{

diff --git a/core/ds/cockroachdb.go b/core/ds/cockroachdb.go
@@ -38,7 +38,7 @@ func NewDataSourceCockroachdbService(colId primitive.ObjectID, dsId primitive.Ob
 	if svc.ds.Host == "" {
 		svc.ds.Host = constants.DefaultHost
 	}
-	if svc.ds.Port == "" {
+	if svc.ds.Port == 0 {
 		svc.ds.Port = constants.DefaultCockroachdbPort
 	}
 

diff --git a/core/ds/es.go b/core/ds/es.go
@@ -179,7 +179,7 @@ func NewDataSourceElasticsearchService(colId primitive.ObjectID, dsId primitive.
 	if svc.ds.Host == "" {
 		svc.ds.Host = constants.DefaultHost
 	}
-	if svc.ds.Port == "" {
+	if svc.ds.Port == 0 {
 		svc.ds.Port = constants.DefaultElasticsearchPort
 	}
 

diff --git a/core/ds/kafka.go b/core/ds/kafka.go
@@ -81,7 +81,7 @@ func NewDataSourceKafkaService(colId primitive.ObjectID, dsId primitive.ObjectID
 	if svc.ds.Host == "" {
 		svc.ds.Host = constants.DefaultHost
 	}
-	if svc.ds.Port == "" {
+	if svc.ds.Port == 0 {
 		svc.ds.Port = constants.DefaultKafkaPort
 	}
 

diff --git a/core/ds/mongo.go b/core/ds/mongo.go
@@ -71,7 +71,7 @@ func NewDataSourceMongoService(colId primitive.ObjectID, dsId primitive.ObjectID
 	if svc.ds.Host == "" {
 		svc.ds.Host = constants.DefaultHost
 	}
-	if svc.ds.Port == "" {
+	if svc.ds.Port == 0 {
 		svc.ds.Port = constants.DefaultMongoPort
 	}
 

diff --git a/core/ds/mssql.go b/core/ds/mssql.go
@@ -39,7 +39,7 @@ func NewDataSourceMssqlService(colId primitive.ObjectID, dsId primitive.ObjectID
 	if svc.ds.Host == "" {
 		svc.ds.Host = constants.DefaultHost
 	}
-	if svc.ds.Port == "" {
+	if svc.ds.Port == 0 {
 		svc.ds.Port = constants.DefaultMssqlPort
 	}
 

diff --git a/core/ds/mysql.go b/core/ds/mysql.go
@@ -38,7 +38,7 @@ func NewDataSourceMysqlService(colId primitive.ObjectID, dsId primitive.ObjectID
 	if svc.ds.Host == "" {
 		svc.ds.Host = constants.DefaultHost
 	}
-	if svc.ds.Port == "" {
+	if svc.ds.Port == 0 {
 		svc.ds.Port = constants.DefaultMysqlPort
 	}
 

diff --git a/core/ds/postgresql.go b/core/ds/postgresql.go
@@ -39,7 +39,7 @@ func NewDataSourcePostgresqlService(colId primitive.ObjectID, dsId primitive.Obj
 	if svc.ds.Host == "" {
 		svc.ds.Host = constants.DefaultHost
 	}
-	if svc.ds.Port == "" {
+	if svc.ds.Port == 0 {
 		svc.ds.Port = constants.DefaultPostgresqlPort
 	}
 

diff --git a/core/models/models/data_source.go b/core/models/models/data_source.go
@@ -11,7 +11,7 @@ type DataSource struct {
 	Type        string             `json:"type" bson:"type"`
 	Description string             `json:"description" bson:"description"`
 	Host        string             `json:"host" bson:"host"`
-	Port        string             `json:"port" bson:"port"`
+	Port        int                `json:"port" bson:"port"`
 	Url         string             `json:"url" bson:"url"`
 	Hosts       []string           `json:"hosts" bson:"hosts"`
 	Database    string             `json:"database" bson:"database"`

diff --git a/core/models/models/v2/database_v2.go b/core/models/models/v2/database_v2.go
@@ -4,17 +4,22 @@ type DatabaseV2 struct {
 	any                     `collection:"databases"`
 	BaseModelV2[DatabaseV2] `bson:",inline"`
 	Name                    string            `json:"name" bson:"name"`
+	DataSource              string            `json:"data_source" bson:"data_source"`
 	Type                    string            `json:"type" bson:"type"`
 	Description             string            `json:"description" bson:"description"`
 	Host                    string            `json:"host" bson:"host"`
-	Port                    string            `json:"port" bson:"port"`
+	Port                    int               `json:"port" bson:"port"`
 	Url                     string            `json:"url" bson:"url"`
-	Hosts                   []string          `json:"hosts" bson:"hosts"`
 	Database                string            `json:"database" bson:"database"`
 	Username                string            `json:"username" bson:"username"`
 	Password                string            `json:"-,omitempty" bson:"password"`
 	ConnectType             string            `json:"connect_type" bson:"connect_type"`
 	Status                  string            `json:"status" bson:"status"`
 	Error                   string            `json:"error" bson:"error"`
 	Extra                   map[string]string `json:"extra,omitempty" bson:"extra,omitempty"`
+
+	MongoParams *struct {
+		AuthSource    string `json:"auth_source,omitempty" bson:"auth_source,omitempty"`
+		AuthMechanism string `json:"auth_mechanism,omitempty" bson:"auth_mechanism,omitempty"`
+	} `json:"mongo_params,omitempty" bson:"mongo_params,omitempty"`
 }
diff --git a/core/utils/cockroachdb.go b/core/utils/cockroachdb.go
@@ -28,7 +28,7 @@ func getCockroachdbSession(ctx context.Context, ds *models.DataSource) (s db.Ses
 	if ds.Host == "" {
 		host = constants.DefaultHost
 	}
-	if ds.Port == "" {
+	if ds.Port == 0 {
 		port = constants.DefaultCockroachdbPort
 	}
 
@@ -73,7 +73,7 @@ func getCockroachdbSessionV2(ctx context.Context, ds *models2.DatabaseV2) (s db.
 	if ds.Host == "" {
 		host = constants.DefaultHost
 	}
-	if ds.Port == "" {
+	if ds.Port == 0 {
 		port = constants.DefaultCockroachdbPort
 	}
 

diff --git a/core/utils/es.go b/core/utils/es.go
@@ -34,7 +34,7 @@ func getElasticsearchClient(ctx context.Context, ds *models.DataSource) (c *elas
 	if ds.Host == "" {
 		host = constants.DefaultHost
 	}
-	if ds.Port == "" {
+	if ds.Port == 0 {
 		port = constants.DefaultElasticsearchPort
 	}
 
@@ -97,7 +97,7 @@ func getElasticsearchClientV2(ctx context.Context, ds *models2.DatabaseV2) (c *e
 	if ds.Host == "" {
 		host = constants.DefaultHost
 	}
-	if ds.Port == "" {
+	if ds.Port == 0 {
 		port = constants.DefaultElasticsearchPort
 	}
 

diff --git a/core/utils/kafka.go b/core/utils/kafka.go
@@ -27,7 +27,7 @@ func getKafkaConnection(ctx context.Context, ds *models.DataSource) (c *kafka.Co
 	if ds.Host == "" {
 		host = constants.DefaultHost
 	}
-	if ds.Port == "" {
+	if ds.Port == 0 {
 		port = constants.DefaultKafkaPort
 	}
 
@@ -54,7 +54,7 @@ func getKafkaConnectionV2(ctx context.Context, ds *models2.DatabaseV2) (c *kafka
 	if ds.Host == "" {
 		host = constants.DefaultHost
 	}
-	if ds.Port == "" {
+	if ds.Port == 0 {
 		port = constants.DefaultKafkaPort
 	}
 

diff --git a/core/utils/mongo.go b/core/utils/mongo.go
@@ -66,7 +66,7 @@ func getMongoClient(ctx context.Context, ds *models.DataSource) (c *mongo2.Clien
 	if ds.Host == "" {
 		ds.Host = constants.DefaultHost
 	}
-	if ds.Port == "" {
+	if ds.Port == 0 {
 		ds.Port = constants.DefaultMongoPort
 	}
 
@@ -79,7 +79,6 @@ func getMongoClient(ctx context.Context, ds *models.DataSource) (c *mongo2.Clien
 	opts = append(opts, mongo.WithDb(ds.Database))
 	opts = append(opts, mongo.WithUsername(ds.Username))
 	opts = append(opts, mongo.WithPassword(ds.Password))
-	opts = append(opts, mongo.WithHosts(ds.Hosts))
 
 	// extra
 	if ds.Extra != nil {
@@ -105,7 +104,7 @@ func getMongoClientV2(ctx context.Context, ds *models2.DatabaseV2) (c *mongo2.Cl
 	if ds.Host == "" {
 		ds.Host = constants.DefaultHost
 	}
-	if ds.Port == "" {
+	if ds.Port == 0 {
 		ds.Port = constants.DefaultMongoPort
 	}
 
@@ -118,7 +117,6 @@ func getMongoClientV2(ctx context.Context, ds *models2.DatabaseV2) (c *mongo2.Cl
 	opts = append(opts, mongo.WithDb(ds.Database))
 	opts = append(opts, mongo.WithUsername(ds.Username))
 	opts = append(opts, mongo.WithPassword(ds.Password))
-	opts = append(opts, mongo.WithHosts(ds.Hosts))
 
 	// extra
 	if ds.Extra != nil {

diff --git a/core/utils/mssql.go b/core/utils/mssql.go
@@ -28,7 +28,7 @@ func getMssqlSession(ctx context.Context, ds *models.DataSource) (s db.Session,
 	if ds.Host == "" {
 		host = constants.DefaultHost
 	}
-	if ds.Port == "" {
+	if ds.Port == 0 {
 		port = constants.DefaultMssqlPort
 	}
 
@@ -73,7 +73,7 @@ func getMssqlSessionV2(ctx context.Context, ds *models2.DatabaseV2) (s db.Sessio
 	if ds.Host == "" {
 		host = constants.DefaultHost
 	}
-	if ds.Port == "" {
+	if ds.Port == 0 {
 		port = constants.DefaultMssqlPort
 	}
 

diff --git a/core/utils/mysql.go b/core/utils/mysql.go
@@ -28,7 +28,7 @@ func getMysqlSession(ctx context.Context, ds *models.DataSource) (s db.Session,
 	if ds.Host == "" {
 		host = constants.DefaultHost
 	}
-	if ds.Port == "" {
+	if ds.Port == 0 {
 		port = constants.DefaultMysqlPort
 	}
 
@@ -73,7 +73,7 @@ func getMysqlSessionV2(ctx context.Context, ds *models2.DatabaseV2) (s db.Sessio
 	if ds.Host == "" {
 		host = constants.DefaultHost
 	}
-	if ds.Port == "" {
+	if ds.Port == 0 {
 		port = constants.DefaultMysqlPort
 	}
 

diff --git a/core/utils/postgresql.go b/core/utils/postgresql.go
@@ -28,7 +28,7 @@ func getPostgresqlSession(ctx context.Context, ds *models.DataSource) (s db.Sess
 	if ds.Host == "" {
 		host = constants.DefaultHost
 	}
-	if ds.Port == "" {
+	if ds.Port == 0 {
 		port = constants.DefaultPostgresqlPort
 	}
 
@@ -73,7 +73,7 @@ func getPostgresqlSessionV2(ctx context.Context, ds *models2.DatabaseV2) (s db.S
 	if ds.Host == "" {
 		host = constants.DefaultHost
 	}
-	if ds.Port == "" {
+	if ds.Port == 0 {
 		port = constants.DefaultPostgresqlPort
 	}
 

diff --git a/db/mongo/client.go b/db/mongo/client.go
@@ -33,10 +33,10 @@ func GetMongoClient(opts ...ClientOption) (c *mongo.Client, err error) {
 			_opts.Host = "localhost"
 		}
 	}
-	if _opts.Port == "" {
-		_opts.Port = viper.GetString("mongo.port")
-		if _opts.Port == "" {
-			_opts.Port = "27017"
+	if _opts.Port == 0 {
+		_opts.Port = viper.GetInt("mongo.port")
+		if _opts.Port == 0 {
+			_opts.Port = 27017
 		}
 	}
 	if _opts.Db == "" {
@@ -123,7 +123,7 @@ func newMongoClient(ctx context.Context, _opts *ClientOptions) (c *mongo.Client,
 			mongoOpts.SetHosts(_opts.Hosts)
 		} else {
 			// hosts are unset
-			mongoOpts.ApplyURI(fmt.Sprintf("mongodb://%s:%s/%s", _opts.Host, _opts.Port, _opts.Db))
+			mongoOpts.ApplyURI(fmt.Sprintf("mongodb://%s:%d/%s", _opts.Host, _opts.Port, _opts.Db))
 		}
 	}
 

diff --git a/db/mongo/client_options.go b/db/mongo/client_options.go
@@ -8,7 +8,7 @@ type ClientOptions struct {
 	Context                 context.Context
 	Uri                     string
 	Host                    string
-	Port                    string
+	Port                    int
 	Db                      string
 	Hosts                   []string
 	Username                string
@@ -36,7 +36,7 @@ func WithHost(value string) ClientOption {
 	}
 }
 
-func WithPort(value string) ClientOption {
+func WithPort(value int) ClientOption {
 	return func(options *ClientOptions) {
 		options.Port = value
 	}