From 064c5bd7ee0b8cfba28707fba2f2bd3507d1c9d6 Mon Sep 17 00:00:00 2001 From: jc3wish <> Date: Sun, 25 Feb 2024 21:16:27 +0800 Subject: [PATCH 1/4] fixed output starrocks varchar(255) tansfer error bug https://github.com/brokercap/Bifrost/issues/266 --- plugin/mysql/src/transfer.go | 12 +- plugin/mysql/src/transfer_test.go | 183 ++++++++++++++++++++++++++++++ plugin/mysql/src/version.go | 4 +- 3 files changed, 193 insertions(+), 6 deletions(-) create mode 100644 plugin/mysql/src/transfer_test.go diff --git a/plugin/mysql/src/transfer.go b/plugin/mysql/src/transfer.go index 167658d4..9bb99b39 100644 --- a/plugin/mysql/src/transfer.go +++ b/plugin/mysql/src/transfer.go @@ -149,15 +149,15 @@ func (This *Conn) TransferToTypeByColumnType_Starrocks(columnType string, nullab toType = "VARCHAR(20)" case "int64": toType = "BIGINT(20)" - case "uint32", "uint24": + case "uint32": toType = "BIGINT(20)" - case "int32", "int24": + case "int32", "int24", "uint24": toType = "INT(11)" case "uint16": toType = "INT(11)" case "int16", "year(4)", "year(2)", "year": toType = "SMALLINT(6)" - case "uint8", "SMALLINT(uint8)": + case "uint8": toType = "SMALLINT(6)" case "int8", "bool": toType = "TINYINT(4)" @@ -227,7 +227,11 @@ func (This *Conn) TransferToTypeByColumnType_Starrocks(columnType string, nullab if lenStr == "" { toType = "VARCHAR(255)" } else { - toType = fmt.Sprintf("CHAR(%s)", lenStr) + if i == 0 { + toType = fmt.Sprintf("CHAR(%s)", lenStr) + } else { + toType = fmt.Sprintf("VARCHAR(%s)", lenStr) + } } break } diff --git a/plugin/mysql/src/transfer_test.go b/plugin/mysql/src/transfer_test.go new file mode 100644 index 00000000..fa23503b --- /dev/null +++ b/plugin/mysql/src/transfer_test.go @@ -0,0 +1,183 @@ +package src + +import ( + "testing" + + . "github.com/smartystreets/goconvey/convey" +) + +func TestTransferToTypeByColumnType_Starrocks(t *testing.T) { + type ColumnTypeStruct struct { + columnTypeList []string + nullable bool + destColumnType string + } + var caseList = []ColumnTypeStruct{ + { + columnTypeList: []string{"uint64"}, + nullable: false, + destColumnType: "VARCHAR(20)", + }, + { + columnTypeList: []string{"int64", "uint32"}, + nullable: false, + destColumnType: "BIGINT(20)", + }, + { + columnTypeList: []string{"int32", "uint24", "int24", "uint16"}, + nullable: false, + destColumnType: "INT(11)", + }, + { + columnTypeList: []string{"int16", "year(4)", "year(2)", "year", "uint8"}, + nullable: false, + destColumnType: "SMALLINT(6)", + }, + { + columnTypeList: []string{"int8", "bool"}, + nullable: false, + destColumnType: "TINYINT(4)", + }, + { + columnTypeList: []string{"float"}, + nullable: false, + destColumnType: "FLOAT", + }, + { + columnTypeList: []string{"double", "real"}, + nullable: false, + destColumnType: "DOUBLE", + }, + { + columnTypeList: []string{"decimal", "numeric"}, + nullable: false, + destColumnType: "DECIMAL", + }, + { + columnTypeList: []string{"decimal(9,2)"}, + nullable: false, + destColumnType: "Decimal(9,2)", + }, + { + columnTypeList: []string{"decimal(19,5)"}, + nullable: false, + destColumnType: "Decimal(19,5)", + }, + { + columnTypeList: []string{"decimal(38,2)"}, + nullable: false, + destColumnType: "Decimal(38,2)", + }, + { + columnTypeList: []string{"decimal(39,2)"}, + nullable: false, + destColumnType: "VARCHAR(78)", + }, + { + columnTypeList: []string{"decimal(88,2)"}, + nullable: false, + destColumnType: "VARCHAR(255)", + }, + { + columnTypeList: []string{"date", "Nullable(date)"}, + nullable: false, + destColumnType: "DATE", + }, + { + columnTypeList: []string{"json"}, + nullable: false, + destColumnType: "JSON", + }, + { + columnTypeList: []string{"time"}, + nullable: false, + destColumnType: "VARCHAR(10)", + }, + { + columnTypeList: []string{"enum"}, + nullable: false, + destColumnType: "VARCHAR(255)", + }, + { + columnTypeList: []string{"set"}, + nullable: false, + destColumnType: "VARCHAR(2048)", + }, + { + columnTypeList: []string{"string", "longblob", "longtext"}, + nullable: false, + destColumnType: "VARCHAR(163841)", + }, + { + columnTypeList: []string{"double(9,2)", "real(10,2)"}, + nullable: false, + destColumnType: "DOUBLE", + }, + { + columnTypeList: []string{"float(9,2)"}, + nullable: false, + destColumnType: "FLOAT", + }, + { + columnTypeList: []string{"bit"}, + nullable: false, + destColumnType: "BIGINT(20)", + }, + { + columnTypeList: []string{"timestamp(6)", "datetime(3)"}, + nullable: false, + destColumnType: "DATETIME", + }, + { + columnTypeList: []string{"time(6)", "time(1)"}, + nullable: false, + destColumnType: "VARCHAR(16)", + }, + { + columnTypeList: []string{"enum('a','b')"}, + nullable: false, + destColumnType: "VARCHAR(255)", + }, + { + columnTypeList: []string{"set('a','b')"}, + nullable: false, + destColumnType: "VARCHAR(2048)", + }, + { + columnTypeList: []string{"char(1)"}, + nullable: false, + destColumnType: "CHAR(1)", + }, + { + columnTypeList: []string{"char(255)"}, + nullable: false, + destColumnType: "CHAR(255)", + }, + { + columnTypeList: []string{"varchar(500)"}, + nullable: false, + destColumnType: "VARCHAR(500)", + }, + + { + columnTypeList: []string{"Nullable(varchar(500))"}, + nullable: false, + destColumnType: "VARCHAR(500)", + }, + { + columnTypeList: []string{"Nullable(int64)"}, + nullable: true, + destColumnType: "BIGINT(20) DEFAULT NULL", + }, + } + + c := &Conn{} + for _, caseInfo := range caseList { + for _, columnType := range caseInfo.columnTypeList { + Convey(caseInfo.destColumnType, t, func() { + toDestColumnType := c.TransferToTypeByColumnType_Starrocks(columnType, caseInfo.nullable) + So(toDestColumnType, ShouldEqual, caseInfo.destColumnType) + }) + } + } +} diff --git a/plugin/mysql/src/version.go b/plugin/mysql/src/version.go index 26d89730..9fe92f4d 100644 --- a/plugin/mysql/src/version.go +++ b/plugin/mysql/src/version.go @@ -1,4 +1,4 @@ package src -const VERSION = "v2.2.0" -const BIFROST_VERION = "v2.2.0" +const VERSION = "v2.3.5" +const BIFROST_VERION = "v2.3.4" From ec609d7ac9f574dc4b460d498d2e7353f183f2d8 Mon Sep 17 00:00:00 2001 From: jc3wish <> Date: Fri, 5 Apr 2024 12:05:58 +0800 Subject: [PATCH 2/4] record first boot time --- Bifrost.go | 108 +++++++++++++++--------------- admin/controller/index.go | 1 + admin/view/template/index.html | 9 ++- server/recovery_test.go | 33 +++++++--- server/server.go | 33 ++++++++++ server/server_save_recovery.go | 116 +++++++++++++++++---------------- server/server_test.go | 64 ++++++++++++++++++ 7 files changed, 243 insertions(+), 121 deletions(-) create mode 100644 server/server.go create mode 100644 server/server_test.go diff --git a/Bifrost.go b/Bifrost.go index c7cbbd4c..41320782 100755 --- a/Bifrost.go +++ b/Bifrost.go @@ -53,18 +53,19 @@ ___ ___ _ 纪念2022.08.28认识Monty并合影 ` -func printLogo(){ + +func printLogo() { var IpAndPort2 string //IpAndPort2 = strings.Replace(IpAndPort,"0.0.0.0","127.0.0.1",-1) if config.TLS { - IpAndPort2 = "https://"+config.Listen - }else{ - IpAndPort2 = "http://"+config.Listen + IpAndPort2 = "https://" + config.Listen + } else { + IpAndPort2 = "http://" + config.Listen } - logo = strings.Replace(logo,"{$version}",config.VERSION,-1) - logo = strings.Replace(logo,"{$Port}",IpAndPort2,-1) - logo = strings.Replace(logo,"{$Pid}",fmt.Sprint(os.Getpid()),-1) - logo = strings.Replace(logo,"{$system}",fmt.Sprint(runtime.GOARCH),-1) + logo = strings.Replace(logo, "{$version}", config.VERSION, -1) + logo = strings.Replace(logo, "{$Port}", IpAndPort2, -1) + logo = strings.Replace(logo, "{$Pid}", fmt.Sprint(os.Getpid()), -1) + logo = strings.Replace(logo, "{$system}", fmt.Sprint(runtime.GOARCH), -1) fmt.Println(logo) } @@ -80,33 +81,32 @@ Options: } func main() { - var BifrostConfigFile string var BifrostPid string var BifrostDataDir string var Version bool var Help bool defer func() { doSeverDbInfoFun() - if os.Getppid() == 1 && config.BifrostPidFile != ""{ + if os.Getppid() == 1 && config.BifrostPidFile != "" { os.Remove(config.BifrostPidFile) } - if err := recover();err != nil{ + if err := recover(); err != nil { log.Println(err) log.Println(string(debug.Stack())) return } }() - flag.StringVar(&BifrostConfigFile,"config", "", "-config") - flag.StringVar(&BifrostPid,"pid", "", "-pid") + flag.StringVar(&config.BifrostConfigFile, "config", "", "-config") + flag.StringVar(&BifrostPid, "pid", "", "-pid") flag.BoolVar(&BifrostDaemon, "d", false, "-d") - flag.StringVar(&BifrostDataDir,"data_dir", "", "-data_dir") + flag.StringVar(&BifrostDataDir, "data_dir", "", "-data_dir") flag.BoolVar(&Version, "v", false, "-v") flag.BoolVar(&Help, "h", false, "-h") flag.Usage = usage flag.Parse() - if Help{ + if Help { flag.Usage() os.Exit(0) } @@ -120,18 +120,18 @@ func main() { if os.Getppid() != 1 && runtime.GOOS != "windows" { // 因为有一些桌面系统,父进程开了子进程之后,父进程退出之后,并不是由 pid=1 的 systemd 进程接管,可能有些系统给每个桌面帐号重新分配了一下systemd 进程 // 这里每去判断 一下是不是 systemd 进程名，如果是的话，也认为是父进程被退出了 - cmdString := "ps -ef|grep "+strconv.Itoa(os.Getppid())+" | grep systemd|grep -v grep" - resultBytes,err := CmdShell(cmdString) - if err == nil && resultBytes != nil && string(resultBytes) != ""{ + cmdString := "ps -ef|grep " + strconv.Itoa(os.Getppid()) + " | grep systemd|grep -v grep" + resultBytes, err := CmdShell(cmdString) + if err == nil && resultBytes != nil && string(resultBytes) != "" { isDaemoProcess = true } - }else { + } else { isDaemoProcess = true } if !isDaemoProcess { - filePath,_:=filepath.Abs(os.Args[0]) //将命令行参数中执行文件路径转换成可用路径 - args:=append([]string{filePath},os.Args[1:]...) - os.StartProcess(filePath,args,&os.ProcAttr{Files:[]*os.File{os.Stdin,os.Stdout,os.Stderr}}) + filePath, _ := filepath.Abs(os.Args[0]) //将命令行参数中执行文件路径转换成可用路径 + args := append([]string{filePath}, os.Args[1:]...) + os.StartProcess(filePath, args, &os.ProcAttr{Files: []*os.File{os.Stdin, os.Stdout, os.Stderr}}) return } if config.BifrostPidFile == "" { @@ -139,18 +139,18 @@ func main() { } } - config.LoadConf(BifrostConfigFile) + config.LoadConf(config.BifrostConfigFile) if BifrostDataDir != "" { - config.SetConfigVal("Bifrostd","data_dir",BifrostDataDir) + config.SetConfigVal("Bifrostd", "data_dir", BifrostDataDir) } if BifrostPid != "" { - config.SetConfigVal("Bifrostd","pid",BifrostPid) + config.SetConfigVal("Bifrostd", "pid", BifrostPid) } config.InitParam() if !BifrostDaemon { printLogo() - }else{ + } else { printLogo() initLog() fmt.Printf("Please press the `Enter`\r") @@ -163,7 +163,7 @@ func main() { plugin.DoDynamicPlugin() server.InitStorage() - log.Println("Server started, Bifrost version",config.VERSION) + log.Println("Server started, Bifrost version", config.VERSION) doRecovery() @@ -171,41 +171,41 @@ func main() { ListenSignal() } -func initLog(){ - os.MkdirAll(config.BifrostLogDir,0700) +func initLog() { + os.MkdirAll(config.BifrostLogDir, 0700) t := time.Now().Format("2006-01-02") - LogFileName := config.BifrostLogDir+"/Bifrost_"+t+".log" + LogFileName := config.BifrostLogDir + "/Bifrost_" + t + ".log" f, err := os.OpenFile(LogFileName, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0700) //打开文件 - if err != nil{ - log.Println("log init error:",err) + if err != nil { + log.Println("log init error:", err) } log.SetOutput(f) - fmt.Println("log input to",LogFileName) + fmt.Println("log input to", LogFileName) } -func WritePid(){ +func WritePid() { if config.BifrostPidFile == "" { return } var err error var pidFileFd *os.File pidFileFd, err = os.OpenFile(config.BifrostPidFile, os.O_CREATE|os.O_RDWR, 0700) //打开文件 - if err !=nil{ - log.Println("Open BifrostPid Error; File:",config.BifrostPidFile,"; Error:",err) + if err != nil { + log.Println("Open BifrostPid Error; File:", config.BifrostPidFile, "; Error:", err) os.Exit(1) return } defer pidFileFd.Close() pidContent, err2 := ioutil.ReadAll(pidFileFd) - if string(pidContent) != ""{ + if string(pidContent) != "" { ExitBool := true - cmdString := "ps -ef|grep "+string(pidContent)+" | grep "+filepath.Base(os.Args[0]+"|grep -v grep") - resultBytes,err := CmdShell(cmdString) + cmdString := "ps -ef|grep " + string(pidContent) + " | grep " + filepath.Base(os.Args[0]+"|grep -v grep") + resultBytes, err := CmdShell(cmdString) // err 不为 nil 则代表没有grep 到进程，可以认为有可能被 kill -9 等操作了 - if err != nil && resultBytes != nil{ + if err != nil && resultBytes != nil { ExitBool = false - }else{ - log.Println(cmdString," result:",string(resultBytes)," err:",err,) + } else { + log.Println(cmdString, " result:", string(resultBytes), " err:", err) } if ExitBool { log.Println("Birostd server quit without updating PID file ; File:", config.BifrostPidFile, "; Error:", err2) @@ -213,34 +213,34 @@ func WritePid(){ } } os.Truncate(config.BifrostPidFile, 0) - pidFileFd.Seek(0,0) - io.WriteString(pidFileFd,fmt.Sprint(os.Getpid())) + pidFileFd.Seek(0, 0) + io.WriteString(pidFileFd, fmt.Sprint(os.Getpid())) } -func CmdShell(cmdString string)([]byte,error){ +func CmdShell(cmdString string) ([]byte, error) { switch runtime.GOOS { - case "linux","darwin","freebsd": + case "linux", "darwin", "freebsd": cmd := exec.Command("/bin/bash", "-c", cmdString) return cmd.Output() break default: break } - return nil,fmt.Errorf(runtime.GOOS+" not supported") + return nil, fmt.Errorf(runtime.GOOS + " not supported") } -func doSaveDbInfo(){ - if os.Getppid() != 1 && BifrostDaemon{ +func doSaveDbInfo() { + if os.Getppid() != 1 && BifrostDaemon { return } server.DoSaveSnapshotData() } -func doRecovery(){ +func doRecovery() { server.DoRecoverySnapshotData() } -func doSeverDbInfoFun() { +func doSeverDbInfoFun() { log.Println("save db server info data start... ") defer func() { if err := recover(); err != nil { @@ -260,15 +260,15 @@ func doSeverDbInfoFun() { log.Println("save db server info data success! ") } -func ListenSignal(){ +func ListenSignal() { signals := make(chan os.Signal, 1) signal.Notify(signals, syscall.SIGHUP, syscall.SIGTERM, syscall.SIGINT) for sig := range signals { - if sig == nil{ + if sig == nil { continue } doSeverDbInfoFun() - if config.BifrostPidFile != ""{ + if config.BifrostPidFile != "" { os.Remove(config.BifrostPidFile) } os.Exit(0) diff --git a/admin/controller/index.go b/admin/controller/index.go index 94314b7d..ffd4121e 100644 --- a/admin/controller/index.go +++ b/admin/controller/index.go @@ -43,6 +43,7 @@ type IndexController struct { // 首页 func (c *IndexController) Index() { c.SetTitle("Index") + c.SetData("ServerStartTime", server.GetServerStartTime().Format("2006-01-02")) c.AddAdminTemplate("index.html", "header.html", "footer.html") } diff --git a/admin/view/template/index.html b/admin/view/template/index.html index f4998484..32a7c62d 100644 --- a/admin/view/template/index.html +++ b/admin/view/template/index.html @@ -355,7 +355,7 @@
Bifrost动态
$("#GoVersion").text(d.GoVersion); $("#OSVersion").text(d.GOOS +" / "+d.GOARCH); }; - Ajax("GET","/overview",{},callback,true); + Ajax("GET","/overview",{},callback,false); } getOverView(); @@ -380,9 +380,11 @@
Bifrost动态
function loadNews() { var version = "{{.Version}}"; + var ServerStartTime = "{{.ServerStartTime}}"; $.ajax({ - url: "https://www.xbifrost.com/check_news/?version="+version, + url: "https://www.xbifrost.com/check_news/?version="+version+"&start="+ServerStartTime, type: "GET", + timeout: 5000, dataType: "jsonp", //指定服务器返回的数据类型 jsonpCallback:"jsonp", success: function (data) { @@ -422,6 +424,9 @@
Bifrost动态
html += ""; } $("#BifrostNewsTable").html(html); + }, + error: function (XMLHttpRequest, textStatus, errorThrown) { + showDonate(); } }); } diff --git a/server/recovery_test.go b/server/recovery_test.go index 3d875715..757ca130 100644 --- a/server/recovery_test.go +++ b/server/recovery_test.go @@ -1,9 +1,9 @@ package server import ( - "testing" - "os" "encoding/json" + "os" + "testing" ) func TestRecoveryJSON(t *testing.T) { @@ -11,21 +11,36 @@ func TestRecoveryJSON(t *testing.T) { var data map[string]dbSaveInfo var recoveryData recovery - errors := json.Unmarshal([]byte(str),&recoveryData) - if errors != nil{ - t.Fatal("recovery error:",errors.Error()) + errors := json.Unmarshal([]byte(str), &recoveryData) + if errors != nil { + t.Fatal("recovery error:", errors.Error()) return } - errors = json.Unmarshal(*recoveryData.DbInfo,&data) - if errors != nil{ - t.Fatal( "recorery db content errors;",errors) + errors = json.Unmarshal(*recoveryData.DbInfo, &data) + if errors != nil { + t.Fatal("recorery db content errors;", errors) os.Exit(1) return } if len(data["bk26"].TableMap) == 0 { t.Fatal("TableMap is empty") } - + if !recoveryData.StartTime.IsZero() { + t.Fatal("StartTime is not zero") + } t.Log(data) +} +func TestRecoveryJSON_StartTime(t *testing.T) { + var str string = `{"Version":"v1.4.2-release","StartTime":"2024-04-03T15:04:05Z"}` + var recoveryData recovery + errors := json.Unmarshal([]byte(str), &recoveryData) + if errors != nil { + t.Fatal("recovery error:", errors.Error()) + return + } + if recoveryData.StartTime.Format("2006-01-02") != "2024-04-03" { + t.Fatal("recovery StartTime != 2024-04-03") + return + } } diff --git a/server/server.go b/server/server.go new file mode 100644 index 00000000..0a8fa6fc --- /dev/null +++ b/server/server.go @@ -0,0 +1,33 @@ +package server + +import ( + "github.com/brokercap/Bifrost/config" + "os" + "time" +) + +var serverStartTime = time.Now() + +func GetServerStartTime() time.Time { + return serverStartTime +} + +func setServerStartTime(t time.Time) { + if t.IsZero() { + t = GetServerStartTimeByConfigFile() + } + if !serverStartTime.IsZero() { + if t.After(serverStartTime) { + return + } + } + serverStartTime = t +} + +func GetServerStartTimeByConfigFile() time.Time { + fInfo, err := os.Stat(config.BifrostConfigFile) + if err != nil { + return time.Now() + } + return fInfo.ModTime() +} diff --git a/server/server_save_recovery.go b/server/server_save_recovery.go index e4d5e192..319a3804 100644 --- a/server/server_save_recovery.go +++ b/server/server_save_recovery.go @@ -2,139 +2,143 @@ package server import ( "encoding/json" - "log" - "github.com/brokercap/Bifrost/plugin" "github.com/brokercap/Bifrost/config" - "sync" + "github.com/brokercap/Bifrost/plugin" + "github.com/brokercap/Bifrost/server/storage" "github.com/brokercap/Bifrost/server/user" "github.com/brokercap/Bifrost/server/warning" - "github.com/brokercap/Bifrost/server/storage" + "log" + "sync" "time" ) - var l sync.RWMutex type recovery struct { - Version string - ToServer *json.RawMessage - DbInfo *json.RawMessage - User *json.RawMessage - Warning *json.RawMessage + Version string + StartTime time.Time + ToServer *json.RawMessage + DbInfo *json.RawMessage + User *json.RawMessage + Warning *json.RawMessage } type recoveryDataSturct struct { - Version string - ToServer interface{} - DbInfo interface{} - User interface{} - Warning interface{} + Version string + StartTime time.Time + ToServer interface{} + DbInfo interface{} + User interface{} + Warning interface{} } -func DoRecoverySnapshotData(){ +func DoRecoverySnapshotData() { //这里初始化用户,第一次启动的情况下,配置文件中的用户需要初始化 user.InitUser() - fd,err := storage.GetDBInfo() + fd, err := storage.GetDBInfo() if err != nil { return } - if string(fd) == ""{ + if string(fd) == "" { return } var data recovery - errors := json.Unmarshal(fd,&data) - if errors != nil{ - log.Printf("recovery error:%s, data:%s \r\n",errors,string(fd)) + errors := json.Unmarshal(fd, &data) + if errors != nil { + log.Printf("recovery error:%s, data:%s \r\n", errors, string(fd)) return } - if data.ToServer != nil && string(*data.ToServer) != "{}"{ + setServerStartTime(data.StartTime) + if data.ToServer != nil && string(*data.ToServer) != "{}" { plugin.Recovery(data.ToServer) } - if data.DbInfo != nil && string(*data.DbInfo) != "{}"{ - Recovery(data.DbInfo,false) + if data.DbInfo != nil && string(*data.DbInfo) != "{}" { + Recovery(data.DbInfo, false) } if data.User != nil && string(*data.User) != "[]" { user.RecoveryUser(data.User) } - if data.Warning != nil && string(*data.Warning) != "{}"{ + if data.Warning != nil && string(*data.Warning) != "{}" { warning.RecoveryWarning(data.Warning) } } -func GetSnapshotData() ([]byte,error){ +func GetSnapshotData() ([]byte, error) { l.Lock() - defer func(){ + defer func() { l.Unlock() - if err :=recover();err!=nil{ + if err := recover(); err != nil { log.Println(err) } }() data := recoveryDataSturct{ - Version:config.VERSION, - ToServer:plugin.SaveToServerData(), - DbInfo:SaveDBInfoToFileData(), - User:user.GetUserList(), - Warning:warning.GetWarningConfigList(), + Version: config.VERSION, + StartTime: GetServerStartTime(), + ToServer: plugin.SaveToServerData(), + DbInfo: SaveDBInfoToFileData(), + User: user.GetUserList(), + Warning: warning.GetWarningConfigList(), } return json.Marshal(data) } -//只获取 数据源 和 目标库的镜像数据 -func GetSnapshotData2() ([]byte,error){ +// 只获取 数据源 和 目标库的镜像数据 +func GetSnapshotData2() ([]byte, error) { l.Lock() - defer func(){ + defer func() { l.Unlock() - if err :=recover();err!=nil{ + if err := recover(); err != nil { log.Println(err) } }() data := recoveryDataSturct{ - Version:config.VERSION, - ToServer:plugin.SaveToServerData(), - DbInfo:SaveDBInfoToFileData(), + Version: config.VERSION, + StartTime: GetServerStartTime(), + ToServer: plugin.SaveToServerData(), + DbInfo: SaveDBInfoToFileData(), } return json.Marshal(data) } - -func DoSaveSnapshotData(){ +func DoSaveSnapshotData() { var data []byte var err error - for i:=0;i<3;i++{ - data,err = GetSnapshotData2() - if err == nil{ + for i := 0; i < 3; i++ { + data, err = GetSnapshotData2() + if err == nil { break } time.Sleep(time.Duration(100) * time.Millisecond) } - if err != nil{ + if err != nil { SaveDBConfigInfo() return } storage.SaveDBInfo(data) } -func DoRecoveryByBackupData(fileContent string){ +func DoRecoveryByBackupData(fileContent string) { var data recovery - errors := json.Unmarshal([]byte(fileContent),&data) - if errors != nil{ - log.Printf("recovery error:%s, data:%s \r\n",errors,fileContent) + errors := json.Unmarshal([]byte(fileContent), &data) + if errors != nil { + log.Printf("recovery error:%s, data:%s \r\n", errors, fileContent) return } - if string(*data.ToServer) != "{}"{ + setServerStartTime(data.StartTime) + if string(*data.ToServer) != "{}" { plugin.Recovery(data.ToServer) } - if string(*data.DbInfo) != "{}"{ - Recovery(data.DbInfo,true) + if string(*data.DbInfo) != "{}" { + Recovery(data.DbInfo, true) } - if string(*data.Warning) != "{}"{ + if string(*data.Warning) != "{}" { warning.RecoveryWarning(data.Warning) } - if string(*data.User) != "[]"{ + if string(*data.User) != "[]" { user.RecoveryUser(data.User) } -} \ No newline at end of file +} diff --git a/server/server_test.go b/server/server_test.go new file mode 100644 index 00000000..471e3448 --- /dev/null +++ b/server/server_test.go @@ -0,0 +1,64 @@ +package server + +import ( + "errors" + "github.com/agiledragon/gomonkey" + "github.com/smartystreets/goconvey/convey" + "os" + "testing" + "time" +) + +func Test_setServerStartTime(t *testing.T) { + convey.Convey("set zero time,get time by config file return no-zero", t, func() { + patch := gomonkey.ApplyFunc(GetServerStartTimeByConfigFile, func() time.Time { + return time.Now() + }) + defer patch.Reset() + var t time.Time + setServerStartTime(t) + convey.So(GetServerStartTime().IsZero(), convey.ShouldBeFalse) + }) + + convey.Convey("set zero time,get time by config file return zero", t, func() { + patch := gomonkey.ApplyFunc(GetServerStartTimeByConfigFile, func() time.Time { + var t time.Time + return t + }) + defer patch.Reset() + var t time.Time + setServerStartTime(t) + convey.So(GetServerStartTime().IsZero(), convey.ShouldBeTrue) + }) +} + +func Test_setServerStartTime_AfterTime(t *testing.T) { + convey.Convey("set before time", t, func() { + var nowTime = time.Now() + var beforeTime = nowTime.AddDate(0, 0, -1) + setServerStartTime(nowTime) + setServerStartTime(beforeTime) + convey.So(GetServerStartTime(), convey.ShouldEqual, beforeTime) + }) +} + +func Test_setServerStartTime_BeforeTime(t *testing.T) { + convey.Convey("set before time", t, func() { + var nowTime = time.Now() + var beforeTime = nowTime.AddDate(0, 0, -1) + setServerStartTime(beforeTime) + setServerStartTime(nowTime) + convey.So(GetServerStartTime(), convey.ShouldEqual, beforeTime) + }) +} + +func Test_GetServerStartTimeByConfigFile(t *testing.T) { + convey.Convey("get config file error", t, func() { + patch := gomonkey.ApplyFunc(os.Stat, func(file string) (os.FileInfo, error) { + return nil, errors.New("test error") + }) + defer patch.Reset() + modTime := GetServerStartTimeByConfigFile() + convey.So(time.Now().Sub(modTime).Seconds(), convey.ShouldBeBetween, 0, 2) + }) +} From 1e473c99a2c3f56cb18ec8ac7c70e51ddf19ad81 Mon Sep 17 00:00:00 2001 From: jc3wish <> Date: Fri, 5 Apr 2024 12:33:56 +0800 Subject: [PATCH 3/4] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dmongo=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E6=BA=90=E5=85=A8=E9=87=8F=E5=8A=9F=E8=83=BD=E6=97=A0=E4=BA=A7?= =?UTF-8?q?=E7=9A=84bug=20https://github.com/brokercap/Bifrost/issues/272?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- input/mongo/batch.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/input/mongo/batch.go b/input/mongo/batch.go index 7c2dec98..3e0225c4 100644 --- a/input/mongo/batch.go +++ b/input/mongo/batch.go @@ -93,7 +93,7 @@ func (c *MongoInput) TableBatchStart(collection *mongo.Collection, perBatchLimit if len(batchResult) < perBatchLimit { break } - nextMinId = batchResult[len(batchResult)-1] + nextMinId = batchResult[len(batchResult)-1]["_id"] } return nil } From 4d4928b6af5cc9d9f6546481ef908e536208adba Mon Sep 17 00:00:00 2001 From: jc3wish <> Date: Fri, 5 Apr 2024 17:31:38 +0800 Subject: [PATCH 4/4] v2.3.5 --- README.EN.MD | 6 +++--- README.MD | 6 +++--- changelog.txt | 4 ++++ config/version.go | 2 +- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/README.EN.MD b/README.EN.MD index 0bd162cc..8f0c9cd6 100644 --- a/README.EN.MD +++ b/README.EN.MD @@ -121,11 +121,11 @@ After compiling, the corresponding platform name folder will be created in the t ##### Binary ``` -wget https://github.com/brokercap/Bifrost/releases/download/v2.3.4-beta/bifrost_v2.3.4-beta_Linux-amd64-bin.tar.gz +wget https://github.com/brokercap/Bifrost/releases/download/v2.3.5-beta/bifrost_v2.3.5-beta_Linux-amd64-bin.tar.gz -tar -zxvf bifrost_v2.3.4-beta_Linux-amd64-bin.tar.gz +tar -zxvf bifrost_v2.3.5-beta_Linux-amd64-bin.tar.gz -cd bifrost_v2.3.4-beta_Linux-amd64-bin/bin && chmod a+x ./Bifrost* +cd bifrost_v2.3.5-beta_Linux-amd64-bin/bin && chmod a+x ./Bifrost* ``` diff --git a/README.MD b/README.MD index 3dfbba0a..4e3c2544 100755 --- a/README.MD +++ b/README.MD @@ -124,11 +124,11 @@ make install prefix=./target ##### 二进制文件安装 `````sh -wget https://github.com/brokercap/Bifrost/releases/download/v2.3.4-beta/bifrost_v2.3.4-beta_Linux-amd64-bin.tar.gz +wget https://github.com/brokercap/Bifrost/releases/download/v2.3.5-beta/bifrost_v2.3.5-beta_Linux-amd64-bin.tar.gz -tar -zxvf bifrost_v2.3.4-beta_Linux-amd64-bin.tar.gz +tar -zxvf bifrost_v2.3.5-beta_Linux-amd64-bin.tar.gz -cd bifrost_v2.3.4-beta_Linux-amd64-bin/bin && chmod a+x ./Bifrost* +cd bifrost_v2.3.5-beta_Linux-amd64-bin/bin && chmod a+x ./Bifrost* ````` diff --git a/changelog.txt b/changelog.txt index d765b758..6d1c29aa 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,3 +1,7 @@ +v2.3.5-beta 2024-04-05 +1. fixed output starrocks varchar(255) tansfer error bug https://github.com/brokercap/Bifrost/issues/266 +2. 修复mongo数据源全量功能无效的bug https://github.com/brokercap/Bifrost/issues/272 + v2.3.4-beta 2023-12-31 1. 修复MySQL源,指定同步表配置后,但可能没生效的BUG BUG复现过程: a. 配置了一个T1表同步 b. 重启进程,并且T1表数据不再做任何更新 c. 配置T2表同步 d. 更新T1表的数据 2. 修复mysql 带有触发器的时候binlog解析失败的bug https://github.com/brokercap/Bifrost/issues/263 diff --git a/config/version.go b/config/version.go index 25edd533..771b4495 100755 --- a/config/version.go +++ b/config/version.go @@ -16,4 +16,4 @@ limitations under the License. package config -const VERSION = "v2.3.4-beta" +const VERSION = "v2.3.5-beta"