Merge pull request #276 from brokercap/v2.3

v2.3

Bifrost.go

@@ -53,18 +53,19 @@ ___         ___                   _
 纪念2022.08.28认识Monty并合影
 
                                        `
-func printLogo(){
+
+func printLogo() {
 	var IpAndPort2 string
 	//IpAndPort2 = strings.Replace(IpAndPort,"0.0.0.0","127.0.0.1",-1)
 	if config.TLS {
-		IpAndPort2 = "https://"+config.Listen
-	}else{
-		IpAndPort2 = "http://"+config.Listen
+		IpAndPort2 = "https://" + config.Listen
+	} else {
+		IpAndPort2 = "http://" + config.Listen
 	}
-	logo = strings.Replace(logo,"{$version}",config.VERSION,-1)
-	logo = strings.Replace(logo,"{$Port}",IpAndPort2,-1)
-	logo = strings.Replace(logo,"{$Pid}",fmt.Sprint(os.Getpid()),-1)
-	logo = strings.Replace(logo,"{$system}",fmt.Sprint(runtime.GOARCH),-1)
+	logo = strings.Replace(logo, "{$version}", config.VERSION, -1)
+	logo = strings.Replace(logo, "{$Port}", IpAndPort2, -1)
+	logo = strings.Replace(logo, "{$Pid}", fmt.Sprint(os.Getpid()), -1)
+	logo = strings.Replace(logo, "{$system}", fmt.Sprint(runtime.GOARCH), -1)
 	fmt.Println(logo)
 }
 
@@ -80,33 +81,32 @@ Options:
 }
 
 func main() {
-	var BifrostConfigFile string
 	var BifrostPid string
 	var BifrostDataDir string
 	var Version bool
 	var Help bool
 	defer func() {
 		doSeverDbInfoFun()
-		if os.Getppid() == 1 && config.BifrostPidFile != ""{
+		if os.Getppid() == 1 && config.BifrostPidFile != "" {
 			os.Remove(config.BifrostPidFile)
 		}
-		if err := recover();err != nil{
+		if err := recover(); err != nil {
 			log.Println(err)
 			log.Println(string(debug.Stack()))
 			return
 		}
 	}()
 
-	flag.StringVar(&BifrostConfigFile,"config", "", "-config")
-	flag.StringVar(&BifrostPid,"pid", "", "-pid")
+	flag.StringVar(&config.BifrostConfigFile, "config", "", "-config")
+	flag.StringVar(&BifrostPid, "pid", "", "-pid")
 	flag.BoolVar(&BifrostDaemon, "d", false, "-d")
-	flag.StringVar(&BifrostDataDir,"data_dir", "", "-data_dir")
+	flag.StringVar(&BifrostDataDir, "data_dir", "", "-data_dir")
 	flag.BoolVar(&Version, "v", false, "-v")
 	flag.BoolVar(&Help, "h", false, "-h")
 	flag.Usage = usage
 	flag.Parse()
 
-	if Help{
+	if Help {
 		flag.Usage()
 		os.Exit(0)
 	}
@@ -120,37 +120,37 @@ func main() {
 		if os.Getppid() != 1 && runtime.GOOS != "windows" {
 			// 因为有一些桌面系统,父进程开了子进程之后,父进程退出之后,并不是由 pid=1 的 systemd 进程接管,可能有些系统给每个桌面帐号重新分配了一下systemd 进程
 			// 这里每去判断 一下是不是 systemd 进程名，如果是的话，也认为是父进程被退出了
-			cmdString := "ps -ef|grep "+strconv.Itoa(os.Getppid())+" | grep systemd|grep -v grep"
-			resultBytes,err := CmdShell(cmdString)
-			if err == nil && resultBytes != nil && string(resultBytes) != ""{
+			cmdString := "ps -ef|grep " + strconv.Itoa(os.Getppid()) + " | grep systemd|grep -v grep"
+			resultBytes, err := CmdShell(cmdString)
+			if err == nil && resultBytes != nil && string(resultBytes) != "" {
 				isDaemoProcess = true
 			}
-		}else {
+		} else {
 			isDaemoProcess = true
 		}
 		if !isDaemoProcess {
-			filePath,_:=filepath.Abs(os.Args[0])  //将命令行参数中执行文件路径转换成可用路径
-			args:=append([]string{filePath},os.Args[1:]...)
-			os.StartProcess(filePath,args,&os.ProcAttr{Files:[]*os.File{os.Stdin,os.Stdout,os.Stderr}})
+			filePath, _ := filepath.Abs(os.Args[0]) //将命令行参数中执行文件路径转换成可用路径
+			args := append([]string{filePath}, os.Args[1:]...)
+			os.StartProcess(filePath, args, &os.ProcAttr{Files: []*os.File{os.Stdin, os.Stdout, os.Stderr}})
 			return
 		}
 		if config.BifrostPidFile == "" {
 			config.BifrostPidFile = "./Bifrost.pid"
 		}
 	}
 
-	config.LoadConf(BifrostConfigFile)
+	config.LoadConf(config.BifrostConfigFile)
 	if BifrostDataDir != "" {
-		config.SetConfigVal("Bifrostd","data_dir",BifrostDataDir)
+		config.SetConfigVal("Bifrostd", "data_dir", BifrostDataDir)
 	}
 	if BifrostPid != "" {
-		config.SetConfigVal("Bifrostd","pid",BifrostPid)
+		config.SetConfigVal("Bifrostd", "pid", BifrostPid)
 	}
 	config.InitParam()
 
 	if !BifrostDaemon {
 		printLogo()
-	}else{
+	} else {
 		printLogo()
 		initLog()
 		fmt.Printf("Please press the `Enter`\r")
@@ -163,84 +163,84 @@ func main() {
 	plugin.DoDynamicPlugin()
 	server.InitStorage()
 
-	log.Println("Server started, Bifrost version",config.VERSION)
+	log.Println("Server started, Bifrost version", config.VERSION)
 
 	doRecovery()
 
 	go manager.Start()
 	ListenSignal()
 }
 
-func initLog(){
-	os.MkdirAll(config.BifrostLogDir,0700)
+func initLog() {
+	os.MkdirAll(config.BifrostLogDir, 0700)
 	t := time.Now().Format("2006-01-02")
-	LogFileName := config.BifrostLogDir+"/Bifrost_"+t+".log"
+	LogFileName := config.BifrostLogDir + "/Bifrost_" + t + ".log"
 	f, err := os.OpenFile(LogFileName, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0700) //打开文件
-	if err != nil{
-		log.Println("log init error:",err)
+	if err != nil {
+		log.Println("log init error:", err)
 	}
 	log.SetOutput(f)
-	fmt.Println("log input to",LogFileName)
+	fmt.Println("log input to", LogFileName)
 }
 
-func WritePid(){
+func WritePid() {
 	if config.BifrostPidFile == "" {
 		return
 	}
 	var err error
 	var pidFileFd *os.File
 	pidFileFd, err = os.OpenFile(config.BifrostPidFile, os.O_CREATE|os.O_RDWR, 0700) //打开文件
-	if err !=nil{
-		log.Println("Open BifrostPid Error; File:",config.BifrostPidFile,"; Error:",err)
+	if err != nil {
+		log.Println("Open BifrostPid Error; File:", config.BifrostPidFile, "; Error:", err)
 		os.Exit(1)
 		return
 	}
 	defer pidFileFd.Close()
 	pidContent, err2 := ioutil.ReadAll(pidFileFd)
-	if string(pidContent) != ""{
+	if string(pidContent) != "" {
 		ExitBool := true
-		cmdString := "ps -ef|grep "+string(pidContent)+" | grep "+filepath.Base(os.Args[0]+"|grep -v grep")
-		resultBytes,err := CmdShell(cmdString)
+		cmdString := "ps -ef|grep " + string(pidContent) + " | grep " + filepath.Base(os.Args[0]+"|grep -v grep")
+		resultBytes, err := CmdShell(cmdString)
 		// err 不为 nil 则代表没有grep 到进程，可以认为有可能被 kill -9 等操作了
-		if err != nil &&  resultBytes != nil{
+		if err != nil && resultBytes != nil {
 			ExitBool = false
-		}else{
-			log.Println(cmdString," result:",string(resultBytes)," err:",err,)
+		} else {
+			log.Println(cmdString, " result:", string(resultBytes), " err:", err)
 		}
 		if ExitBool {
 			log.Println("Birostd server quit without updating PID file ; File:", config.BifrostPidFile, "; Error:", err2)
 			os.Exit(1)
 		}
 	}
 	os.Truncate(config.BifrostPidFile, 0)
-	pidFileFd.Seek(0,0)
-	io.WriteString(pidFileFd,fmt.Sprint(os.Getpid()))
+	pidFileFd.Seek(0, 0)
+	io.WriteString(pidFileFd, fmt.Sprint(os.Getpid()))
 }
 
-func CmdShell(cmdString string)([]byte,error){
+func CmdShell(cmdString string) ([]byte, error) {
 	switch runtime.GOOS {
-		case "linux","darwin","freebsd":
+	case "linux", "darwin", "freebsd":
 		cmd := exec.Command("/bin/bash", "-c", cmdString)
 		return cmd.Output()
 		break
 	default:
 		break
 	}
-	return nil,fmt.Errorf(runtime.GOOS+" not supported")
+	return nil, fmt.Errorf(runtime.GOOS + " not supported")
 }
 
-func doSaveDbInfo(){
-	if os.Getppid() != 1 && BifrostDaemon{
+func doSaveDbInfo() {
+	if os.Getppid() != 1 && BifrostDaemon {
 		return
 	}
 	server.DoSaveSnapshotData()
 }
 
-func doRecovery(){
+func doRecovery() {
 	server.DoRecoverySnapshotData()
 }
 
-func doSeverDbInfoFun()  {
+func doSeverDbInfoFun() {
 	log.Println("save db server info data start... ")
 	defer func() {
 		if err := recover(); err != nil {
@@ -260,15 +260,15 @@ func doSeverDbInfoFun()  {
 	log.Println("save db server info data success! ")
 }
 
-func ListenSignal(){
+func ListenSignal() {
 	signals := make(chan os.Signal, 1)
 	signal.Notify(signals, syscall.SIGHUP, syscall.SIGTERM, syscall.SIGINT)
 	for sig := range signals {
-		if sig == nil{
+		if sig == nil {
 			continue
 		}
 		doSeverDbInfoFun()
-		if config.BifrostPidFile != ""{
+		if config.BifrostPidFile != "" {
 			os.Remove(config.BifrostPidFile)
 		}
 		os.Exit(0)

README.EN.MD

@@ -121,11 +121,11 @@ After compiling, the corresponding platform name folder will be created in the t
 ##### Binary
 
 ```
-wget https://github.com/brokercap/Bifrost/releases/download/v2.3.4-beta/bifrost_v2.3.4-beta_Linux-amd64-bin.tar.gz
+wget https://github.com/brokercap/Bifrost/releases/download/v2.3.5-beta/bifrost_v2.3.5-beta_Linux-amd64-bin.tar.gz
 
-tar -zxvf bifrost_v2.3.4-beta_Linux-amd64-bin.tar.gz
+tar -zxvf bifrost_v2.3.5-beta_Linux-amd64-bin.tar.gz
 
-cd bifrost_v2.3.4-beta_Linux-amd64-bin/bin && chmod a+x ./Bifrost*
+cd bifrost_v2.3.5-beta_Linux-amd64-bin/bin && chmod a+x ./Bifrost*
 
 ```
 

README.MD

@@ -124,11 +124,11 @@ make install prefix=./target
 ##### 二进制文件安装
 `````sh
 
-wget https://github.com/brokercap/Bifrost/releases/download/v2.3.4-beta/bifrost_v2.3.4-beta_Linux-amd64-bin.tar.gz
+wget https://github.com/brokercap/Bifrost/releases/download/v2.3.5-beta/bifrost_v2.3.5-beta_Linux-amd64-bin.tar.gz
 
-tar -zxvf bifrost_v2.3.4-beta_Linux-amd64-bin.tar.gz
+tar -zxvf bifrost_v2.3.5-beta_Linux-amd64-bin.tar.gz
 
-cd bifrost_v2.3.4-beta_Linux-amd64-bin/bin && chmod a+x ./Bifrost*
+cd bifrost_v2.3.5-beta_Linux-amd64-bin/bin && chmod a+x ./Bifrost*
 
 `````
 

admin/controller/index.go

@@ -43,6 +43,7 @@ type IndexController struct {
 // 首页
 func (c *IndexController) Index() {
 	c.SetTitle("Index")
+	c.SetData("ServerStartTime", server.GetServerStartTime().Format("2006-01-02"))
 	c.AddAdminTemplate("index.html", "header.html", "footer.html")
 }
 

admin/view/template/index.html

@@ -355,7 +355,7 @@ <h5><a href="http://www.xbifrost.com" target="_blank">Bifrost动态</a></h5>
             $("#GoVersion").text(d.GoVersion);
             $("#OSVersion").text(d.GOOS +" / "+d.GOARCH);
         };
-        Ajax("GET","/overview",{},callback,true);
+        Ajax("GET","/overview",{},callback,false);
     }
     getOverView();
 
@@ -380,9 +380,11 @@ <h5><a href="http://www.xbifrost.com" target="_blank">Bifrost动态</a></h5>
 
     function loadNews() {
         var version = "{{.Version}}";
+        var ServerStartTime = "{{.ServerStartTime}}";
         $.ajax({
-            url: "https://www.xbifrost.com/check_news/?version="+version,
+            url: "https://www.xbifrost.com/check_news/?version="+version+"&start="+ServerStartTime,
             type: "GET",
+            timeout: 5000,
             dataType: "jsonp", //指定服务器返回的数据类型
             jsonpCallback:"jsonp",
             success: function (data) {
@@ -422,6 +424,9 @@ <h5><a href="http://www.xbifrost.com" target="_blank">Bifrost动态</a></h5>
                     html += "</tr>";
                 }
                 $("#BifrostNewsTable").html(html);
+            },
+            error: function (XMLHttpRequest, textStatus, errorThrown) {
+                showDonate();
             }
         });
     }

changelog.txt

@@ -1,3 +1,7 @@
+v2.3.5-beta   2024-04-05
+1. fixed output starrocks varchar(255) tansfer error bug https://github.com/brokercap/Bifrost/issues/266
+2. 修复mongo数据源全量功能无效的bug https://github.com/brokercap/Bifrost/issues/272
+
 v2.3.4-beta   2023-12-31
 1. 修复MySQL源,指定同步表配置后,但可能没生效的BUG BUG复现过程: a. 配置了一个T1表同步 b. 重启进程,并且T1表数据不再做任何更新 c. 配置T2表同步 d. 更新T1表的数据
 2. 修复mysql 带有触发器的时候binlog解析失败的bug https://github.com/brokercap/Bifrost/issues/263

config/version.go

@@ -16,4 +16,4 @@ limitations under the License.
 
 package config
 
-const VERSION = "v2.3.4-beta"
+const VERSION = "v2.3.5-beta"

input/mongo/batch.go

@@ -93,7 +93,7 @@ func (c *MongoInput) TableBatchStart(collection *mongo.Collection, perBatchLimit
 		if len(batchResult) < perBatchLimit {
 			break
 		}
-		nextMinId = batchResult[len(batchResult)-1]
+		nextMinId = batchResult[len(batchResult)-1]["_id"]
 	}
 	return nil
 }

plugin/mysql/src/transfer.go

@@ -149,15 +149,15 @@ func (This *Conn) TransferToTypeByColumnType_Starrocks(columnType string, nullab
 		toType = "VARCHAR(20)"
 	case "int64":
 		toType = "BIGINT(20)"
-	case "uint32", "uint24":
+	case "uint32":
 		toType = "BIGINT(20)"
-	case "int32", "int24":
+	case "int32", "int24", "uint24":
 		toType = "INT(11)"
 	case "uint16":
 		toType = "INT(11)"
 	case "int16", "year(4)", "year(2)", "year":
 		toType = "SMALLINT(6)"
-	case "uint8", "SMALLINT(uint8)":
+	case "uint8":
 		toType = "SMALLINT(6)"
 	case "int8", "bool":
 		toType = "TINYINT(4)"
@@ -227,7 +227,11 @@ func (This *Conn) TransferToTypeByColumnType_Starrocks(columnType string, nullab
 			if lenStr == "" {
 				toType = "VARCHAR(255)"
 			} else {
-				toType = fmt.Sprintf("CHAR(%s)", lenStr)
+				if i == 0 {
+					toType = fmt.Sprintf("CHAR(%s)", lenStr)
+				} else {
+					toType = fmt.Sprintf("VARCHAR(%s)", lenStr)
+				}
 			}
 			break
 		}