Merge pull request #354 from YangSen-qn/optimize_check_file_change

batch upload & download: optimize check whether file change

Author: bachue

iqshell/common/utils/file.go

@@ -132,8 +132,8 @@ func NetworkFileLength(srcResUrl string) (fileSize int64, err *data.CodeError) {
 	return
 }
 
-func IsFileMatchFileModifyTime(filePath string, modifyTime int64) (match bool, err *data.CodeError) {
-	if time, e := FileModify(filePath); e != nil {
+func IsLocalFileMatchFileModifyTime(filePath string, modifyTime int64) (match bool, err *data.CodeError) {
+	if time, e := LocalFileModify(filePath); e != nil {
 		return false, e
 	} else if time != modifyTime {
 		return false, data.NewEmptyError().AppendDescF("modifyTime don't match, except:%d but:%d", modifyTime, time)
@@ -142,22 +142,6 @@ func IsFileMatchFileModifyTime(filePath string, modifyTime int64) (match bool, e
 	}
 }
 
-func FileModify(filePath string) (int64, *data.CodeError) {
-	if IsNetworkSource(filePath) {
-		return NetworkFileModify(filePath)
-	} else {
-		return LocalFileModify(filePath)
-	}
-}
-
-func NetworkFileModify(filePath string) (int64, *data.CodeError) {
-	fileStatus, err := os.Stat(filePath)
-	if err != nil {
-		return 0, data.NewEmptyError().AppendDescF("get file : get status error:%v", err)
-	}
-	return fileStatus.ModTime().Unix(), nil
-}
-
 func LocalFileModify(filePath string) (int64, *data.CodeError) {
 	fileStatus, err := os.Stat(filePath)
 	if err != nil {

iqshell/storage/object/download/downloader.go

@@ -84,7 +84,7 @@ func Download(info *ApiInfo) (res *ApiResult, err *data.CodeError) {
 		if !res.IsExist {
 			err = utils.CreateDirIfNotExist(f.toAbsFile)
 		}
-		res.FileModifyTime, _ = utils.FileModify(f.toAbsFile)
+		res.FileModifyTime, _ = utils.LocalFileModify(f.toAbsFile)
 		return res, err
 	}
 
@@ -117,7 +117,7 @@ func Download(info *ApiInfo) (res *ApiResult, err *data.CodeError) {
 
 		if mErr == nil && checkResult.Match {
 			// 文件已下载，并在文件匹配，不再下载
-			if fileModifyTime, fErr := utils.FileModify(f.toAbsFile); fErr != nil {
+			if fileModifyTime, fErr := utils.LocalFileModify(f.toAbsFile); fErr != nil {
 				log.WarningF("Get file ModifyTime error:%v", fErr)
 			} else {
 				res.FileModifyTime = fileModifyTime
@@ -152,9 +152,10 @@ func Download(info *ApiInfo) (res *ApiResult, err *data.CodeError) {
 		return
 	}
 
-	res.FileModifyTime, err = utils.FileModify(f.toAbsFile)
-	if err != nil {
-		return
+	if fStatus, sErr := os.Stat(f.toAbsFile); sErr != nil {
+		return res, data.NewEmptyError().AppendDesc("get file stat error after download").AppendError(sErr)
+	} else {
+		res.FileModifyTime = fStatus.ModTime().Unix()
 	}
 
 	// 检查下载后的数据是否符合预期

iqshell/storage/object/download/operations/batch.go

@@ -2,6 +2,7 @@ package operations
 
 import (
 	"github.com/qiniu/qshell/v2/iqshell"
+	"github.com/qiniu/qshell/v2/iqshell/common/alert"
 	"github.com/qiniu/qshell/v2/iqshell/common/data"
 	"github.com/qiniu/qshell/v2/iqshell/common/export"
 	"github.com/qiniu/qshell/v2/iqshell/common/flow"
@@ -30,6 +31,9 @@ func (info *BatchDownloadWithConfigInfo) Check() *data.CodeError {
 	if err := info.Info.Check(); err != nil {
 		return err
 	}
+	if len(info.LocalDownloadConfig) == 0 {
+		return alert.CannotEmptyError("LocalDownloadConfig", "")
+	}
 	return nil
 }
 
@@ -251,8 +255,8 @@ func BatchDownload(cfg *iqshell.Config, info BatchDownloadInfo) {
 				return true, data.NewEmptyError().AppendDescF("result is invalid:%+v", result)
 			}
 
-			isLocalFileNotChange, _ := utils.IsFileMatchFileModifyTime(apiInfo.ToFile, result.FileModifyTime)
-			isServerFileNotChange := apiInfo.ServerFilePutTime == recordApiInfo.ServerFilePutTime
+			isLocalFileNotChange, _ := utils.IsLocalFileMatchFileModifyTime(apiInfo.ToFile, result.FileModifyTime)
+			isServerFileNotChange := apiInfo.ServerFileHash == recordApiInfo.ServerFileHash
 			// 本地文件和服务端文件均没有变化，则不需要重新下载
 			if isLocalFileNotChange && isServerFileNotChange {
 				return false, nil

iqshell/storage/object/upload/operations/batch.go

@@ -328,18 +328,22 @@ func batchUploadFlow(info BatchUpload2Info, uploadConfig UploadConfig, dbPath st
 			}
 
 			// 本地文件和服务端文件均没有变化，则不需要重新上传
-			stat, sErr := object.Status(object.StatusApiInfo{
-				Bucket:   uploadInfo.ToBucket,
-				Key:      uploadInfo.SaveKey,
-				NeedPart: false,
-			})
-			if sErr != nil {
-				return true, data.NewEmptyError().AppendDesc("get stat from server").AppendError(sErr)
+			isServerFileNotChange := true
+			if uploadConfig.CheckHash {
+				// 检测 hash 需要调用 Stat 接口查询 hash，如果用户不检测 hash 则认为服务端文件没有变化。
+				stat, sErr := object.Status(object.StatusApiInfo{
+					Bucket:   uploadInfo.ToBucket,
+					Key:      uploadInfo.SaveKey,
+					NeedPart: false,
+				})
+				if sErr != nil {
+					return true, data.NewEmptyError().AppendDesc("get stat from server").AppendError(sErr)
+				}
+				isServerFileNotChange = stat.Hash == result.ServerFileHash
 			}
 
 			// LocalFileModifyTime 单位是 100ns
-			isLocalFileNotChange, mErr := utils.IsFileMatchFileModifyTime(uploadInfo.FilePath, recordUploadInfo.LocalFileModifyTime/10000000)
-			isServerFileNotChange := stat.PutTime == result.ServerPutTime
+			isLocalFileNotChange, mErr := utils.IsLocalFileMatchFileModifyTime(uploadInfo.FilePath, recordUploadInfo.LocalFileModifyTime/10000000)
 			// 本地文件没有变化，服务端文件没有变化，则不需要再重新上传
 			if isLocalFileNotChange && isServerFileNotChange {
 				return false, nil
@@ -348,7 +352,7 @@ func batchUploadFlow(info BatchUpload2Info, uploadConfig UploadConfig, dbPath st
 				return true, data.NewEmptyError().AppendDescF("local file has change, %v", mErr)
 			} else {
 				// 服务端文件有变动，尝试检查 hash，hash 统一由单文件上传之前检查
-				return true, data.NewEmptyError().AppendDescF("server file has change, PutTime don't match, except:%d but:%d", result.ServerPutTime, stat.PutTime)
+				return true, data.NewEmptyError().AppendDesc("server file has change, hash don't match")
 			}
 		}).
 		FlowWillStartFunc(func(flow *flow.Flow) (err *data.CodeError) {

iqshell/storage/object/upload/uploader.go

@@ -170,17 +170,6 @@ func Upload(info *ApiInfo) (res *ApiResult, err *data.CodeError) {
 		return
 	}
 
-	if stat, sErr := object.Status(object.StatusApiInfo{
-		Bucket:   info.ToBucket,
-		Key:      info.SaveKey,
-		NeedPart: false,
-	}); sErr != nil {
-		err = data.NewEmptyError().AppendDesc("get file status after upload").AppendDescF("upload error:%v", sErr)
-		return
-	} else {
-		res.ServerPutTime = stat.PutTime
-	}
-
 	if info.CheckHash {
 		if _, mErr := object.Match(object.MatchApiInfo{
 			Bucket:         info.ToBucket,