Merge pull request #356 from YangSen-qn/list_bucket_optimize

List bucket optimize

Author: bachue

.gitignore

@@ -8,6 +8,8 @@
 *.dylib
 src/test/
 src/.qshell
+marker.go
+marker_test.go
 
 # Folders
 _obj
@@ -29,7 +31,6 @@ _cgo_export.*
 [568vq].out
 report.xml
 .qshell
-qshell_*
 qshell-*
 _testmain.go
 

CHANGELOG.md

@@ -1,3 +1,10 @@
+# 2.9.1
+1. 支持通过 $? 获取命令执行状态
+2. get / qdownload 下载大文件支持切片下载
+3. 优化批量上传
+4. batch 并发量自适应
+5. listbucket2 支持 v1、支持断点继续列举、支持结果滚动输出
+
 # 2.9.0
 1. 新增 batchforbidden 命令，支持批量修改文件启用/禁用状态，具体参考命令使用说明
 2. qdownload 命令新增支持自定义文件本地存储路径， 具体参考命令使用说明

cmd/asyncfetch.go

@@ -19,6 +19,7 @@ func asyncFetchCmdBuilder(cfg *iqshell.Config) *cobra.Command {
 			cfg.CmdCfg.CmdId = docs.ABFetch
 			info.BatchInfo.ItemSeparate = "\t" // 此处用户不可定义
 			info.BatchInfo.EnableStdin = true
+			info.BatchInfo.OperationCountPerRequest = 1
 			if len(args) > 0 {
 				info.Bucket = args[0]
 			}

cmd/bucket.go

@@ -110,6 +110,9 @@ var listBucketCmd2Builder = func(cfg *iqshell.Config) *cobra.Command {
 	cmd.Flags().StringVarP(&info.Suffixes, "suffixes", "q", "", "list by key suffixes, separated by comma")
 	cmd.Flags().IntVarP(&info.MaxRetry, "max-retry", "x", -1, "max retries when error occurred")
 	cmd.Flags().StringVarP(&info.SaveToFile, "out", "o", "", "output file")
+	cmd.Flags().IntVarP(&info.OutputLimit, "limit", "", -1, "max count of items to output")
+	cmd.Flags().Int64VarP(&info.OutputFileMaxLines, "output-file-max-lines", "", 0, "maximum number of lines per output file.")
+	cmd.Flags().Int64VarP(&info.OutputFileMaxSize, "output-file-max-size", "", 0, "maximum size of each output file")
 	cmd.Flags().StringVarP(&info.StartDate, "start", "s", "", "start date with format yyyy-mm-dd-hh-MM-ss")
 	cmd.Flags().StringVarP(&info.EndDate, "end", "e", "", "end date with format yyyy-mm-dd-hh-MM-ss")
 
@@ -120,10 +123,12 @@ var listBucketCmd2Builder = func(cfg *iqshell.Config) *cobra.Command {
 
 	cmd.Flags().BoolVarP(&info.AppendMode, "append", "a", false, "result append to file instead of overwriting")
 	cmd.Flags().BoolVarP(&info.Readable, "readable", "r", false, "present file size with human readable format")
-	cmd.Flags().IntVarP(&info.Limit, "limit", "", -1, "max count of items to output")
+	cmd.Flags().StringVarP(&info.ApiVersion, "api-version", "", "v2", "list api version, one of v1 and v2.")
+	cmd.Flags().IntVarP(&info.ApiLimit, "api-limit", "", 1000, "one enumeration will make multiple requests, and the maximum number of items returned for each request; currently only v1 is supported; in the range 1-1000.")
+	cmd.Flags().BoolVarP(&info.EnableRecord, "enable-record", "", false, "record the execution status of the listbucket2 command. When the listbucket2 command is executed next time, the marker will be automatically filled and the listbucket2 will continue. Enabling this option will automatically enable append (see the --append option for details). The id of the record is related to the bucket where the file is located, the prefix listed, and the path where the file is saved.")
 
-	cmd.Flags().StringVarP(&info.ShowFields, "show-fields", "", "", "The file attributes to be displayed on each line, separated by commas. Optional range: Key, Hash, FileSize, PutTime, MimeType, StorageType, EndUser.")
 	cmd.Flags().StringVarP(&info.OutputFieldsSep, "output-fields-sep", "", data.DefaultLineSeparate, "Each line needs to display the delimiter of the file information.")
+	cmd.Flags().StringVarP(&info.ShowFields, "show-fields", "", "", "The file attributes to be displayed on each line, separated by commas. Optional range: Key, Hash, FileSize, PutTime, MimeType, StorageType, EndUser.")
 
 	return cmd
 }

cmd/root.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+	"github.com/qiniu/qshell/v2/docs"
 	"github.com/qiniu/qshell/v2/iqshell"
 	"github.com/qiniu/qshell/v2/iqshell/common/config"
 	"github.com/qiniu/qshell/v2/iqshell/common/data"
@@ -47,8 +48,13 @@ var rootCmdBuilder = func(cfg *iqshell.Config) *cobra.Command {
 		Short:                  "Qiniu commandline tool for managing your bucket and CDN",
 		Version:                version.Version(),
 		BashCompletionFunction: bash_completion_func,
+		Run: func(cmd *cobra.Command, args []string) {
+			cfg.CmdCfg.CmdId = docs.QShellType
+			docs.ShowCmdDocument(docs.QShellType)
+		},
 	}
 	cmd.PersistentFlags().BoolVarP(&cfg.StdoutColorful, "colorful", "", false, "console colorful mode")
+	cmd.PersistentFlags().BoolVarP(&cfg.Silence, "silence", "", false, "silence mode, The console only outputs warnings、errors and some important information")
 	cmd.PersistentFlags().BoolVarP(&cfg.DebugEnable, "debug", "d", false, "debug mode")
 	// ddebug 开启 client debug
 	cmd.PersistentFlags().BoolVarP(&cfg.DDebugEnable, "ddebug", "D", false, "deep debug mode")

cmd/rsbatch.go

@@ -23,7 +23,9 @@ var batchStatCmdBuilder = func(cfg *iqshell.Config) *cobra.Command {
 		},
 	}
 	setBatchCmdInputFileFlags(cmd, &info.BatchInfo)
-	setBatchCmdWorkCountFlags(cmd, &info.BatchInfo)
+	setBatchCmdWorkerCountFlags(cmd, &info.BatchInfo)
+	setBatchCmdMinWorkerCountFlags(cmd, &info.BatchInfo)
+	setBatchCmdWorkerCountIncreasePeriodFlags(cmd, &info.BatchInfo)
 	setBatchCmdSuccessExportFileFlags(cmd, &info.BatchInfo)
 	setBatchCmdFailExportFileFlags(cmd, &info.BatchInfo)
 	setBatchCmdForceFlags(cmd, &info.BatchInfo)
@@ -47,7 +49,9 @@ var batchForbiddenCmdBuilder = func(cfg *iqshell.Config) *cobra.Command {
 		},
 	}
 	setBatchCmdInputFileFlags(cmd, &info.BatchInfo)
-	setBatchCmdWorkCountFlags(cmd, &info.BatchInfo)
+	setBatchCmdWorkerCountFlags(cmd, &info.BatchInfo)
+	setBatchCmdMinWorkerCountFlags(cmd, &info.BatchInfo)
+	setBatchCmdWorkerCountIncreasePeriodFlags(cmd, &info.BatchInfo)
 	setBatchCmdSuccessExportFileFlags(cmd, &info.BatchInfo)
 	setBatchCmdFailExportFileFlags(cmd, &info.BatchInfo)
 	setBatchCmdForceFlags(cmd, &info.BatchInfo)
@@ -265,7 +269,9 @@ var batchFetchCmdBuilder = func(cfg *iqshell.Config) *cobra.Command {
 
 func setBatchCmdDefaultFlags(cmd *cobra.Command, info *batch.Info) {
 	setBatchCmdInputFileFlags(cmd, info)
-	setBatchCmdWorkCountFlags(cmd, info)
+	setBatchCmdWorkerCountFlags(cmd, info)
+	setBatchCmdMinWorkerCountFlags(cmd, info)
+	setBatchCmdWorkerCountIncreasePeriodFlags(cmd, info)
 	setBatchCmdEnableRecordFlags(cmd, info)
 	setBatchCmdRecordRedoWhileErrorFlags(cmd, info)
 	setBatchCmdSuccessExportFileFlags(cmd, info)
@@ -279,8 +285,14 @@ func setBatchCmdInputFileFlags(cmd *cobra.Command, info *batch.Info) {
 func setBatchCmdForceFlags(cmd *cobra.Command, info *batch.Info) {
 	cmd.Flags().BoolVarP(&info.Force, "force", "y", false, "force mode, default false")
 }
-func setBatchCmdWorkCountFlags(cmd *cobra.Command, info *batch.Info) {
-	cmd.Flags().IntVarP(&info.WorkerCount, "worker", "c", 1, "worker count. 1 means the number of objects in one operation is 1000 and if configured as 3 , the number of objects in one operation is 3000. This value needs to be consistent with the upper limit of Qiniu’s operation, otherwise unexpected errors will occur. Under normal circumstances you do not need to adjust this value and if you need please carefully.")
+func setBatchCmdWorkerCountFlags(cmd *cobra.Command, info *batch.Info) {
+	cmd.Flags().IntVarP(&info.WorkerCount, "worker", "c", 4, "worker count. 1 means the number of objects in one operation is 250 and if configured as 10 , the number of objects in one operation is 2500. This value needs to be consistent with the upper limit of Qiniu’s operation, otherwise unexpected errors will occur. Under normal circumstances you do not need to adjust this value and if you need please carefully.")
+}
+func setBatchCmdMinWorkerCountFlags(cmd *cobra.Command, info *batch.Info) {
+	cmd.Flags().IntVarP(&info.MinWorkerCount, "min-worker", "", 1, "min worker count. 1 means the number of objects in one operation is 1000 and if configured as 3 , the number of objects in one operation is 3000. for more, please refer to worker")
+}
+func setBatchCmdWorkerCountIncreasePeriodFlags(cmd *cobra.Command, info *batch.Info) {
+	cmd.Flags().IntVarP(&info.WorkerCountIncreasePeriod, "worker-count-increase-period", "", 60, "worker count increase period. when the worker count is too big, an overrun error will be triggered. In order to alleviate this problem, qshell will automatically reduce the worker count. In order to complete the operation as quickly as possible, qshell will periodically increase the worker count. unit: second")
 }
 func setBatchCmdItemSeparateFlags(cmd *cobra.Command, info *batch.Info) {
 	cmd.Flags().StringVarP(&info.ItemSeparate, "sep", "F", "\t", "Separator used for split line fields, default is \\t (tab)")

cmd_test/change_minetype_test.go

@@ -90,6 +90,8 @@ func TestBatchChangeMimeType(t *testing.T) {
 		"--success-list", successLogPath,
 		"--failure-list", failLogPath,
 		"--worker", "4",
+		"--min-worker", "10",
+		"--worker-count-increase-period", "50",
 		"-y")
 	defer func() {
 		test.RemoveFile(successLogPath)

cmd_test/change_type_test.go

@@ -170,6 +170,8 @@ func TestBatchChangeTypeRecord(t *testing.T) {
 		"--enable-record",
 		"--record-redo-while-error",
 		"--worker", "4",
+		"--min-worker", "10",
+		"--worker-count-increase-period", "50",
 		"-y",
 		"-d")
 	if !strings.Contains(result, "because have done and success") {

cmd_test/copy_test.go

@@ -94,6 +94,8 @@ func TestBatchCopy(t *testing.T) {
 		"--failure-list", failLogPath,
 		"--overwrite",
 		"--worker", "4",
+		"--min-worker", "10",
+		"--worker-count-increase-period", "50",
 		"-w",
 		"-y")
 	defer func() {

cmd_test/delete_test.go

@@ -83,6 +83,8 @@ func TestBatchDelete(t *testing.T) {
 		"--success-list", successLogPath,
 		"--failure-list", failLogPath,
 		"--worker", "4",
+		"--min-worker", "10",
+		"--worker-count-increase-period", "50",
 		"-y")
 	defer func() {
 		test.RemoveFile(successLogPath)

cmd_test/fetch_test.go

@@ -142,7 +142,6 @@ func TestBatchFetchWithRecord(t *testing.T) {
 		"--failure-list", failLogPath,
 		"--enable-record",
 		"--worker", "4",
-		"--worker", "1",
 		"-y",
 		"-d")
 	if !strings.Contains(result, "Skip") {

cmd_test/forbidden_test.go

@@ -150,6 +150,8 @@ func TestBatchForbiddenRecord(t *testing.T) {
 		"--enable-record",
 		"--record-redo-while-error",
 		"--worker", "4",
+		"--min-worker", "10",
+		"--worker-count-increase-period", "50",
 		"-y",
 		"-d")
 	if !strings.Contains(result, "because have done and success") {

cmd_test/move_test.go

@@ -98,6 +98,8 @@ func TestBatchMove(t *testing.T) {
 		"--success-list", successLogPath,
 		"--failure-list", failLogPath,
 		"--worker", "4",
+		"--min-worker", "10",
+		"--worker-count-increase-period", "50",
 		"-y",
 		"-w")
 	defer func() {

cmd_test/qshell_test.go

@@ -0,0 +1,23 @@
+package cmd
+
+import (
+	"github.com/qiniu/qshell/v2/cmd_test/test"
+	"strings"
+	"testing"
+)
+
+func TestQShellDocument01(t *testing.T) {
+	prefix := "# 简介\n`qshell`"
+	result, _ := test.RunCmdWithError()
+	if !strings.HasPrefix(result, prefix) {
+		t.Fatal("document test fail for cmd: qshell")
+	}
+}
+
+func TestQShellDocument02(t *testing.T) {
+	prefix := "# 简介\n`qshell`"
+	result, _ := test.RunCmdWithError("--doc")
+	if !strings.HasPrefix(result, prefix) {
+		t.Fatal("document test fail for cmd: qshell")
+	}
+}

cmd_test/rename_test.go

@@ -151,6 +151,8 @@ func TestBatchRenameWithRecord(t *testing.T) {
 		"--enable-record",
 		"--record-redo-while-error",
 		"--worker", "4",
+		"--min-worker", "10",
+		"--worker-count-increase-period", "50",
 		"-y",
 		"-w",
 		"-d")

cmd_test/restore_archive_test.go

@@ -152,6 +152,8 @@ func TestBatchRestoreArchiveRecord(t *testing.T) {
 		"--enable-record",
 		"--record-redo-while-error",
 		"--worker", "4",
+		"--min-worker", "10",
+		"--worker-count-increase-period", "50",
 		"-y",
 		"-d")
 	if !strings.Contains(result, "because have done and success") {

cmd_test/status_test.go

@@ -82,6 +82,8 @@ func TestBatchStatus(t *testing.T) {
 		"--success-list", successLogPath,
 		"--failure-list", failLogPath,
 		"--worker", "4",
+		"--min-worker", "10",
+		"--worker-count-increase-period", "50",
 		"-y")
 	defer func() {
 		test.RemoveFile(successLogPath)

docs/batchchgm.md

@@ -24,7 +24,9 @@ qshell batchchgm -h
 - -s/--success-list：该选项指定一个文件，qshell 会把操作成功的文件行导入到该文件；默认不导出。【可选】
 - -e/--failure-list：该选项指定一个文件，qshell 会把操作失败的文件行加上错误状态码，错误的原因导入该文件；默认不导出。【可选】
 - -F/--sep：该选项可以自定义每行输入内容中字段之间的分隔符（文件输入或标准输入，参考 -i 选项说明）；默认为 tab 制表符。【可选】
-- -c/--worker：该选项可以定义 Batch 任务并发数；1 路并发单次操作对象数为 1000 ，如果配置为 2 并发，则 2 路并发单次操作对象数为 2000，此值需要和七牛对您的操作上限相吻合，否则会出现非预期错误，正常情况不需要调节此值，如果需要请谨慎调节；默认为 1。【可选】
+- -c/--worker：该选项可以定义 Batch 任务并发数；1 路并发单次操作对象数为 250 ，如果配置为 10 并发，则 10 路并发单次操作对象数为 2500，此值需要和七牛对您的操作上限相吻合，否则会出现非预期错误，正常情况不需要调节此值，如果需要请谨慎调节；默认为 4。【可选】
+- --min-worker：最小 Batch 任务并发数；当并发设置过高时，会触发超限错误，为了缓解此问题，qshell 会自动减小并发度，此值为减小的最低值。默认：1【可选】
+- --worker-count-increase-period：为了尽可能快的完成操作 qshell 会周期性尝试增加并发度，此值为尝试增加并发数的周期，单位：秒，最小 10，默认 60。【可选】
 - --enable-record：记录任务执行状态，当下次执行命令时会检测任务执行的状态并跳过已执行的任务。 【可选】
 - --record-redo-while-error：依赖于 --enable-record，当检测任务状态时（命令重新执行时，所有任务会从头到尾重新执行；任务执行前会先检测当前任务是否已经执行），如果任务已执行且失败，则再执行一次；默认此选项不生效，当任务执行失败不重新执行。 【可选】
 

docs/batchchtype.md

@@ -26,7 +26,9 @@ qshell batchchtype -h
 - -s/--success-list：该选项指定一个文件，qshell 会把操作成功的文件行导入到该文件；默认不导出。【可选】
 - -e/--failure-list：该选项指定一个文件， qshell 会把操作失败的文件行加上错误状态码，错误的原因导入该文件；默认不导出。【可选】
 - -F/--sep：该选项可以自定义每行输入内容中字段之间的分隔符（文件输入或标准输入，参考 -i 选项说明）；默认为 tab 制表符。【可选】
-- -c/--worker：该选项可以定义 Batch 任务并发数；1 路并发单次操作对象数为 1000 ，如果配置为 2 并发，则 2 路并发单次操作对象数为 2000，此值需要和七牛对您的操作上限相吻合，否则会出现非预期错误，正常情况不需要调节此值，如果需要请谨慎调节；默认为 1。【可选】
+- -c/--worker：该选项可以定义 Batch 任务并发数；1 路并发单次操作对象数为 250 ，如果配置为 10 并发，则 10 路并发单次操作对象数为 2500，此值需要和七牛对您的操作上限相吻合，否则会出现非预期错误，正常情况不需要调节此值，如果需要请谨慎调节；默认为 4。【可选】
+- --min-worker：最小 Batch 任务并发数；当并发设置过高时，会触发超限错误，为了缓解此问题，qshell 会自动减小并发度，此值为减小的最低值。默认：1【可选】
+- --worker-count-increase-period：为了尽可能快的完成操作 qshell 会周期性尝试增加并发度，此值为尝试增加并发数的周期，单位：秒，最小 10，默认 60。【可选】
 - --enable-record：记录任务执行状态，当下次执行命令时会检测任务执行的状态并跳过已执行的任务。 【可选】
 - --record-redo-while-error：依赖于 --enable-record，当检测任务状态时（命令重新执行时，所有任务会从头到尾重新执行；任务执行前会先检测当前任务是否已经执行），如果任务已执行且失败，则再执行一次；默认此选项不生效，当任务执行失败不重新执行。 【可选】
 

docs/batchcopy.md

@@ -34,7 +34,9 @@ qshell batchcopy -h
 - -s/--success-list：该选项指定一个文件，qshell 会把操作成功的文件行导入到该文件；默认不导出。【可选】
 - -e/--failure-list：该选项指定一个文件， qshell 会把操作失败的文件行加上错误状态码，错误的原因导入该文件；默认不导出。【可选】
 - -F/--sep：该选项可以自定义每行输入内容中字段之间的分隔符（文件输入或标准输入，参考 -i 选项说明）；默认为 tab 制表符。【可选】】
-- -c/--worker：该选项可以定义 Batch 任务并发数；1 路并发单次操作对象数为 1000 ，如果配置为 2 并发，则 2 路并发单次操作对象数为 2000，此值需要和七牛对您的操作上限相吻合，否则会出现非预期错误，正常情况不需要调节此值，如果需要请谨慎调节；默认为 1。【可选】
+- -c/--worker：该选项可以定义 Batch 任务并发数；1 路并发单次操作对象数为 250 ，如果配置为 10 并发，则 10 路并发单次操作对象数为 2500，此值需要和七牛对您的操作上限相吻合，否则会出现非预期错误，正常情况不需要调节此值，如果需要请谨慎调节；默认为 4。【可选】
+- --min-worker：最小 Batch 任务并发数；当并发设置过高时，会触发超限错误，为了缓解此问题，qshell 会自动减小并发度，此值为减小的最低值。默认：1【可选】
+- --worker-count-increase-period：为了尽可能快的完成操作 qshell 会周期性尝试增加并发度，此值为尝试增加并发数的周期，单位：秒，最小 10，默认 60。【可选】
 - --overwrite：默认情况下，如果批量重命名的文件列表中存在目标空间已有同名文件的情况，针对该文件的重命名会失败，如果希望能够强制覆盖目标文件，那么可以使用 `--overwrite` 选项。【可选】
 - --enable-record：记录任务执行状态，当下次执行命令时会检测任务执行的状态并跳过已执行的任务。 【可选】
 - --record-redo-while-error：依赖于 --enable-record，当检测任务状态时（命令重新执行时，所有任务会从头到尾重新执行；任务执行前会先检测当前任务是否已经执行），如果任务已执行且失败，则再执行一次；默认此选项不生效，当任务执行失败不重新执行。 【可选】