From 46c6c990240f235d618445b0ae1cc1a3643a3d37 Mon Sep 17 00:00:00 2001 From: XMLHexagram Date: Sun, 16 Jun 2024 18:23:54 +0800 Subject: [PATCH 1/5] feat(upload): support `--include` flag --- app/dl/iter.go | 13 +++---------- app/up/up.go | 3 ++- app/up/walk.go | 29 ++++++++++++++++++++--------- cmd/up.go | 1 + pkg/filterMap/filterMap.go | 9 +++++++++ 5 files changed, 35 insertions(+), 20 deletions(-) create mode 100644 pkg/filterMap/filterMap.go diff --git a/app/dl/iter.go b/app/dl/iter.go index df524d288..c3b66edf6 100644 --- a/app/dl/iter.go +++ b/app/dl/iter.go @@ -6,6 +6,7 @@ import ( "crypto/sha256" "encoding/binary" "fmt" + "github.com/iyear/tdl/pkg/filterMap" "os" "path/filepath" "sort" @@ -74,8 +75,8 @@ func newIter(pool dcpool.Pool, manager *peers.Manager, dialog [][]*tmessage.Dial } // include and exclude - includeMap := filterMap(opts.Include, fsutil.AddPrefixDot) - excludeMap := filterMap(opts.Exclude, fsutil.AddPrefixDot) + includeMap := filterMap.New(opts.Include, fsutil.AddPrefixDot) + excludeMap := filterMap.New(opts.Exclude, fsutil.AddPrefixDot) // to keep fingerprint stable sortDialogs(dialogs, opts.Desc) @@ -287,14 +288,6 @@ func flatDialogs(dialogs [][]*tmessage.Dialog) []*tmessage.Dialog { return res } -func filterMap(data []string, keyFn func(key string) string) map[string]struct{} { - m := make(map[string]struct{}) - for _, v := range data { - m[keyFn(v)] = struct{}{} - } - return m -} - func sortDialogs(dialogs []*tmessage.Dialog, desc bool) { sort.Slice(dialogs, func(i, j int) bool { return tutil.GetInputPeerID(dialogs[i].Peer) < diff --git a/app/up/up.go b/app/up/up.go index 00f2d46f3..2833ec698 100644 --- a/app/up/up.go +++ b/app/up/up.go @@ -25,13 +25,14 @@ import ( type Options struct { Chat string Paths []string + Includes []string Excludes []string Remove bool Photo bool } func Run(ctx context.Context, c *telegram.Client, kvd kv.KV, opts Options) (rerr error) { - files, err := walk(opts.Paths, opts.Excludes) + files, err := walk(opts.Paths, opts.Includes, opts.Excludes) if err != nil { return err } diff --git a/app/up/walk.go b/app/up/walk.go index e7f8a325d..011845d61 100644 --- a/app/up/walk.go +++ b/app/up/walk.go @@ -1,6 +1,7 @@ package up import ( + "github.com/iyear/tdl/pkg/filterMap" "io/fs" "path/filepath" "strings" @@ -9,15 +10,19 @@ import ( "github.com/iyear/tdl/pkg/consts" ) -func walk(paths, excludes []string) ([]*file, error) { +func walk(paths, includes, excludes []string) ([]*file, error) { files := make([]*file, 0) - excludesMap := map[string]struct{}{ - consts.UploadThumbExt: {}, // ignore thumbnail files - } - - for _, exclude := range excludes { - excludesMap[exclude] = struct{}{} - } + //excludesMap := map[string]struct{}{ + // consts.UploadThumbExt: {}, // ignore thumbnail files + //} + // + //for _, exclude := range excludes { + // excludesMap[exclude] = struct{}{} + //} + + includesMap := filterMap.New(includes, fsutil.AddPrefixDot) + excludesMap := filterMap.New(excludes, fsutil.AddPrefixDot) + excludesMap[consts.UploadThumbExt] = struct{}{} // ignore thumbnail files for _, path := range paths { err := filepath.WalkDir(path, func(path string, d fs.DirEntry, err error) error { @@ -27,7 +32,13 @@ func walk(paths, excludes []string) ([]*file, error) { if d.IsDir() { return nil } - if _, ok := excludesMap[filepath.Ext(path)]; ok { + + // process include and exclude + ext := filepath.Ext(path) + if _, ok := includesMap[ext]; len(includesMap) > 0 && !ok { + return nil + } + if _, ok := excludesMap[ext]; len(excludesMap) > 0 && ok { return nil } diff --git a/cmd/up.go b/cmd/up.go index 2e3aa9149..6191adab8 100644 --- a/cmd/up.go +++ b/cmd/up.go @@ -31,6 +31,7 @@ func NewUpload() *cobra.Command { ) cmd.Flags().StringVarP(&opts.Chat, _chat, "c", "", "chat id or domain, and empty means 'Saved Messages'") cmd.Flags().StringSliceVarP(&opts.Paths, path, "p", []string{}, "dirs or files") + cmd.Flags().StringSliceVarP(&opts.Includes, "includes", "i", []string{}, "include the specified file extensions") cmd.Flags().StringSliceVarP(&opts.Excludes, "excludes", "e", []string{}, "exclude the specified file extensions") cmd.Flags().BoolVar(&opts.Remove, "rm", false, "remove the uploaded files after uploading") cmd.Flags().BoolVar(&opts.Photo, "photo", false, "upload the image as a photo instead of a file") diff --git a/pkg/filterMap/filterMap.go b/pkg/filterMap/filterMap.go new file mode 100644 index 000000000..5400c9e6f --- /dev/null +++ b/pkg/filterMap/filterMap.go @@ -0,0 +1,9 @@ +package filterMap + +func New(data []string, keyFn func(key string) string) map[string]struct{} { + m := make(map[string]struct{}) + for _, v := range data { + m[keyFn(v)] = struct{}{} + } + return m +} From cd4a3eff6092512a49f19663bf52e5a996a313bf Mon Sep 17 00:00:00 2001 From: XMLHexagram Date: Sun, 16 Jun 2024 18:42:08 +0800 Subject: [PATCH 2/5] docs(hugo): add include and exclude flag explain to up --- docs/content/zh/guide/download.md | 2 +- docs/content/zh/guide/upload.md | 18 ++++++++++++++++-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/docs/content/zh/guide/download.md b/docs/content/zh/guide/download.md index 818aa9e85..f31be63ff 100644 --- a/docs/content/zh/guide/download.md +++ b/docs/content/zh/guide/download.md @@ -112,7 +112,7 @@ tdl dl -u https://t.me/tdl/1 --takeout {{< hint warning >}} 扩展名仅与文件名匹配，而不是 MIME 类型。因此，这可能不会按预期工作。 -白名单和黑名单不能同时使用。 +白名单和黑名单不能同时使用（如果同时使用则仅有白名单生效）。 {{< /hint >}} 白名单：只下载扩展名为 `.jpg` `.png` 的文件 diff --git a/docs/content/zh/guide/upload.md b/docs/content/zh/guide/upload.md index 60a2720bb..c92ee32a3 100644 --- a/docs/content/zh/guide/upload.md +++ b/docs/content/zh/guide/upload.md @@ -33,10 +33,24 @@ tdl up -p /path/to/file -t 8 -s 524288 -l 4 ## 过滤器 -上传除指定扩展名之外的文件： +使用扩展名过滤器上传文件： + +{{< hint warning >}} +扩展名仅与文件名匹配，而不是 MIME 类型。因此，这可能不会按预期工作。 + +白名单和黑名单不能同时使用（如果同时使用则仅有白名单生效）。 +{{< /hint >}} + +白名单：只上传扩展名为 `.jpg` `.png` 的文件 + +{{< command >}} +tdl up -p /path/to/file -p /path/to/dir -i jpg,png +{{< /command >}} + +黑名单：上传除了扩展名为 `.mp4` `.flv` 的所有文件 {{< command >}} -tdl up -p /path/to/file -p /path/to/dir -e .so -e .tmp +tdl up -p /path/to/file -p /path/to/dir -e mp4 -e flv {{< /command >}} ## 自动删除 From 67701236ee4a519389460b01f0125fda6cbf21c7 Mon Sep 17 00:00:00 2001 From: XMLHexagram Date: Sun, 16 Jun 2024 18:42:55 +0800 Subject: [PATCH 3/5] fix(cmd/up): use const on include and exclude flag name --- cmd/up.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/cmd/up.go b/cmd/up.go index 6191adab8..6a3b3298d 100644 --- a/cmd/up.go +++ b/cmd/up.go @@ -26,13 +26,15 @@ func NewUpload() *cobra.Command { } const ( - _chat = "chat" - path = "path" + _chat = "chat" + path = "path" + include = "include" + exclude = "exclude" ) cmd.Flags().StringVarP(&opts.Chat, _chat, "c", "", "chat id or domain, and empty means 'Saved Messages'") cmd.Flags().StringSliceVarP(&opts.Paths, path, "p", []string{}, "dirs or files") - cmd.Flags().StringSliceVarP(&opts.Includes, "includes", "i", []string{}, "include the specified file extensions") - cmd.Flags().StringSliceVarP(&opts.Excludes, "excludes", "e", []string{}, "exclude the specified file extensions") + cmd.Flags().StringSliceVarP(&opts.Includes, include, "i", []string{}, "include the specified file extensions") + cmd.Flags().StringSliceVarP(&opts.Excludes, exclude, "e", []string{}, "exclude the specified file extensions") cmd.Flags().BoolVar(&opts.Remove, "rm", false, "remove the uploaded files after uploading") cmd.Flags().BoolVar(&opts.Photo, "photo", false, "upload the image as a photo instead of a file") From f43cd2f5114b75f509e103a4ed18b140a4257008 Mon Sep 17 00:00:00 2001 From: XMLHexagram Date: Sun, 16 Jun 2024 19:03:38 +0800 Subject: [PATCH 4/5] docs(hugo): update upload filter english doc --- docs/content/en/guide/download.md | 2 +- docs/content/en/guide/upload.md | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/docs/content/en/guide/download.md b/docs/content/en/guide/download.md index 2b44a8073..06814d560 100644 --- a/docs/content/en/guide/download.md +++ b/docs/content/en/guide/download.md @@ -117,7 +117,7 @@ Download files with extension filters: {{< hint warning >}} The extension is only matched with the file name, not the MIME type. So it may not work as expected. -Whitelist and blacklist can not be used at the same time. +Whitelist and blacklist can not be used at the same time(if you use both whitelist and blacklist, only the whitelist will take effect.). {{< /hint >}} Whitelist: Only download files with `.jpg` `.png` extension diff --git a/docs/content/en/guide/upload.md b/docs/content/en/guide/upload.md index b89ee8476..2b36bbad3 100644 --- a/docs/content/en/guide/upload.md +++ b/docs/content/en/guide/upload.md @@ -39,6 +39,28 @@ Upload files except specified extensions: tdl up -p /path/to/file -p /path/to/dir -e .so -e .tmp {{< /command >}} +## Filters + +Upload files with extension filters: + +{{< hint warning >}} +The extension is only matched with the file name, not the MIME type. So it may not work as expected. + +Whitelist and blacklist can not be used at the same time(if you use both whitelist and blacklist, only the whitelist will take effect.). +{{< /hint >}} + +Whitelist: Only upload files with `.jpg` `.png` extension + +{{< command >}} +tdl up -p /path/to/file -p /path/to/dir -i jpg,png +{{< /command >}} + +Blacklist: Upload all files except `.mp4` `.flv` extension + +{{< command >}} +tdl up -p /path/to/file -p /path/to/dir -e mp4 -e flv +{{< /command >}} + ## Delete Local Delete the uploaded file after uploading successfully: From c2fdae9ab6fc432ad6b14900e93264009700058c Mon Sep 17 00:00:00 2001 From: XMLHexagram Date: Mon, 17 Jun 2024 13:33:49 +0800 Subject: [PATCH 5/5] fix: delete legacy comments --- app/up/walk.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/app/up/walk.go b/app/up/walk.go index 011845d61..3ee7954fd 100644 --- a/app/up/walk.go +++ b/app/up/walk.go @@ -12,13 +12,6 @@ import ( func walk(paths, includes, excludes []string) ([]*file, error) { files := make([]*file, 0) - //excludesMap := map[string]struct{}{ - // consts.UploadThumbExt: {}, // ignore thumbnail files - //} - // - //for _, exclude := range excludes { - // excludesMap[exclude] = struct{}{} - //} includesMap := filterMap.New(includes, fsutil.AddPrefixDot) excludesMap := filterMap.New(excludes, fsutil.AddPrefixDot)