diff --git a/browser/files/localassets.go b/browser/files/localassets.go index 51f2f303..89f72581 100644 --- a/browser/files/localassets.go +++ b/browser/files/localassets.go @@ -95,10 +95,10 @@ func (la *LocalAssetBrowser) handleFolder(ctx context.Context, fsys fs.FS, fileC if e.IsDir() { continue } - fileName := path.Join(folder, e.Name()) - la.log.AddEntry(fileName, logger.DiscoveredFile, "") name := e.Name() + fileName := path.Join(folder, name) ext := strings.ToLower(path.Ext(name)) + la.log.AddEntry(fileName, logger.DiscoveredFile, "") t := la.sm.TypeFromExt(ext) switch t { diff --git a/cmd/upload/e2e_upload_folder_test.go b/cmd/upload/e2e_upload_folder_test.go index 4c2c9462..13103fe5 100644 --- a/cmd/upload/e2e_upload_folder_test.go +++ b/cmd/upload/e2e_upload_folder_test.go @@ -248,6 +248,25 @@ func Test_XMP(t *testing.T) { runCase(t, tc) } +func Test_XMP2(t *testing.T) { + initMyEnv(t) + + tc := testCase{ + name: "Test_XMP2", + args: []string{ + "-create-stacks=false", + "-create-album-folder", + // myEnv["IMMICH_TESTFILES"] + "/xmp/files", + // myEnv["IMMICH_TESTFILES"] + "/xmp/files/*.CR2", + myEnv["IMMICH_TESTFILES"] + "/xmp/files*/*.CR2", + }, + resetImmich: true, + expectError: false, + APITrace: false, + } + runCase(t, tc) +} + func Test_Album_Issue_119(t *testing.T) { initMyEnv(t) @@ -334,6 +353,8 @@ func Test_Issue_129(t *testing.T) { runCase(t, tc) } +// Test_Issue_128 +// Manage GP with no names func Test_Issue_128(t *testing.T) { initMyEnv(t) @@ -350,6 +371,22 @@ func Test_Issue_128(t *testing.T) { runCase(t, tc) } +// Test_GP_MultiZip test the new way to pars arguments (post 0.12.0) +func Test_GP_MultiZip(t *testing.T) { + initMyEnv(t) + + tc := testCase{ + name: "Test_Issue_128", + args: []string{ + "-google-photos", + myEnv["IMMICH_TESTFILES"] + "/google-photos/zip*.zip"}, + resetImmich: true, + expectError: false, + APITrace: false, + } + runCase(t, tc) +} + func Test_ExtensionsFromTheServer(t *testing.T) { initMyEnv(t) @@ -388,6 +425,23 @@ func Test_Issue_173(t *testing.T) { runCase(t, tc) } +// Test_Issue_159: Albums from subdirectories with matching names +func Test_Issue_159(t *testing.T) { + initMyEnv(t) + + tc := testCase{ + name: "Test_Issue_159", + args: []string{ + "-create-album-folder=true", + "TEST_DATA/folder/high/Album*", + }, + resetImmich: true, + expectError: false, + APITrace: false, + } + runCase(t, tc) +} + // ResetImmich // ⛔: will remove the content of the server.‼️ // Give the user of the connection to confirm the server instance: debug@example.com diff --git a/cmd/upload/upload.go b/cmd/upload/upload.go index c70c5ce7..fe0a10b4 100644 --- a/cmd/upload/upload.go +++ b/cmd/upload/upload.go @@ -202,6 +202,9 @@ func UploadCommand(ctx context.Context, common *cmd.SharedFlags, args []string) if err != nil { return err } + defer func() { + _ = fshelper.CloseFSs(app.fsys) + }() return app.Run(ctx, app.fsys) } diff --git a/docs/releases.md b/docs/releases.md index a72b6e75..2cf5dd5a 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -1,8 +1,17 @@ # Release notes +## Release next + +### Improvement: Better handling of wild cards in path +`Immich-go` now accepts to handle path like `photos/Holydays*`. This, combined with the `-create-album-folder` will create +an album per folder Holydays*. + +It can handle patterns like : /photo/\*/raw/\*.dmg + ### fix: Append Log #182 Log are now appended to the log file + ## Release 0.12.0 ### fix: #173 [Feature Request:] Set date from file system timestamp diff --git a/go.mod b/go.mod index b4baaa61..e0283713 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,6 @@ require ( github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd github.com/thlib/go-timezone-local v0.0.0-20210907160436-ef149e42d28e github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31 - github.com/yalue/merged_fs v1.3.0 ) require ( diff --git a/go.sum b/go.sum index 78e4417f..3576163a 100644 --- a/go.sum +++ b/go.sum @@ -40,8 +40,6 @@ github.com/thlib/go-timezone-local v0.0.0-20210907160436-ef149e42d28e h1:Buzhfgf github.com/thlib/go-timezone-local v0.0.0-20210907160436-ef149e42d28e/go.mod h1:/Tnicc6m/lsJE0irFMA0LfIwTBo4QP7A8IfyIv4zZKI= github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31 h1:OXcKh35JaYsGMRzpvFkLv/MEyPuL49CThT1pZ8aSml4= github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31/go.mod h1:onvgF043R+lC5RZ8IT9rBXDaEDnpnw/Cl+HFiw+v/7Q= -github.com/yalue/merged_fs v1.3.0 h1:qCeh9tMPNy/i8cwDsQTJ5bLr6IRxbs6meakNE5O+wyY= -github.com/yalue/merged_fs v1.3.0/go.mod h1:WqqchfVYQyclV2tnR7wtRhBddzBvLVR83Cjw9BKQw0M= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= diff --git a/helpers/fshelper/globwalkfs.go b/helpers/fshelper/globwalkfs.go new file mode 100644 index 00000000..fde573a4 --- /dev/null +++ b/helpers/fshelper/globwalkfs.go @@ -0,0 +1,113 @@ +package fshelper + +import ( + "io/fs" + "path" + "path/filepath" + "strings" +) + +// GlobWalkFS create a FS that limits the WalkDir function to the +// list of files that match the glob expression, and cheats to +// matches *.XMP files in all circumstances +// +// It implements ReadDir and Stat to filter the file list +// + +type GlobWalkFS struct { + rootFS fs.FS + pattern string + parts []string +} + +func NewGlobWalkFS(fsys fs.FS, pattern string) (fs.FS, error) { + pattern = filepath.ToSlash(pattern) + + return &GlobWalkFS{ + rootFS: fsys, + pattern: pattern, + parts: strings.Split(pattern, "/"), + }, nil +} + +// match the current file name with the pattern +// matches files having a path starting by the patten +// +// ex: file /path/to/file matches with the pattern /*/to +func (gw GlobWalkFS) match(name string) (bool, error) { + if name == "." { + return true, nil + } + nParts := strings.Split(name, "/") + for i := 0; i < min(len(gw.parts), len(nParts)); i++ { + match, err := path.Match(gw.parts[i], nParts[i]) + if !match || err != nil { + return match, err + } + } + return true, nil +} + +// Open the name only if the name matches with the pattern +func (gw GlobWalkFS) Open(name string) (fs.File, error) { + return gw.rootFS.Open(name) +} + +// Stat the name only if the name matches with the pattern +func (gw GlobWalkFS) Stat(name string) (fs.FileInfo, error) { + return fs.Stat(gw.rootFS, name) +} + +// ReadDir return all DirEntries that match with the pattern or .XMP files +func (gw GlobWalkFS) ReadDir(name string) ([]fs.DirEntry, error) { + match, err := gw.match(name) + if err != nil { + return nil, err + } + if !match { + return nil, fs.ErrNotExist + } + entries, err := fs.ReadDir(gw.rootFS, name) + if err != nil { + return nil, err + } + + returned := []fs.DirEntry{} + for _, e := range entries { + p := path.Join(name, e.Name()) + + // Always matches .XMP files... + if !e.IsDir() { + ext := strings.ToUpper(path.Ext(e.Name())) + if ext == ".XMP" { + returned = append(returned, e) + continue + } + } + match, _ = gw.match(p) + if match { + returned = append(returned, e) + } + } + return returned, nil +} + +// FixedPathAndMagic split the path with the fixed part and the variable part +func FixedPathAndMagic(name string) (string, string) { + if !HasMagic(name) { + return name, "" + } + name = filepath.ToSlash(name) + parts := strings.Split(name, "/") + p := 0 + for p = range parts { + if HasMagic(parts[p]) { + break + } + } + fixed := "" + if name[0] == '/' { + fixed = "/" + } + return fixed + path.Join(parts[:p]...), path.Join(parts[p:]...) +} diff --git a/helpers/fshelper/globwalkfs_test.go b/helpers/fshelper/globwalkfs_test.go new file mode 100644 index 00000000..d59f2ec5 --- /dev/null +++ b/helpers/fshelper/globwalkfs_test.go @@ -0,0 +1,132 @@ +package fshelper + +import ( + "io/fs" + "os" + "reflect" + "testing" +) + +func Test_GlobWalkFS(t *testing.T) { + tc := []struct { + pattern string + expected []string + }{ + { + pattern: "A/T/10.jpg", + expected: []string{ + "A/T/10.jpg", + }, + }, + { + pattern: "A/T/*.*", + expected: []string{ + "A/T/10.jpg", + "A/T/10.json", + }, + }, + { + pattern: "A/T/*.jpg", + expected: []string{ + "A/T/10.jpg", + }, + }, + { + pattern: "*/T/*.*", + expected: []string{ + "A/T/10.jpg", + "A/T/10.json", + "B/T/20.jpg", + "B/T/20.json", + }, + }, + { + pattern: "*/T/*.jpg", + expected: []string{ + "A/T/10.jpg", + "B/T/20.jpg", + }, + }, + { + pattern: "*/*.jpg", + expected: []string{ + "A/1.jpg", + "A/2.jpg", + "B/4.jpg", + }, + }, + { + pattern: "A", + expected: []string{ + "A/1.jpg", + "A/1.json", + "A/2.jpg", + "A/2.json", + "A/T/10.jpg", + "A/T/10.json", + }, + }, + } + + for _, c := range tc { + t.Run(c.pattern, func(t *testing.T) { + fsys, err := NewGlobWalkFS(os.DirFS("TESTDATA"), c.pattern) + if err != nil { + t.Error(err) + return + } + + files := []string{} + + err = fs.WalkDir(fsys, ".", func(p string, d fs.DirEntry, err error) error { + if p == "." || d.IsDir() { + return nil + } + files = append(files, p) + return nil + }) + if err != nil { + t.Error(err) + return + } + if !reflect.DeepEqual(c.expected, files) { + t.Error("Result differs") + } + }) + } +} + +func TestFixedPathAndMagic(t *testing.T) { + tests := []struct { + name string + want string + want1 string + }{ + { + name: "A/B/C/file", + want: "A/B/C/file", + want1: "", + }, + { + name: "A/B/C/*.*", + want: "A/B/C", + want1: "*.*", + }, + { + name: "A/*/C/file", + want: "A", + want1: "*/C/file", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, got1 := FixedPathAndMagic(tt.name) + if got != tt.want { + t.Errorf("FixedPathAndMagic() got = %v, want %v", got, tt.want) + } + if got1 != tt.want1 { + t.Errorf("FixedPathAndMagic() got1 = %v, want %v", got1, tt.want1) + } + }) + } +} diff --git a/helpers/fshelper/multifs.go b/helpers/fshelper/multifs.go deleted file mode 100644 index 8d17608c..00000000 --- a/helpers/fshelper/multifs.go +++ /dev/null @@ -1,21 +0,0 @@ -package fshelper - -import ( - "archive/zip" - "io/fs" - - "github.com/yalue/merged_fs" -) - -func multiZip(names ...string) (fs.FS, error) { - fss := []fs.FS{} - - for _, p := range names { - fsys, err := zip.OpenReader(p) - if err != nil { - return nil, err - } - fss = append(fss, fsys) - } - return merged_fs.MergeMultiple(fss...), nil -} diff --git a/helpers/fshelper/parseArgs.go b/helpers/fshelper/parseArgs.go index 24d8ad35..305e0c8e 100644 --- a/helpers/fshelper/parseArgs.go +++ b/helpers/fshelper/parseArgs.go @@ -1,6 +1,7 @@ package fshelper import ( + "archive/zip" "errors" "fmt" "io/fs" @@ -8,111 +9,88 @@ import ( "path" "path/filepath" "strings" - - "github.com/simulot/immich-go/helpers/gen" ) -type argParser struct { - googlePhotos bool - files []string - paths map[string][]string - zips []string - unsupported map[string]any - err error -} +// ParsePath return a list of FS bases on args +// +// Zip files are opened and returned as FS +// Manage wildcards in path +// +// TODO: Implement a tgz reader for non google-photos archives func ParsePath(args []string, googlePhoto bool) ([]fs.FS, error) { - p := argParser{ - googlePhotos: googlePhoto, - unsupported: map[string]any{}, - paths: map[string][]string{}, - } + var errs error + fsyss := []fs.FS{} - for _, f := range args { - if !HasMagic(f) { - p.handleFile(f) - continue - } else { - globs, err := filepath.Glob(f) + for _, a := range args { + a = filepath.ToSlash(a) + lowA := strings.ToLower(a) + switch { + case strings.HasSuffix(lowA, ".tgz") || strings.HasSuffix(lowA, ".tar.gz"): + errs = errors.Join(fmt.Errorf("immich-go cant use tgz archives: %s", filepath.Base(a))) + case strings.HasSuffix(a, ".zip"): + files, err := expandNames(a) if err != nil { - p.err = errors.Join(err) - continue + errs = errors.Join(errs, err) + break } - if len(globs) == 0 { - p.err = errors.Join(fmt.Errorf("no file matches '%s'", f)) - continue - } - - for _, g := range globs { - if p.googlePhotos && strings.ToLower(path.Ext(g)) != ".zip" { - return nil, fmt.Errorf("wildcard '%s' not allowed with the google-photos options", filepath.Base(f)) + for _, f := range files { + fsys, err := zip.OpenReader(f) + if err != nil { + errs = errors.Join(errs, err) + continue } - p.handleFile(g) + fsyss = append(fsyss, fsys) } - } - } - - fsys := []fs.FS{} - - for _, f := range p.files { - d, b := filepath.Split(f) - d = filepath.Clean(d) - p.paths[d] = append(p.paths[d], b) - } - - for pa, l := range p.paths { - if len(l) > 0 { - f, err := newPathFS(pa, l) - if err != nil { - p.err = errors.Join(err) + default: + fixed, magic := FixedPathAndMagic(a) + if magic == "" { + stat, err := os.Stat(a) + if err != nil { + errs = errors.Join(errs, err) + continue + } + if stat.IsDir() { + fsyss = append(fsyss, os.DirFS(fixed)) + } else { + d, f := path.Split(a) + fsys, err := NewGlobWalkFS(os.DirFS(strings.TrimSuffix(d, "/")), f) + if err != nil { + errs = errors.Join(errs, err) + continue + } + fsyss = append(fsyss, fsys) + } } else { - fsys = append(fsys, f) + fsys, err := NewGlobWalkFS(os.DirFS(fixed), magic) + if err != nil { + errs = errors.Join(errs, err) + continue + } + fsyss = append(fsyss, fsys) } - } else { - fsys = append(fsys, os.DirFS(pa)) - } - } - - if len(p.zips) > 0 { - f, err := multiZip(p.zips...) - if err != nil { - p.err = errors.Join(err) - } else { - fsys = append(fsys, f) } } - if len(p.unsupported) > 0 { - keys := gen.MapKeys(p.unsupported) - for _, k := range keys { - p.err = errors.Join(fmt.Errorf("files with extension '%s' are not supported. Check the discussion here https://github.com/simulot/immich-go/discussions/109", k)) - } + if errs != nil { + return nil, errs } - return fsys, p.err + return fsyss, nil } -func (p *argParser) handleFile(f string) { - i, err := os.Stat(f) - if err != nil { - p.err = errors.Join(err) - return +func expandNames(name string) ([]string, error) { + if HasMagic(name) { + return filepath.Glob(name) } - if i.IsDir() { - if _, exists := p.paths[f]; !exists { - p.paths[f] = nil + return []string{name}, nil +} + +// CloseFSs closes each FS that provides a Close() error interface +func CloseFSs(fsyss []fs.FS) error { + var errs error + for _, fsys := range fsyss { + if closer, ok := fsys.(interface{ Close() error }); ok { + errs = errors.Join(errs, closer.Close()) } - return - } - ext := strings.ToLower(filepath.Ext(f)) - if ext == ".zip" { - p.zips = append(p.zips, f) - return - } - if ext == ".tgz" { - p.unsupported[ext] = nil - return - } - if p.googlePhotos { - return } - p.files = append(p.files, f) + return errs } diff --git a/helpers/fshelper/pathfs.go b/helpers/fshelper/pathfs.go deleted file mode 100644 index 1cbf7c8e..00000000 --- a/helpers/fshelper/pathfs.go +++ /dev/null @@ -1,67 +0,0 @@ -package fshelper - -import ( - "io/fs" - "os" - "path" - "path/filepath" - "slices" - "strings" - - "github.com/simulot/immich-go/helpers/gen" -) - -type pathFS struct { - dir string - files []string -} - -func newPathFS(dir string, files []string) (*pathFS, error) { - _, err := os.Stat(dir) - if err != nil { - return nil, err - } - return &pathFS{ - dir: dir, - files: files, - }, nil -} - -func (fsys pathFS) listed(name string) bool { - if len(fsys.files) > 0 { - ext := path.Ext(name) - if ext == strings.ToLower(".xmp") { - name = strings.TrimSuffix(name, ext) - } - return slices.Contains(fsys.files, name) - } - return true -} - -func (fsys pathFS) Open(name string) (fs.File, error) { - if !fsys.listed(name) { - return nil, fs.ErrNotExist - } - return os.Open(filepath.Join(fsys.dir, name)) -} - -func (fsys pathFS) Stat(name string) (fs.FileInfo, error) { - if name == "." { - return os.Stat(fsys.dir) - } - if !fsys.listed(name) { - return nil, fs.ErrNotExist - } - return os.Stat(filepath.Join(fsys.dir, name)) -} - -func (fsys pathFS) ReadDir(name string) ([]fs.DirEntry, error) { - d, err := os.ReadDir(filepath.Join(fsys.dir, name)) - - if err == nil && len(fsys.files) > 0 { - d = gen.Filter(d, func(i fs.DirEntry) bool { - return fsys.listed(i.Name()) - }) - } - return d, err -}