Better handling of wild cards in pathSimulot/issue159

browser/files/localassets.go

@@ -95,10 +95,10 @@ func (la *LocalAssetBrowser) handleFolder(ctx context.Context, fsys fs.FS, fileC
 		if e.IsDir() {
 			continue
 		}
-		fileName := path.Join(folder, e.Name())
-		la.log.AddEntry(fileName, logger.DiscoveredFile, "")
 		name := e.Name()
+		fileName := path.Join(folder, name)
 		ext := strings.ToLower(path.Ext(name))
+		la.log.AddEntry(fileName, logger.DiscoveredFile, "")
 
 		t := la.sm.TypeFromExt(ext)
 		switch t {

cmd/upload/e2e_upload_folder_test.go

@@ -248,6 +248,25 @@ func Test_XMP(t *testing.T) {
 	runCase(t, tc)
 }
 
+func Test_XMP2(t *testing.T) {
+	initMyEnv(t)
+
+	tc := testCase{
+		name: "Test_XMP2",
+		args: []string{
+			"-create-stacks=false",
+			"-create-album-folder",
+			// myEnv["IMMICH_TESTFILES"] + "/xmp/files",
+			// myEnv["IMMICH_TESTFILES"] + "/xmp/files/*.CR2",
+			myEnv["IMMICH_TESTFILES"] + "/xmp/files*/*.CR2",
+		},
+		resetImmich: true,
+		expectError: false,
+		APITrace:    false,
+	}
+	runCase(t, tc)
+}
+
 func Test_Album_Issue_119(t *testing.T) {
 	initMyEnv(t)
 
@@ -334,6 +353,8 @@ func Test_Issue_129(t *testing.T) {
 	runCase(t, tc)
 }
 
+// Test_Issue_128
+// Manage GP with no names
 func Test_Issue_128(t *testing.T) {
 	initMyEnv(t)
 
@@ -350,6 +371,22 @@ func Test_Issue_128(t *testing.T) {
 	runCase(t, tc)
 }
 
+// Test_GP_MultiZip test the new way to pars arguments (post 0.12.0)
+func Test_GP_MultiZip(t *testing.T) {
+	initMyEnv(t)
+
+	tc := testCase{
+		name: "Test_Issue_128",
+		args: []string{
+			"-google-photos",
+			myEnv["IMMICH_TESTFILES"] + "/google-photos/zip*.zip"},
+		resetImmich: true,
+		expectError: false,
+		APITrace:    false,
+	}
+	runCase(t, tc)
+}
+
 func Test_ExtensionsFromTheServer(t *testing.T) {
 	initMyEnv(t)
 
@@ -388,6 +425,23 @@ func Test_Issue_173(t *testing.T) {
 	runCase(t, tc)
 }
 
+// Test_Issue_159: Albums from subdirectories with matching names
+func Test_Issue_159(t *testing.T) {
+	initMyEnv(t)
+
+	tc := testCase{
+		name: "Test_Issue_159",
+		args: []string{
+			"-create-album-folder=true",
+			"TEST_DATA/folder/high/Album*",
+		},
+		resetImmich: true,
+		expectError: false,
+		APITrace:    false,
+	}
+	runCase(t, tc)
+}
+
 // ResetImmich
 // ⛔: will remove the content of the server.‼️
 // Give the user of the connection to confirm the server instance: [email protected]

cmd/upload/upload.go

@@ -202,6 +202,9 @@ func UploadCommand(ctx context.Context, common *cmd.SharedFlags, args []string)
 	if err != nil {
 		return err
 	}
+	defer func() {
+		_ = fshelper.CloseFSs(app.fsys)
+	}()
 	return app.Run(ctx, app.fsys)
 }
 

docs/releases.md

@@ -1,8 +1,17 @@
 # Release notes 
 
+## Release next
+
+### Improvement: Better handling of wild cards in path 
+`Immich-go` now accepts to handle path like `photos/Holydays*`. This, combined with the `-create-album-folder` will create 
+an album per folder Holydays*.
+
+It can handle patterns like : /photo/\*/raw/\*.dmg
+
 ### fix: Append Log #182
 Log are now appended to the log file
 
+
 ## Release 0.12.0
 
 ### fix: #173 [Feature Request:] Set date from file system timestamp

go.mod

@@ -11,7 +11,6 @@ require (
 	github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd
 	github.com/thlib/go-timezone-local v0.0.0-20210907160436-ef149e42d28e
 	github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31
-	github.com/yalue/merged_fs v1.3.0
 )
 
 require (

go.sum

@@ -40,8 +40,6 @@ github.com/thlib/go-timezone-local v0.0.0-20210907160436-ef149e42d28e h1:Buzhfgf
 github.com/thlib/go-timezone-local v0.0.0-20210907160436-ef149e42d28e/go.mod h1:/Tnicc6m/lsJE0irFMA0LfIwTBo4QP7A8IfyIv4zZKI=
 github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31 h1:OXcKh35JaYsGMRzpvFkLv/MEyPuL49CThT1pZ8aSml4=
 github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31/go.mod h1:onvgF043R+lC5RZ8IT9rBXDaEDnpnw/Cl+HFiw+v/7Q=
-github.com/yalue/merged_fs v1.3.0 h1:qCeh9tMPNy/i8cwDsQTJ5bLr6IRxbs6meakNE5O+wyY=
-github.com/yalue/merged_fs v1.3.0/go.mod h1:WqqchfVYQyclV2tnR7wtRhBddzBvLVR83Cjw9BKQw0M=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=

helpers/fshelper/globwalkfs.go

@@ -0,0 +1,113 @@
+package fshelper
+
+import (
+	"io/fs"
+	"path"
+	"path/filepath"
+	"strings"
+)
+
+//  GlobWalkFS create a FS that limits the WalkDir function to the
+//  list of files that match the glob expression, and cheats to
+//  matches *.XMP files in all circumstances
+//
+//  It implements ReadDir and Stat to filter the file list
+//
+
+type GlobWalkFS struct {
+	rootFS  fs.FS
+	pattern string
+	parts   []string
+}
+
+func NewGlobWalkFS(fsys fs.FS, pattern string) (fs.FS, error) {
+	pattern = filepath.ToSlash(pattern)
+
+	return &GlobWalkFS{
+		rootFS:  fsys,
+		pattern: pattern,
+		parts:   strings.Split(pattern, "/"),
+	}, nil
+}
+
+// match the current file name with the pattern
+// matches files having a path starting by the patten
+//
+//	ex:  file /path/to/file matches with the pattern /*/to
+func (gw GlobWalkFS) match(name string) (bool, error) {
+	if name == "." {
+		return true, nil
+	}
+	nParts := strings.Split(name, "/")
+	for i := 0; i < min(len(gw.parts), len(nParts)); i++ {
+		match, err := path.Match(gw.parts[i], nParts[i])
+		if !match || err != nil {
+			return match, err
+		}
+	}
+	return true, nil
+}
+
+// Open the name only if the name matches with the pattern
+func (gw GlobWalkFS) Open(name string) (fs.File, error) {
+	return gw.rootFS.Open(name)
+}
+
+// Stat the name only if the name matches with the pattern
+func (gw GlobWalkFS) Stat(name string) (fs.FileInfo, error) {
+	return fs.Stat(gw.rootFS, name)
+}
+
+// ReadDir return all DirEntries that match with the pattern or .XMP files
+func (gw GlobWalkFS) ReadDir(name string) ([]fs.DirEntry, error) {
+	match, err := gw.match(name)
+	if err != nil {
+		return nil, err
+	}
+	if !match {
+		return nil, fs.ErrNotExist
+	}
+	entries, err := fs.ReadDir(gw.rootFS, name)
+	if err != nil {
+		return nil, err
+	}
+
+	returned := []fs.DirEntry{}
+	for _, e := range entries {
+		p := path.Join(name, e.Name())
+
+		// Always matches .XMP files...
+		if !e.IsDir() {
+			ext := strings.ToUpper(path.Ext(e.Name()))
+			if ext == ".XMP" {
+				returned = append(returned, e)
+				continue
+			}
+		}
+		match, _ = gw.match(p)
+		if match {
+			returned = append(returned, e)
+		}
+	}
+	return returned, nil
+}
+
+// FixedPathAndMagic split the path with the fixed part and the variable part
+func FixedPathAndMagic(name string) (string, string) {
+	if !HasMagic(name) {
+		return name, ""
+	}
+	name = filepath.ToSlash(name)
+	parts := strings.Split(name, "/")
+	p := 0
+	for p = range parts {
+		if HasMagic(parts[p]) {
+			break
+		}
+	}
+	fixed := ""
+	if name[0] == '/' {
+		fixed = "/"
+	}
+	return fixed + path.Join(parts[:p]...), path.Join(parts[p:]...)
+}

helpers/fshelper/globwalkfs_test.go

@@ -0,0 +1,132 @@
+package fshelper
+
+import (
+	"io/fs"
+	"os"
+	"reflect"
+	"testing"
+)
+
+func Test_GlobWalkFS(t *testing.T) {
+	tc := []struct {
+		pattern  string
+		expected []string
+	}{
+		{
+			pattern: "A/T/10.jpg",
+			expected: []string{
+				"A/T/10.jpg",
+			},
+		},
+		{
+			pattern: "A/T/*.*",
+			expected: []string{
+				"A/T/10.jpg",
+				"A/T/10.json",
+			},
+		},
+		{
+			pattern: "A/T/*.jpg",
+			expected: []string{
+				"A/T/10.jpg",
+			},
+		},
+		{
+			pattern: "*/T/*.*",
+			expected: []string{
+				"A/T/10.jpg",
+				"A/T/10.json",
+				"B/T/20.jpg",
+				"B/T/20.json",
+			},
+		},
+		{
+			pattern: "*/T/*.jpg",
+			expected: []string{
+				"A/T/10.jpg",
+				"B/T/20.jpg",
+			},
+		},
+		{
+			pattern: "*/*.jpg",
+			expected: []string{
+				"A/1.jpg",
+				"A/2.jpg",
+				"B/4.jpg",
+			},
+		},
+		{
+			pattern: "A",
+			expected: []string{
+				"A/1.jpg",
+				"A/1.json",
+				"A/2.jpg",
+				"A/2.json",
+				"A/T/10.jpg",
+				"A/T/10.json",
+			},
+		},
+	}
+
+	for _, c := range tc {
+		t.Run(c.pattern, func(t *testing.T) {
+			fsys, err := NewGlobWalkFS(os.DirFS("TESTDATA"), c.pattern)
+			if err != nil {
+				t.Error(err)
+				return
+			}
+
+			files := []string{}
+
+			err = fs.WalkDir(fsys, ".", func(p string, d fs.DirEntry, err error) error {
+				if p == "." || d.IsDir() {
+					return nil
+				}
+				files = append(files, p)
+				return nil
+			})
+			if err != nil {
+				t.Error(err)
+				return
+			}
+			if !reflect.DeepEqual(c.expected, files) {
+				t.Error("Result differs")
+			}
+		})
+	}
+}
+
+func TestFixedPathAndMagic(t *testing.T) {
+	tests := []struct {
+		name  string
+		want  string
+		want1 string
+	}{
+		{
+			name:  "A/B/C/file",
+			want:  "A/B/C/file",
+			want1: "",
+		},
+		{
+			name:  "A/B/C/*.*",
+			want:  "A/B/C",
+			want1: "*.*",
+		},
+		{
+			name:  "A/*/C/file",
+			want:  "A",
+			want1: "*/C/file",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, got1 := FixedPathAndMagic(tt.name)
+			if got != tt.want {
+				t.Errorf("FixedPathAndMagic() got = %v, want %v", got, tt.want)
+			}
+			if got1 != tt.want1 {
+				t.Errorf("FixedPathAndMagic() got1 = %v, want %v", got1, tt.want1)
+			}
+		})
+	}
+}