diff --git a/.env.example b/.env.example index 95a8c046..7b65ffb2 100644 --- a/.env.example +++ b/.env.example @@ -1,6 +1,7 @@ DATABASE_URL=postgresql://postgres:secret@postgres:5432/pico?sslmode=disable POSTGRES_PASSWORD=secret CF_API_TOKEN=secret +PICO_SECRET=secret REGISTRY_URL=registry:5000 PICO_SECRET="" PICO_SECRET_WEBHOOK="" diff --git a/db/postgres/storage.go b/db/postgres/storage.go index f2077d11..1484aa4e 100644 --- a/db/postgres/storage.go +++ b/db/postgres/storage.go @@ -1553,7 +1553,7 @@ func (me *PsqlDB) FindFeedItemsByPostID(postID string) ([]*db.FeedItem, error) { func (me *PsqlDB) InsertProject(userID, name, projectDir string) (string, error) { if !shared.IsValidSubdomain(name) { - return "", fmt.Errorf("(%s) is not a valid project name, must match /^[a-z0-9-]+$/", name) + return "", fmt.Errorf("'%s' is not a valid project name, must match /^[a-z0-9-]+$/", name) } var id string diff --git a/feeds/cron.go b/feeds/cron.go index 94a0bc13..05eb457e 100644 --- a/feeds/cron.go +++ b/feeds/cron.go @@ -218,10 +218,6 @@ func (f *Fetcher) ParseURL(fp *gofeed.Parser, url string) (*gofeed.Feed, error) return nil, fmt.Errorf("fetching feed resulted in an error: %s %s", resp.Status, body) } - if err != nil { - return nil, err - } - feed, err := fp.ParseString(string(body)) if err != nil { diff --git a/filehandlers/assets/handler.go b/filehandlers/assets/handler.go index a9a5ba4f..39253a4e 100644 --- a/filehandlers/assets/handler.go +++ b/filehandlers/assets/handler.go @@ -5,9 +5,12 @@ import ( "encoding/binary" "fmt" "io" + "io/fs" "log/slog" "os" + "path" "path/filepath" + "slices" "strings" "time" @@ -277,20 +280,16 @@ func (h *UploadAssetHandler) Write(s ssh.Session, entry *utils.FileEntry) (strin h.Cfg.Logger.Error("user not found in ctx", "err", err.Error()) return "", err } + + if entry.Mode.IsDir() && strings.Count(entry.Filepath, "/") == 1 { + entry.Filepath = strings.TrimPrefix(entry.Filepath, "/") + } + logger := h.GetLogger().With( "user", user.Name, "file", entry.Filepath, ) - var origText []byte - if b, err := io.ReadAll(entry.Reader); err == nil { - origText = b - } - fileSize := binary.Size(origText) - // TODO: hack for now until I figure out how to get correct - // filesize from sftp,scp,rsync - entry.Size = int64(fileSize) - bucket, err := getBucket(s) if err != nil { logger.Error("could not find bucket in ctx", "err", err.Error()) @@ -325,11 +324,31 @@ func (h *UploadAssetHandler) Write(s ssh.Session, entry *utils.FileEntry) (strin setProject(s, project) } + if entry.Mode.IsDir() { + _, err := h.Storage.PutObject( + bucket, + path.Join(shared.GetAssetFileName(entry), "._pico_keep_dir"), + utils.NopReaderAtCloser(bytes.NewReader([]byte{})), + entry, + ) + return "", err + } + + var origText []byte + if b, err := io.ReadAll(entry.Reader); err == nil { + origText = b + } + fileSize := binary.Size(origText) + // TODO: hack for now until I figure out how to get correct + // filesize from sftp,scp,rsync + entry.Size = int64(fileSize) + storageSize := getStorageSize(s) featureFlag, err := futil.GetFeatureFlag(s) if err != nil { return "", err } + // calculate the filsize difference between the same file already // stored and the updated file being uploaded assetFilename := shared.GetAssetFileName(entry) @@ -389,6 +408,66 @@ func (h *UploadAssetHandler) Write(s ssh.Session, entry *utils.FileEntry) (strin return str, nil } +func (h *UploadAssetHandler) Delete(s ssh.Session, entry *utils.FileEntry) error { + user, err := futil.GetUser(s) + if err != nil { + h.Cfg.Logger.Error("user not found in ctx", "err", err.Error()) + return err + } + + if entry.Mode.IsDir() && strings.Count(entry.Filepath, "/") == 1 { + entry.Filepath = strings.TrimPrefix(entry.Filepath, "/") + } + + assetFilepath := shared.GetAssetFileName(entry) + + logger := h.GetLogger().With( + "user", user.Name, + "file", assetFilepath, + ) + + bucket, err := getBucket(s) + if err != nil { + logger.Error("could not find bucket in ctx", "err", err.Error()) + return err + } + + projectName := shared.GetProjectName(entry) + logger = logger.With("project", projectName) + + if assetFilepath == filepath.Join("/", projectName, "._pico_keep_dir") { + return os.ErrPermission + } + + logger.Info("deleting file") + + pathDir := filepath.Dir(assetFilepath) + fileName := filepath.Base(assetFilepath) + + sibs, err := h.Storage.ListObjects(bucket, pathDir+"/", false) + if err != nil { + return err + } + + sibs = slices.DeleteFunc(sibs, func(sib fs.FileInfo) bool { + return sib.Name() == fileName + }) + + if len(sibs) == 0 { + _, err := h.Storage.PutObject( + bucket, + filepath.Join(pathDir, "._pico_keep_dir"), + utils.NopReaderAtCloser(bytes.NewReader([]byte{})), + entry, + ) + if err != nil { + return err + } + } + + return h.Storage.DeleteObject(bucket, assetFilepath) +} + func (h *UploadAssetHandler) validateAsset(data *FileData) (bool, error) { storageMax := data.FeatureFlag.Data.StorageMax var nextStorageSize uint64 @@ -447,30 +526,23 @@ func (h *UploadAssetHandler) validateAsset(data *FileData) (bool, error) { func (h *UploadAssetHandler) writeAsset(data *FileData) error { assetFilepath := shared.GetAssetFileName(data.FileEntry) - if data.Size == 0 { - err := h.Storage.DeleteObject(data.Bucket, assetFilepath) - if err != nil { - return err - } - } else { - reader := bytes.NewReader(data.Text) + reader := bytes.NewReader(data.Text) - h.Cfg.Logger.Info( - "uploading file to bucket", - "user", data.User.Name, - "bucket", data.Bucket.Name, - "filename", assetFilepath, - ) + h.Cfg.Logger.Info( + "uploading file to bucket", + "user", data.User.Name, + "bucket", data.Bucket.Name, + "filename", assetFilepath, + ) - _, err := h.Storage.PutObject( - data.Bucket, - assetFilepath, - utils.NopReaderAtCloser(reader), - data.FileEntry, - ) - if err != nil { - return err - } + _, err := h.Storage.PutObject( + data.Bucket, + assetFilepath, + utils.NopReaderAtCloser(reader), + data.FileEntry, + ) + if err != nil { + return err } return nil diff --git a/filehandlers/imgs/handler.go b/filehandlers/imgs/handler.go index 969b6961..06d4e200 100644 --- a/filehandlers/imgs/handler.go +++ b/filehandlers/imgs/handler.go @@ -47,23 +47,6 @@ func NewUploadImgHandler(dbpool db.DB, cfg *shared.ConfigSite, storage storage.S } } -func (h *UploadImgHandler) removePost(data *PostMetaData) error { - // skip empty files from being added to db - if data.Post == nil { - h.Cfg.Logger.Info("file is empty, skipping record", "filename", data.Filename) - return nil - } - - h.Cfg.Logger.Info("file is empty, removing record", "filename", data.Filename, "recordId", data.Cur.ID) - err := h.DBPool.RemovePosts([]string{data.Cur.ID}) - if err != nil { - h.Cfg.Logger.Error(err.Error(), "filename", data.Filename) - return fmt.Errorf("error for %s: %v", data.Filename, err) - } - - return nil -} - func (h *UploadImgHandler) Read(s ssh.Session, entry *utils.FileEntry) (os.FileInfo, utils.ReaderAtCloser, error) { user, err := util.GetUser(s) if err != nil { @@ -205,3 +188,47 @@ func (h *UploadImgHandler) Write(s ssh.Session, entry *utils.FileEntry) (string, ) return str, nil } + +func (h *UploadImgHandler) Delete(s ssh.Session, entry *utils.FileEntry) error { + user, err := util.GetUser(s) + if err != nil { + return err + } + + filename := filepath.Base(entry.Filepath) + + logger := h.Cfg.Logger.With( + "user", user.Name, + "filename", filename, + ) + + post, err := h.DBPool.FindPostWithFilename( + filename, + user.ID, + Space, + ) + if err != nil { + logger.Info("unable to find image, continuing", "err", err.Error()) + return err + } + + err = h.DBPool.RemovePosts([]string{post.ID}) + if err != nil { + logger.Error("error removing image", "error", err) + return fmt.Errorf("error for %s: %v", filename, err) + } + + bucket, err := h.Storage.UpsertBucket(user.ID) + if err != nil { + return err + } + + err = h.Storage.DeleteObject(bucket, filename) + if err != nil { + return err + } + + logger.Info("deleting image") + + return nil +} diff --git a/filehandlers/imgs/img.go b/filehandlers/imgs/img.go index 912f6b65..10767659 100644 --- a/filehandlers/imgs/img.go +++ b/filehandlers/imgs/img.go @@ -91,27 +91,18 @@ func (h *UploadImgHandler) writeImg(s ssh.Session, data *PostMetaData) error { return err } - modTime := time.Unix(data.Mtime, 0) + modTime := time.Now() + + if data.Mtime > 0 { + modTime = time.Unix(data.Mtime, 0) + } + logger := h.Cfg.Logger.With( "user", data.Username, "filename", data.Filename, ) - if len(data.OrigText) == 0 { - err = h.removePost(data) - if err != nil { - return err - } - - bucket, err := h.Storage.UpsertBucket(data.User.ID) - if err != nil { - return err - } - err = h.Storage.DeleteObject(bucket, data.Filename) - if err != nil { - return err - } - } else if data.Cur == nil { + if data.Cur == nil { logger.Info("file not found, adding record") insertPost := db.Post{ UserID: user.ID, diff --git a/filehandlers/post_handler.go b/filehandlers/post_handler.go index 3ae95a2e..9e82c54d 100644 --- a/filehandlers/post_handler.go +++ b/filehandlers/post_handler.go @@ -89,6 +89,10 @@ func (h *ScpUploadHandler) Write(s ssh.Session, entry *utils.FileEntry) (string, "filename", filename, ) + if entry.Mode.IsDir() { + return "", fmt.Errorf("file entry is directory, but only files are supported: %s", filename) + } + var origText []byte if b, err := io.ReadAll(entry.Reader); err == nil { origText = b @@ -144,23 +148,14 @@ func (h *ScpUploadHandler) Write(s ssh.Session, entry *utils.FileEntry) (string, return "", err } - modTime := time.Unix(entry.Mtime, 0) + modTime := time.Now() - // if the file is empty we remove it from our database - if len(origText) == 0 { - // skip empty files from being added to db - if post == nil { - logger.Info("file is empty, skipping record") - return "", nil - } + if entry.Mtime > 0 { + modTime = time.Unix(entry.Mtime, 0) + } - err := h.DBPool.RemovePosts([]string{post.ID}) - logger.Info("file is empty, removing record") - if err != nil { - logger.Error(err.Error()) - return "", fmt.Errorf("error for %s: %v", filename, err) - } - } else if post == nil { + // if the file is empty we remove it from our database + if post == nil { logger.Info("file not found, adding record") insertPost := db.Post{ UserID: userID, @@ -264,3 +259,36 @@ func (h *ScpUploadHandler) Write(s ssh.Session, entry *utils.FileEntry) (string, curl := shared.NewCreateURL(h.Cfg) return h.Cfg.FullPostURL(curl, user.Name, metadata.Slug), nil } + +func (h *ScpUploadHandler) Delete(s ssh.Session, entry *utils.FileEntry) error { + logger := h.Cfg.Logger + user, err := util.GetUser(s) + if err != nil { + logger.Error(err.Error()) + return err + } + + userID := user.ID + filename := filepath.Base(entry.Filepath) + logger = logger.With( + "user", user.Name, + "filename", filename, + ) + + post, err := h.DBPool.FindPostWithFilename(filename, userID, h.Cfg.Space) + if err != nil { + return err + } + + if post == nil { + return os.ErrNotExist + } + + err = h.DBPool.RemovePosts([]string{post.ID}) + logger.Info("removing record") + if err != nil { + logger.Error(err.Error()) + return fmt.Errorf("error for %s: %v", filename, err) + } + return nil +} diff --git a/filehandlers/router_handler.go b/filehandlers/router_handler.go index ecfe93b1..dd14e48c 100644 --- a/filehandlers/router_handler.go +++ b/filehandlers/router_handler.go @@ -1,6 +1,8 @@ package filehandlers import ( + "database/sql" + "errors" "fmt" "log/slog" "os" @@ -16,6 +18,7 @@ import ( type ReadWriteHandler interface { Write(ssh.Session, *utils.FileEntry) (string, error) Read(ssh.Session, *utils.FileEntry) (os.FileInfo, utils.ReaderAtCloser, error) + Delete(ssh.Session, *utils.FileEntry) error } type FileHandlerRouter struct { @@ -51,6 +54,10 @@ func (r *FileHandlerRouter) findHandler(entry *utils.FileEntry) (ReadWriteHandle } func (r *FileHandlerRouter) Write(s ssh.Session, entry *utils.FileEntry) (string, error) { + if entry.Mode.IsDir() { + return "", os.ErrInvalid + } + handler, err := r.findHandler(entry) if err != nil { return "", err @@ -58,6 +65,14 @@ func (r *FileHandlerRouter) Write(s ssh.Session, entry *utils.FileEntry) (string return handler.Write(s, entry) } +func (r *FileHandlerRouter) Delete(s ssh.Session, entry *utils.FileEntry) error { + handler, err := r.findHandler(entry) + if err != nil { + return err + } + return handler.Delete(s, entry) +} + func (r *FileHandlerRouter) Read(s ssh.Session, entry *utils.FileEntry) (os.FileInfo, utils.ReaderAtCloser, error) { handler, err := r.findHandler(entry) if err != nil { @@ -98,6 +113,7 @@ func BaseList(s ssh.Session, fpath string, isDir bool, recursive bool, spaces [] } } else { for _, space := range spaces { + p, e := dbpool.FindPostWithFilename(cleanFilename, user.ID, space) if e != nil { err = e @@ -109,11 +125,15 @@ func BaseList(s ssh.Session, fpath string, isDir bool, recursive bool, spaces [] posts = append(posts, post) } - if err != nil { + if err != nil && !errors.Is(err, sql.ErrNoRows) { return nil, err } for _, post := range posts { + if post == nil { + continue + } + fileList = append(fileList, &utils.VirtualFile{ FName: post.Filename, FIsDir: false, diff --git a/go.mod b/go.mod index a4017722..959de3f4 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,13 @@ module github.com/picosh/pico -go 1.21.9 +go 1.22 + +// replace github.com/picosh/ptun => ../ptun + +// replace github.com/picosh/send => ../send + +// replace github.com/picosh/pobj => ../pobj -// replace github.com/picosh/ptun => /home/erock/dev/pico/ptun replace git.sr.ht/~delthas/senpai => github.com/picosh/senpai v0.0.0-20240503200611-af89e73973b0 replace github.com/gdamore/tcell/v2 => github.com/delthas/tcell/v2 v2.4.1-0.20230710100648-1489e78d90fb @@ -26,9 +31,9 @@ require ( github.com/mmcdole/gofeed v1.3.0 github.com/muesli/reflow v0.3.0 github.com/neurosnap/go-exif-remove v0.0.0-20221010134343-50d1e3c35577 - github.com/picosh/pobj v0.0.0-20240417140600-2071618b61c5 - github.com/picosh/ptun v0.0.0-20240417140706-811cc2b70d9a - github.com/picosh/send v0.0.0-20240217194807-77b972121e63 + github.com/picosh/pobj v0.0.0-20240529200402-7b5398cf8a9f + github.com/picosh/ptun v0.0.0-20240529133708-fcf1376b935e + github.com/picosh/send v0.0.0-20240529200640-3667d1ad154e github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 github.com/sendgrid/sendgrid-go v3.14.0+incompatible github.com/simplesurance/go-ip-anonymizer v0.0.0-20200429124537-35a880f8e87d diff --git a/go.sum b/go.sum index 24d96d34..2e520e73 100644 --- a/go.sum +++ b/go.sum @@ -213,12 +213,12 @@ github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/philhofer/fwd v1.1.2 h1:bnDivRJ1EWPjUIRXV5KfORO897HTbpFAQddBdE8t7Gw= github.com/philhofer/fwd v1.1.2/go.mod h1:qkPdfjR2SIEbspLqpe1tO4n5yICnr2DY7mqEx2tUTP0= -github.com/picosh/pobj v0.0.0-20240417140600-2071618b61c5 h1:iS9zagScak8DCVMCXX5Rvdk7OOQzWUwMPiomrbHCks8= -github.com/picosh/pobj v0.0.0-20240417140600-2071618b61c5/go.mod h1:ctpMgMVUAspcK6kjgpEX5WWHGhDw04l6lDd8s4cI3Uo= -github.com/picosh/ptun v0.0.0-20240417140706-811cc2b70d9a h1:sBqfT6KBIYllVaw4bT1MQtFvuPNINdCgol5yttbuGsg= -github.com/picosh/ptun v0.0.0-20240417140706-811cc2b70d9a/go.mod h1:WXCrhe0l9VL3ji0pdhvSJD6LLx99rJoAA/+PUQXf0Mo= -github.com/picosh/send v0.0.0-20240217194807-77b972121e63 h1:VSSbAejFzj2KBThfVnMcNXQwzHmwjPUridgi29LxihU= -github.com/picosh/send v0.0.0-20240217194807-77b972121e63/go.mod h1:1JCq0NVOdTDenQ0/Kd8e4rP80lu06UHJJ+6dQxhcpew= +github.com/picosh/pobj v0.0.0-20240529200402-7b5398cf8a9f h1:9Y0xaTqq/7JiW7iUX4jSgJkN81X2gQucVqBPVXJDux0= +github.com/picosh/pobj v0.0.0-20240529200402-7b5398cf8a9f/go.mod h1:CViLWaCp2KP/zJdd7miNPkJb6i0v9HOgZ2wdbQuxCrQ= +github.com/picosh/ptun v0.0.0-20240529133708-fcf1376b935e h1:Um9aCUg1ysiUaB0nh3400UHlFAnhd8BXBsawqePxxqQ= +github.com/picosh/ptun v0.0.0-20240529133708-fcf1376b935e/go.mod h1:WXCrhe0l9VL3ji0pdhvSJD6LLx99rJoAA/+PUQXf0Mo= +github.com/picosh/send v0.0.0-20240529200640-3667d1ad154e h1:2NMuieR/7GIjiGYNPQsh6KOJiz2WhzU5ispxQCXmOyU= +github.com/picosh/send v0.0.0-20240529200640-3667d1ad154e/go.mod h1:V418obz9YdzjS3/oFzyDFzmPDnLu1nvy3wkLaixiT84= github.com/picosh/senpai v0.0.0-20240503200611-af89e73973b0 h1:pBRIbiCj7K6rGELijb//dYhyCo8A3fvxW5dijrJVtjs= github.com/picosh/senpai v0.0.0-20240503200611-af89e73973b0/go.mod h1:QaBDtybFC5gz7EG/9c3bgzuyW7W5W2rYLFZxWNuWk3Q= github.com/pkg/sftp v1.13.6 h1:JFZT4XbOU7l77xGSpOdW+pwIMqP044IyjXX6FGyEKFo= diff --git a/pico/file_handler.go b/pico/file_handler.go index 5ba7e8a5..7ec9e777 100644 --- a/pico/file_handler.go +++ b/pico/file_handler.go @@ -2,6 +2,7 @@ package pico import ( "bytes" + "errors" "fmt" "io" "log/slog" @@ -50,6 +51,10 @@ func (h *UploadHandler) getAuthorizedKeyFile(user *db.User) (*utils.VirtualFile, return fileInfo, text, nil } +func (h *UploadHandler) Delete(s ssh.Session, entry *utils.FileEntry) error { + return errors.New("unsupported") +} + func (h *UploadHandler) Read(s ssh.Session, entry *utils.FileEntry) (os.FileInfo, utils.ReaderAtCloser, error) { user, err := util.GetUser(s) if err != nil { diff --git a/shared/bucket.go b/shared/bucket.go index 6d039003..1257b551 100644 --- a/shared/bucket.go +++ b/shared/bucket.go @@ -18,6 +18,10 @@ func GetAssetBucketName(userID string) string { } func GetProjectName(entry *utils.FileEntry) string { + if entry.Mode.IsDir() && strings.Count(entry.Filepath, string(os.PathSeparator)) == 0 { + return entry.Filepath + } + dir := filepath.Dir(entry.Filepath) list := strings.Split(dir, string(os.PathSeparator)) return list[1]