diff --git a/Dockerfile b/Dockerfile index 8592e1c..63b1c37 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,19 +1,26 @@ FROM golang:1.22-alpine -RUN apk add make gcc libc-dev mailcap +RUN apk add dlib dlib-dev --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing/ +RUN apk add blas blas-dev cblas lapack lapack-dev libjpeg-turbo-dev cmake make gcc libc-dev g++ unzip libx11-dev pkgconf jpeg jpeg-dev libpng libpng-dev mailcap COPY go.mod /go/src/circled-server/ COPY go.sum /go/src/circled-server/ WORKDIR /go/src/circled-server/ RUN go mod download +RUN go build github.com/Kagami/go-face RUN CGO_CFLAGS="-D_LARGEFILE64_SOURCE" go build github.com/mattn/go-sqlite3 COPY . /go/src/circled-server RUN CGO_ENABLED=1 CGO_CFLAGS="-D_LARGEFILE64_SOURCE" GOOS=linux go build -a -installsuffix cgo -o circled-server . # Final output image FROM alpine:3.20.1 -RUN apk --no-cache add ca-certificates exiftool tzdata ffmpeg +RUN apk add dlib --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing/ +RUN apk --no-cache add ca-certificates exiftool tzdata blas cblas lapack libjpeg-turbo libstdc++ libgcc ffmpeg WORKDIR /opt/circled COPY --from=0 /etc/mime.types /etc/mime.types COPY --from=0 /go/src/circled-server/circled-server . COPY --from=0 /go/src/circled-server/templates ./templates +# Use 68 landmarks model instead of 5 landmarks model +ADD https://github.com/ageitgey/face_recognition_models/raw/master/face_recognition_models/models/shape_predictor_68_face_landmarks.dat ./models/shape_predictor_5_face_landmarks.dat +ADD https://github.com/ageitgey/face_recognition_models/raw/master/face_recognition_models/models/dlib_face_recognition_resnet_model_v1.dat ./models/ +ADD https://github.com/ageitgey/face_recognition_models/raw/master/face_recognition_models/models/mmod_human_face_detector.dat ./models/ ENTRYPOINT ["./circled-server"] \ No newline at end of file diff --git a/README.md b/README.md index 1c9f32d..07d1d75 100644 --- a/README.md +++ b/README.md @@ -31,11 +31,12 @@ ___ - Supports either locally mounted disks or - S3-compatible Services - this allows different users to use their own S3 bucket on the same server - Push notifications for new Album photos, etc +- Face detection and tagging - Albums - Adding local server contributors and viewers - Sharing albums with anyone with a "secret" link - Chat with push notifications -- Filtering photos by year, month, location, etc +- Filtering photos by tagged person, year, month, location, etc - Moments - automatically grouping photos by time and location - Reverse geocoding for all assets - Automatic video conversion to web-compatible H.264 format @@ -61,6 +62,9 @@ Current configuration environment variables: - `DEBUG_MODE` - currently defaults to `yes` - `DEFAULT_BUCKET_DIR` - a directory that will be used as default bucket if no other buckets exist (i.e. the first time you run the server) - `DEFAULT_ASSET_PATH_PATTERN` - the default path pattern to create subdirectories and file names based on asset info. Defaults to `//` +- `PUSH_SERVER` - the push server URL. Defaults to `https://push.circled.me` +- `FACE_DETECT_CNN` - use Convolutional Neural Network for face detection (as opposed to HOG). Much slower, but more accurate at different angles. Defaults to `no` +- `FACE_MAX_DISTANCE_SQ` - squared distance between faces to consider them similar. Defaults to `0.11` ## docker-compose example ```yaml diff --git a/config/config.go b/config/config.go index 1d568db..2ca0992 100644 --- a/config/config.go +++ b/config/config.go @@ -2,6 +2,7 @@ package config import ( "os" + "strconv" "strings" ) @@ -15,6 +16,8 @@ var ( TMP_DIR = "/tmp" // Used for temporary video conversion, etc (in case of S3 bucket) DEFAULT_BUCKET_DIR = "" // Used for creating initial bucket DEBUG_MODE = true + FACE_DETECT_CNN = false // Use Convolutional Neural Network for face detection (as opposed to HOG). Much slower, supposedly more accurate at different angles + FACE_MAX_DISTANCE_SQ = 0.11 // Squared distance between faces to consider them similar ) func init() { @@ -27,6 +30,8 @@ func init() { readEnvString("DEFAULT_BUCKET_DIR", &DEFAULT_BUCKET_DIR) readEnvString("DEFAULT_ASSET_PATH_PATTERN", &DEFAULT_ASSET_PATH_PATTERN) readEnvBool("DEBUG_MODE", &DEBUG_MODE) + readEnvBool("FACE_DETECT_CNN", &FACE_DETECT_CNN) + readEnvFloat("FACE_MAX_DISTANCE_SQ", &FACE_MAX_DISTANCE_SQ) } func readEnvString(name string, value *string) { @@ -45,3 +50,15 @@ func readEnvBool(name string, value *bool) { *value = false } } + +func readEnvFloat(name string, value *float64) { + v := os.Getenv(name) + if v == "" { + return + } + f, err := strconv.ParseFloat(v, 64) + if err != nil { + return + } + *value = f +} diff --git a/db/db.go b/db/db.go index 4c3687e..550d13f 100644 --- a/db/db.go +++ b/db/db.go @@ -30,14 +30,10 @@ func Init() { CreatedDateFunc = "date(from_unixtime(created_at))" } else if config.SQLITE_FILE != "" { // Sqlite setup - db, err = gorm.Open(sqlite.Open(config.SQLITE_FILE), &gorm.Config{}) + db, err = gorm.Open(sqlite.Open(config.SQLITE_FILE+"?_foreign_keys=on"), &gorm.Config{}) if err != nil || db == nil { log.Fatalf("SQLite DB error: %v", err) } - db.Exec("PRAGMA foreign_keys = ON") - // if sqliteDB, err := db.DB(); err == nil && sqliteDB != nil { - // sqliteDB.SetMaxOpenConns(1) - // } TimestampFunc = "strftime('%s', 'now')" CreatedDateFunc = "date(created_at, 'unixepoch')" } else { diff --git a/faces/faces.go b/faces/faces.go new file mode 100644 index 0000000..a44700b --- /dev/null +++ b/faces/faces.go @@ -0,0 +1,35 @@ +package faces + +import ( + "log" + "path/filepath" + "server/config" + + "github.com/Kagami/go-face" +) + +var ( + modelsDir = filepath.Join(".", "models") + rec *face.Recognizer +) + +func init() { + log.Println("Loading face recognition models...") + // Init the recognizer. + var err error + rec, err = face.NewRecognizer(modelsDir) + if err != nil { + log.Fatalf("Can't init face recognizer: %v", err) + } +} + +func Detect(imgPath string) ([]face.Face, error) { + log.Printf("Detecting faces in %s", imgPath) + // Recognize faces on that image. + if !config.FACE_DETECT_CNN { + // HOG (Histogram of Oriented Gradients) based detection + return rec.RecognizeFile(imgPath) + } + // CNN (Convolutional Neural Network) based detection + return rec.RecognizeFileCNN(imgPath) +} diff --git a/faces/types.go b/faces/types.go new file mode 100644 index 0000000..eb50f2a --- /dev/null +++ b/faces/types.go @@ -0,0 +1,37 @@ +package faces + +import "encoding/json" + +const ( + IndexTop = 0 + IndexRight = 1 + IndexBottom = 2 + IndexLeft = 3 +) + +type ( + FaceBoundaries [4]int + FaceBoundariesList []FaceBoundaries + FaceEncoding [128]float64 + FaceEncodingList []FaceEncoding + FaceDetectionResult struct { + Locations FaceBoundariesList `json:"locations"` + Encodings FaceEncodingList `json:"encodings"` + } +) + +func toFacesResult(data []byte) (result FaceDetectionResult, err error) { + // Parse the string + err = json.Unmarshal(data, &result) + return result, err +} + +func (l *FaceBoundaries) ToJSONString() string { + data, _ := json.Marshal(l) + return string(data) +} + +func (e *FaceEncoding) ToJSONString() string { + data, _ := json.Marshal(e) + return string(data) +} diff --git a/go.mod b/go.mod index c8fbdf7..50b3fd5 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module server go 1.21 require ( + github.com/Kagami/go-face v0.0.0-20210630145111-0c14797b4d0e github.com/aws/aws-sdk-go v1.45.25 github.com/gin-contrib/cors v1.6.0 github.com/gin-contrib/gzip v0.0.6 diff --git a/go.sum b/go.sum index 2002318..06bf8d3 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,6 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Kagami/go-face v0.0.0-20210630145111-0c14797b4d0e h1:lqIUFzxaqyYqUn4MhzAvSAh4wIte/iLNcIEWxpT/qbc= +github.com/Kagami/go-face v0.0.0-20210630145111-0c14797b4d0e/go.mod h1:9wdDJkRgo3SGTcFwbQ7elVIQhIr2bbBjecuY7VoqmPU= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/aws/aws-sdk-go v1.45.25 h1:c4fLlh5sLdK2DCRTY1z0hyuJZU4ygxX8m1FswL6/nF4= github.com/aws/aws-sdk-go v1.45.25/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= diff --git a/handlers/asset.go b/handlers/asset.go index b0d8215..33b185b 100644 --- a/handlers/asset.go +++ b/handlers/asset.go @@ -86,20 +86,28 @@ func LoadAssetsFromRows(c *gin.Context, rows *sql.Rows) *[]AssetInfo { } func AssetList(c *gin.Context, user *models.User) { + fr := AssetsForFaceRequest{} + _ = c.ShouldBindQuery(&fr) + // Modified depends on deleted assets as well, that's why the where condition is different tx := db.Instance. Table("assets"). Select("max(updated_at)"). Where("user_id=? AND size>0 AND thumb_size>0", user.ID) - if isNotModified(c, tx) { + if fr.FaceID == 0 && c.Query("reload") != "1" && isNotModified(c, tx) { return } // TODO: For big sets maybe dynamically load asset info individually? - rows, err := db.Instance. + tmp := db.Instance. Table("assets"). Select(AssetsSelectClause). Joins("left join favourite_assets on favourite_assets.asset_id = assets.id"). - Joins(LeftJoinForLocations). + Joins(LeftJoinForLocations) + if fr.FaceID > 0 { + // Find assets with faces similar to the given face or with the same person already assigned + tmp = tmp.Joins("join (select distinct t2.asset_id from faces t1 join faces t2 where t1.id=? and (t1.person_id = t2.person_id OR "+models.FacesVectorDistance+" <= ?)) f on f.asset_id = assets.id", fr.FaceID, fr.Threshold) + } + rows, err := tmp. Where("assets.user_id=? and assets.deleted=0 and assets.size>0 and assets.thumb_size>0", user.ID).Order("assets.created_at DESC").Rows() if err != nil { c.JSON(http.StatusInternalServerError, DBError1Response) @@ -217,23 +225,22 @@ func AssetDelete(c *gin.Context, user *models.User) { log.Printf("Asset: %d, auth error", id) continue } - asset.Deleted = true - err = db.Instance.Save(&asset).Error - if err != nil { + // Delete asset record and rely on cascaded deletes + if db.Instance.Exec("delete from assets where id=?", id).Error != nil { failed = append(failed, id) - log.Printf("Asset: %d, save error %s", id, err) + log.Printf("Asset: %d, delete error %s", id, err) continue } - // TODO: Delete record better (and rely on cascaded deletes) and reinsert with same RemoteID (to stop re-uploading)? - db.Instance.Exec("delete from album_assets where asset_id=?", id) - db.Instance.Exec("delete from favourite_assets where asset_id=?", id) + // Re-insert with same RemoteID to stop backing up the same asset + db.Instance.Exec("insert into assets (user_id, remote_id, updated_at, deleted) values (?, ?, ?, 1)", asset.UserID, asset.RemoteID, time.Now().Unix()) + storage := storage.StorageFrom(&asset.Bucket) if storage == nil { log.Printf("Asset: %d, error: storage is nil", id) failed = append(failed, id) continue } - // Finally delete + // Finally delete the files if err = storage.Delete(asset.ThumbPath); err != nil { log.Printf("Asset: %d, thumb delete error: %s", id, err.Error()) } diff --git a/handlers/faces.go b/handlers/faces.go new file mode 100644 index 0000000..b5fb2f7 --- /dev/null +++ b/handlers/faces.go @@ -0,0 +1,196 @@ +package handlers + +import ( + "net/http" + "server/config" + "server/db" + "server/models" + "strconv" + "strings" + + _ "image/jpeg" + + "github.com/gin-gonic/gin" +) + +type FaceInfo struct { + ID uint64 `json:"id"` + Num int `json:"num"` + PersonID uint64 `json:"person_id"` + PersonName string `json:"person_name"` + AsselID uint64 `json:"asset_id"` + X1 int `json:"x1"` + Y1 int `json:"y1"` + X2 int `json:"x2"` + Y2 int `json:"y2"` +} + +type AssetsForFaceRequest struct { + FaceID uint64 `form:"face_id" binding:"required"` + Threshold float64 `form:"threshold"` +} + +func FacesForAsset(c *gin.Context, user *models.User) { + assetIDSt, exists := c.GetQuery("asset_id") + if !exists { + c.JSON(http.StatusBadRequest, Response{"Missing asset ID"}) + return + } + assetID, err := strconv.ParseUint(assetIDSt, 10, 64) + if err != nil { + c.JSON(http.StatusBadRequest, Response{"Invalid asset ID"}) + return + } + asset := models.Asset{ID: assetID} + db.Instance.First(&asset) + if asset.ID != assetID || asset.UserID != user.ID { + c.JSON(http.StatusUnauthorized, NopeResponse) + return + } + rows, err := db.Instance.Raw("select f.id, f.num, f.x1, f.y1, f.x2, f.y2, p.id, p.name from faces f left join people p on f.person_id=p.id where f.asset_id=?", assetID).Rows() + if err != nil { + c.JSON(http.StatusInternalServerError, DBError1Response) + return + } + defer rows.Close() + result := []FaceInfo{} + for rows.Next() { + face := FaceInfo{} + pID := &face.PersonID + pName := &face.PersonName + if err = rows.Scan(&face.ID, &face.Num, &face.X1, &face.Y1, &face.X2, &face.Y2, &pID, &pName); err != nil { + c.JSON(http.StatusInternalServerError, DBError2Response) + return + } + if pID != nil { + face.PersonID = *pID + } + if pName != nil { + face.PersonName = *pName + } + result = append(result, face) + } + c.JSON(http.StatusOK, result) +} + +func PeopleList(c *gin.Context, user *models.User) { + // Do this in two steps. First load all people information + rows, err := db.Instance.Raw("select id, name from people where user_id=?", user.ID).Rows() + if err != nil { + c.JSON(http.StatusInternalServerError, DBError1Response) + return + } + // Put the info in the FaceInfo struct + people := []FaceInfo{} + for rows.Next() { + person := FaceInfo{} + if err = rows.Scan(&person.PersonID, &person.PersonName); err != nil { + c.JSON(http.StatusInternalServerError, DBError2Response) + rows.Close() + return + } + people = append(people, person) + } + rows.Close() + + // Now load the last face for each person + for i, person := range people { + rows, err = db.Instance.Raw("select id, asset_id, num, x1, y1, x2, y2 from faces where person_id=? order by created_at desc limit 1", person.PersonID).Rows() + if err != nil { + c.JSON(http.StatusInternalServerError, DBError3Response) + return + } + if rows.Next() { + face := &people[i] + if err = rows.Scan(&face.ID, &face.AsselID, &face.Num, &face.X1, &face.Y1, &face.X2, &face.Y2); err != nil { + c.JSON(http.StatusInternalServerError, DBError4Response) + rows.Close() + return + } + } + rows.Close() + } + c.JSON(http.StatusOK, people) +} + +func CreatePerson(c *gin.Context, user *models.User) { + var personFace FaceInfo + err := c.ShouldBindJSON(&personFace) + if err != nil { + c.JSON(http.StatusInternalServerError, Response{err.Error()}) + return + } + personFace.PersonName = strings.Trim(personFace.PersonName, " ") + if personFace.PersonName == "" { + c.JSON(http.StatusBadRequest, Response{"Empty person name"}) + return + } + personModel := models.Person{Name: personFace.PersonName, UserID: user.ID} + if db.Instance.Create(&personModel).Error != nil { + c.JSON(http.StatusInternalServerError, DBError1Response) + return + } + personFace.PersonID = personModel.ID + c.JSON(http.StatusOK, personFace) +} + +func PersonAssignFace(c *gin.Context, user *models.User) { + var face FaceInfo + err := c.ShouldBindJSON(&face) + if err != nil { + c.JSON(http.StatusInternalServerError, Response{err.Error()}) + return + } + if face.PersonID == 0 { + if face.ID == 0 { + c.JSON(http.StatusBadRequest, Response{"Empty face ID and person ID"}) + return + } + // We want to unassign a face from a person + if db.Instance.Exec("update faces set person_id=null where id=?", face.ID).Error != nil { + c.JSON(http.StatusInternalServerError, DBError1Response) + return + } + face.PersonID = 0 + face.PersonName = "" + c.JSON(http.StatusOK, face) + return + } + // Check if this face.PersonID is the same as current user.ID + person := models.Person{ID: face.PersonID} + if db.Instance.First(&person).Error != nil || person.UserID != user.ID { + c.JSON(http.StatusUnauthorized, NopeResponse) + return + } + if db.Instance.Exec("update faces set person_id=? where id=?", face.PersonID, face.ID).Error != nil { + c.JSON(http.StatusInternalServerError, DBError1Response) + return + } + // threshold is squared by default + thresholdStr := c.Query("threshold") + threshold, _ := strconv.ParseFloat(thresholdStr, 64) + if threshold == 0 { + threshold = config.FACE_MAX_DISTANCE_SQ + } + // Set PersonID to all assets with faces similar to the given face based on threshold + // Also, make sure the distance is greater than the current face's distance (i.e. the new face is more similar to the one detected before) + if db.Instance.Exec(`update faces + set person_id=? + where id in ( + select t2.id + from faces t1 join faces t2 + where t1.id=? and + t1.id!=t2.id and + t1.user_id=? and + t1.user_id=t2.user_id and + `+models.FacesVectorDistance+` <= ? and + (t2.distance == 0 OR t2.distance > `+models.FacesVectorDistance+`) + )`, + face.PersonID, face.ID, user.ID, threshold).Error != nil { + + c.JSON(http.StatusInternalServerError, DBError2Response) + return + } + face.PersonName = person.Name + c.JSON(http.StatusOK, face) +} diff --git a/handlers/handlers.go b/handlers/handlers.go index e97b808..51dc4fd 100644 --- a/handlers/handlers.go +++ b/handlers/handlers.go @@ -17,10 +17,6 @@ type MultiResponse struct { Failed []uint64 `json:"failed"` } -const ( - etagHeader = "ETag" -) - var ( // Predefined errors OKResponse = Response{} @@ -42,10 +38,10 @@ func isNotModified(c *gin.Context, tx *gorm.DB) bool { } // Set the current ETag c.Header("cache-control", "private, max-age=1") - c.Header(etagHeader, strconv.FormatUint(lastUpdatedAt, 10)) + c.Header("etag", strconv.FormatUint(lastUpdatedAt, 10)) // ETag contains last updated asset time - remoteLastUpdatedAt, _ := strconv.ParseUint(c.Request.Header.Get("If-None-Match"), 10, 64) + remoteLastUpdatedAt, _ := strconv.ParseUint(c.Request.Header.Get("if-none-match"), 10, 64) if remoteLastUpdatedAt == lastUpdatedAt { c.Status(http.StatusNotModified) return true diff --git a/handlers/moment.go b/handlers/moment.go index d520e48..2553428 100644 --- a/handlers/moment.go +++ b/handlers/moment.go @@ -43,7 +43,10 @@ func MomentList(c *gin.Context, user *models.User) { // TODO: Minimum number of assets for a location should be configurable (now 6 below) rows, err := db.Instance.Raw(` select date, - if(city = '', area, city), + CASE + WHEN city = '' THEN area + ELSE city + END, group_concat(place_id) places, max(hero), min(start), diff --git a/handlers/tag.go b/handlers/tag.go index 58d8343..a5afcc0 100644 --- a/handlers/tag.go +++ b/handlers/tag.go @@ -69,7 +69,7 @@ func (t *Tags) add(typ int, val any, assetId uint64) { func TagList(c *gin.Context, user *models.User) { // Modified depends on deleted assets as well, that's why the where condition is different tx := db.Instance.Table("assets").Select("max(updated_at)").Where("user_id=? AND size>0 AND thumb_size>0", user.ID) - if isNotModified(c, tx) { + if c.Query("reload") != "1" && isNotModified(c, tx) { return } rows, err := db.Instance.Table("assets").Select("id, mime_type, favourite, created_at, locations.gps_lat, locations.gps_long, area, city, country"). @@ -121,6 +121,23 @@ func TagList(c *gin.Context, user *models.User) { tags.add(tagTypeFavourite, "Favourite", assetId) } } + // Find all people, all their faces in assets and add them as tags + rows, err = db.Instance.Raw("select p.id, p.name, f.asset_id from people p join faces f on f.person_id=p.id").Rows() + if err != nil { + c.JSON(http.StatusInternalServerError, DBError3Response) + return + } + var personId uint64 + var personName string + defer rows.Close() + for rows.Next() { + if err = rows.Scan(&personId, &personName, &assetId); err != nil { + c.JSON(http.StatusInternalServerError, DBError4Response) + return + } + tags.add(tagTypePerson, personName, assetId) + } + result := tags.toArray() // Sort tags by popularity (num assets) sort.Slice(result, func(i, j int) bool { diff --git a/main.go b/main.go index 4481249..d3349d1 100644 --- a/main.go +++ b/main.go @@ -1,3 +1,7 @@ +// Copyright 2024 Nikolay Dimitrov. +// All rights reserved. +// Use of this source code is governed by a MIT style license that can be found in the LICENSE file. + package main import ( @@ -91,6 +95,10 @@ func main() { authRouter.POST("/asset/delete", handlers.AssetDelete, models.PermissionPhotoUpload) // TODO: S3 Delete done? authRouter.POST("/asset/favourite", handlers.AssetFavourite) authRouter.POST("/asset/unfavourite", handlers.AssetUnfavourite) + authRouter.GET("/faces/for-asset", handlers.FacesForAsset, models.PermissionPhotoUpload) + authRouter.GET("/faces/people", handlers.PeopleList, models.PermissionPhotoUpload) + authRouter.POST("/faces/create-person", handlers.CreatePerson, models.PermissionPhotoUpload) + authRouter.POST("/faces/assign", handlers.PersonAssignFace, models.PermissionPhotoUpload) // Album handlers authRouter.GET("/album/list", handlers.AlbumList) authRouter.POST("/album/create", handlers.AlbumCreate, models.PermissionPhotoUpload) diff --git a/models/asset.go b/models/asset.go index b8cd340..66d79f8 100644 --- a/models/asset.go +++ b/models/asset.go @@ -52,7 +52,6 @@ type Asset struct { Duration uint32 Path string `gorm:"type:varchar(2048)"` // Full path of the asset, including file/object name ThumbPath string `gorm:"type:varchar(2048)"` // Same but for thumbnail - Processed bool `gorm:"not null;default 0"` PresignedUntil int64 PresignedURL string `gorm:"type:varchar(2000)"` PresignedThumbUntil int64 diff --git a/models/asset_test.go b/models/asset_test.go index cfb2c11..c346f0b 100644 --- a/models/asset_test.go +++ b/models/asset_test.go @@ -38,7 +38,6 @@ func TestAsset_GetCreatedTimeInLocation(t *testing.T) { Duration uint32 Path string ThumbPath string - Processed bool PresignedUntil int64 PresignedURL string PresignedThumbUntil int64 @@ -97,7 +96,6 @@ func TestAsset_GetCreatedTimeInLocation(t *testing.T) { Duration: tt.fields.Duration, Path: tt.fields.Path, ThumbPath: tt.fields.ThumbPath, - Processed: tt.fields.Processed, PresignedUntil: tt.fields.PresignedUntil, PresignedURL: tt.fields.PresignedURL, PresignedThumbUntil: tt.fields.PresignedThumbUntil, diff --git a/models/face.go b/models/face.go new file mode 100644 index 0000000..ab05605 --- /dev/null +++ b/models/face.go @@ -0,0 +1,150 @@ +package models + +const ( + FacesVectorDistance = "(t2.v0-t1.v0)*(t2.v0-t1.v0) + (t2.v1-t1.v1)*(t2.v1-t1.v1) + (t2.v2-t1.v2)*(t2.v2-t1.v2) + (t2.v3-t1.v3)*(t2.v3-t1.v3) + (t2.v4-t1.v4)*(t2.v4-t1.v4) + (t2.v5-t1.v5)*(t2.v5-t1.v5) + (t2.v6-t1.v6)*(t2.v6-t1.v6) + (t2.v7-t1.v7)*(t2.v7-t1.v7) + (t2.v8-t1.v8)*(t2.v8-t1.v8) + (t2.v9-t1.v9)*(t2.v9-t1.v9) + (t2.v10-t1.v10)*(t2.v10-t1.v10) + (t2.v11-t1.v11)*(t2.v11-t1.v11) + (t2.v12-t1.v12)*(t2.v12-t1.v12) + (t2.v13-t1.v13)*(t2.v13-t1.v13) + (t2.v14-t1.v14)*(t2.v14-t1.v14) + (t2.v15-t1.v15)*(t2.v15-t1.v15) + (t2.v16-t1.v16)*(t2.v16-t1.v16) + (t2.v17-t1.v17)*(t2.v17-t1.v17) + (t2.v18-t1.v18)*(t2.v18-t1.v18) + (t2.v19-t1.v19)*(t2.v19-t1.v19) + (t2.v20-t1.v20)*(t2.v20-t1.v20) + (t2.v21-t1.v21)*(t2.v21-t1.v21) + (t2.v22-t1.v22)*(t2.v22-t1.v22) + (t2.v23-t1.v23)*(t2.v23-t1.v23) + (t2.v24-t1.v24)*(t2.v24-t1.v24) + (t2.v25-t1.v25)*(t2.v25-t1.v25) + (t2.v26-t1.v26)*(t2.v26-t1.v26) + (t2.v27-t1.v27)*(t2.v27-t1.v27) + (t2.v28-t1.v28)*(t2.v28-t1.v28) + (t2.v29-t1.v29)*(t2.v29-t1.v29) + (t2.v30-t1.v30)*(t2.v30-t1.v30) + (t2.v31-t1.v31)*(t2.v31-t1.v31) + (t2.v32-t1.v32)*(t2.v32-t1.v32) + (t2.v33-t1.v33)*(t2.v33-t1.v33) + (t2.v34-t1.v34)*(t2.v34-t1.v34) + (t2.v35-t1.v35)*(t2.v35-t1.v35) + (t2.v36-t1.v36)*(t2.v36-t1.v36) + (t2.v37-t1.v37)*(t2.v37-t1.v37) + (t2.v38-t1.v38)*(t2.v38-t1.v38) + (t2.v39-t1.v39)*(t2.v39-t1.v39) + (t2.v40-t1.v40)*(t2.v40-t1.v40) + (t2.v41-t1.v41)*(t2.v41-t1.v41) + (t2.v42-t1.v42)*(t2.v42-t1.v42) + (t2.v43-t1.v43)*(t2.v43-t1.v43) + (t2.v44-t1.v44)*(t2.v44-t1.v44) + (t2.v45-t1.v45)*(t2.v45-t1.v45) + (t2.v46-t1.v46)*(t2.v46-t1.v46) + (t2.v47-t1.v47)*(t2.v47-t1.v47) + (t2.v48-t1.v48)*(t2.v48-t1.v48) + (t2.v49-t1.v49)*(t2.v49-t1.v49) + (t2.v50-t1.v50)*(t2.v50-t1.v50) + (t2.v51-t1.v51)*(t2.v51-t1.v51) + (t2.v52-t1.v52)*(t2.v52-t1.v52) + (t2.v53-t1.v53)*(t2.v53-t1.v53) + (t2.v54-t1.v54)*(t2.v54-t1.v54) + (t2.v55-t1.v55)*(t2.v55-t1.v55) + (t2.v56-t1.v56)*(t2.v56-t1.v56) + (t2.v57-t1.v57)*(t2.v57-t1.v57) + (t2.v58-t1.v58)*(t2.v58-t1.v58) + (t2.v59-t1.v59)*(t2.v59-t1.v59) + (t2.v60-t1.v60)*(t2.v60-t1.v60) + (t2.v61-t1.v61)*(t2.v61-t1.v61) + (t2.v62-t1.v62)*(t2.v62-t1.v62) + (t2.v63-t1.v63)*(t2.v63-t1.v63) + (t2.v64-t1.v64)*(t2.v64-t1.v64) + (t2.v65-t1.v65)*(t2.v65-t1.v65) + (t2.v66-t1.v66)*(t2.v66-t1.v66) + (t2.v67-t1.v67)*(t2.v67-t1.v67) + (t2.v68-t1.v68)*(t2.v68-t1.v68) + (t2.v69-t1.v69)*(t2.v69-t1.v69) + (t2.v70-t1.v70)*(t2.v70-t1.v70) + (t2.v71-t1.v71)*(t2.v71-t1.v71) + (t2.v72-t1.v72)*(t2.v72-t1.v72) + (t2.v73-t1.v73)*(t2.v73-t1.v73) + (t2.v74-t1.v74)*(t2.v74-t1.v74) + (t2.v75-t1.v75)*(t2.v75-t1.v75) + (t2.v76-t1.v76)*(t2.v76-t1.v76) + (t2.v77-t1.v77)*(t2.v77-t1.v77) + (t2.v78-t1.v78)*(t2.v78-t1.v78) + (t2.v79-t1.v79)*(t2.v79-t1.v79) + (t2.v80-t1.v80)*(t2.v80-t1.v80) + (t2.v81-t1.v81)*(t2.v81-t1.v81) + (t2.v82-t1.v82)*(t2.v82-t1.v82) + (t2.v83-t1.v83)*(t2.v83-t1.v83) + (t2.v84-t1.v84)*(t2.v84-t1.v84) + (t2.v85-t1.v85)*(t2.v85-t1.v85) + (t2.v86-t1.v86)*(t2.v86-t1.v86) + (t2.v87-t1.v87)*(t2.v87-t1.v87) + (t2.v88-t1.v88)*(t2.v88-t1.v88) + (t2.v89-t1.v89)*(t2.v89-t1.v89) + (t2.v90-t1.v90)*(t2.v90-t1.v90) + (t2.v91-t1.v91)*(t2.v91-t1.v91) + (t2.v92-t1.v92)*(t2.v92-t1.v92) + (t2.v93-t1.v93)*(t2.v93-t1.v93) + (t2.v94-t1.v94)*(t2.v94-t1.v94) + (t2.v95-t1.v95)*(t2.v95-t1.v95) + (t2.v96-t1.v96)*(t2.v96-t1.v96) + (t2.v97-t1.v97)*(t2.v97-t1.v97) + (t2.v98-t1.v98)*(t2.v98-t1.v98) + (t2.v99-t1.v99)*(t2.v99-t1.v99) + (t2.v100-t1.v100)*(t2.v100-t1.v100) + (t2.v101-t1.v101)*(t2.v101-t1.v101) + (t2.v102-t1.v102)*(t2.v102-t1.v102) + (t2.v103-t1.v103)*(t2.v103-t1.v103) + (t2.v104-t1.v104)*(t2.v104-t1.v104) + (t2.v105-t1.v105)*(t2.v105-t1.v105) + (t2.v106-t1.v106)*(t2.v106-t1.v106) + (t2.v107-t1.v107)*(t2.v107-t1.v107) + (t2.v108-t1.v108)*(t2.v108-t1.v108) + (t2.v109-t1.v109)*(t2.v109-t1.v109) + (t2.v110-t1.v110)*(t2.v110-t1.v110) + (t2.v111-t1.v111)*(t2.v111-t1.v111) + (t2.v112-t1.v112)*(t2.v112-t1.v112) + (t2.v113-t1.v113)*(t2.v113-t1.v113) + (t2.v114-t1.v114)*(t2.v114-t1.v114) + (t2.v115-t1.v115)*(t2.v115-t1.v115) + (t2.v116-t1.v116)*(t2.v116-t1.v116) + (t2.v117-t1.v117)*(t2.v117-t1.v117) + (t2.v118-t1.v118)*(t2.v118-t1.v118) + (t2.v119-t1.v119)*(t2.v119-t1.v119) + (t2.v120-t1.v120)*(t2.v120-t1.v120) + (t2.v121-t1.v121)*(t2.v121-t1.v121) + (t2.v122-t1.v122)*(t2.v122-t1.v122) + (t2.v123-t1.v123)*(t2.v123-t1.v123) + (t2.v124-t1.v124)*(t2.v124-t1.v124) + (t2.v125-t1.v125)*(t2.v125-t1.v125) + (t2.v126-t1.v126)*(t2.v126-t1.v126) + (t2.v127-t1.v127)*(t2.v127-t1.v127)" +) + +type Face struct { + ID uint64 `gorm:"primaryKey"` + UserID uint64 `gorm:"index:user_index,priority:1"` + User User `gorm:"constraint:OnUpdate:CASCADE,OnDelete:CASCADE;"` + CreatedAt int64 `gorm:"index:user_index,priority:2"` + AssetID uint64 `gorm:"index:uniq_asset_face,unique;priority:1"` + Asset Asset `gorm:"constraint:OnUpdate:CASCADE,OnDelete:CASCADE;"` + PersonID *uint64 `gorm:""` + Person Person `gorm:"constraint:OnUpdate:CASCADE,OnDelete:SET NULL;"` + Distance float64 `gorm:"type:double"` + Num int `gorm:"index:uniq_asset_face,unique;"` + X1 int `gorm:"type:int"` + Y1 int `gorm:"type:int"` + X2 int `gorm:"type:int"` + Y2 int `gorm:"type:int"` + V0 float32 `gorm:"type:double"` + V1 float32 `gorm:"type:double"` + V2 float32 `gorm:"type:double"` + V3 float32 `gorm:"type:double"` + V4 float32 `gorm:"type:double"` + V5 float32 `gorm:"type:double"` + V6 float32 `gorm:"type:double"` + V7 float32 `gorm:"type:double"` + V8 float32 `gorm:"type:double"` + V9 float32 `gorm:"type:double"` + V10 float32 `gorm:"type:double"` + V11 float32 `gorm:"type:double"` + V12 float32 `gorm:"type:double"` + V13 float32 `gorm:"type:double"` + V14 float32 `gorm:"type:double"` + V15 float32 `gorm:"type:double"` + V16 float32 `gorm:"type:double"` + V17 float32 `gorm:"type:double"` + V18 float32 `gorm:"type:double"` + V19 float32 `gorm:"type:double"` + V20 float32 `gorm:"type:double"` + V21 float32 `gorm:"type:double"` + V22 float32 `gorm:"type:double"` + V23 float32 `gorm:"type:double"` + V24 float32 `gorm:"type:double"` + V25 float32 `gorm:"type:double"` + V26 float32 `gorm:"type:double"` + V27 float32 `gorm:"type:double"` + V28 float32 `gorm:"type:double"` + V29 float32 `gorm:"type:double"` + V30 float32 `gorm:"type:double"` + V31 float32 `gorm:"type:double"` + V32 float32 `gorm:"type:double"` + V33 float32 `gorm:"type:double"` + V34 float32 `gorm:"type:double"` + V35 float32 `gorm:"type:double"` + V36 float32 `gorm:"type:double"` + V37 float32 `gorm:"type:double"` + V38 float32 `gorm:"type:double"` + V39 float32 `gorm:"type:double"` + V40 float32 `gorm:"type:double"` + V41 float32 `gorm:"type:double"` + V42 float32 `gorm:"type:double"` + V43 float32 `gorm:"type:double"` + V44 float32 `gorm:"type:double"` + V45 float32 `gorm:"type:double"` + V46 float32 `gorm:"type:double"` + V47 float32 `gorm:"type:double"` + V48 float32 `gorm:"type:double"` + V49 float32 `gorm:"type:double"` + V50 float32 `gorm:"type:double"` + V51 float32 `gorm:"type:double"` + V52 float32 `gorm:"type:double"` + V53 float32 `gorm:"type:double"` + V54 float32 `gorm:"type:double"` + V55 float32 `gorm:"type:double"` + V56 float32 `gorm:"type:double"` + V57 float32 `gorm:"type:double"` + V58 float32 `gorm:"type:double"` + V59 float32 `gorm:"type:double"` + V60 float32 `gorm:"type:double"` + V61 float32 `gorm:"type:double"` + V62 float32 `gorm:"type:double"` + V63 float32 `gorm:"type:double"` + V64 float32 `gorm:"type:double"` + V65 float32 `gorm:"type:double"` + V66 float32 `gorm:"type:double"` + V67 float32 `gorm:"type:double"` + V68 float32 `gorm:"type:double"` + V69 float32 `gorm:"type:double"` + V70 float32 `gorm:"type:double"` + V71 float32 `gorm:"type:double"` + V72 float32 `gorm:"type:double"` + V73 float32 `gorm:"type:double"` + V74 float32 `gorm:"type:double"` + V75 float32 `gorm:"type:double"` + V76 float32 `gorm:"type:double"` + V77 float32 `gorm:"type:double"` + V78 float32 `gorm:"type:double"` + V79 float32 `gorm:"type:double"` + V80 float32 `gorm:"type:double"` + V81 float32 `gorm:"type:double"` + V82 float32 `gorm:"type:double"` + V83 float32 `gorm:"type:double"` + V84 float32 `gorm:"type:double"` + V85 float32 `gorm:"type:double"` + V86 float32 `gorm:"type:double"` + V87 float32 `gorm:"type:double"` + V88 float32 `gorm:"type:double"` + V89 float32 `gorm:"type:double"` + V90 float32 `gorm:"type:double"` + V91 float32 `gorm:"type:double"` + V92 float32 `gorm:"type:double"` + V93 float32 `gorm:"type:double"` + V94 float32 `gorm:"type:double"` + V95 float32 `gorm:"type:double"` + V96 float32 `gorm:"type:double"` + V97 float32 `gorm:"type:double"` + V98 float32 `gorm:"type:double"` + V99 float32 `gorm:"type:double"` + V100 float32 `gorm:"type:double"` + V101 float32 `gorm:"type:double"` + V102 float32 `gorm:"type:double"` + V103 float32 `gorm:"type:double"` + V104 float32 `gorm:"type:double"` + V105 float32 `gorm:"type:double"` + V106 float32 `gorm:"type:double"` + V107 float32 `gorm:"type:double"` + V108 float32 `gorm:"type:double"` + V109 float32 `gorm:"type:double"` + V110 float32 `gorm:"type:double"` + V111 float32 `gorm:"type:double"` + V112 float32 `gorm:"type:double"` + V113 float32 `gorm:"type:double"` + V114 float32 `gorm:"type:double"` + V115 float32 `gorm:"type:double"` + V116 float32 `gorm:"type:double"` + V117 float32 `gorm:"type:double"` + V118 float32 `gorm:"type:double"` + V119 float32 `gorm:"type:double"` + V120 float32 `gorm:"type:double"` + V121 float32 `gorm:"type:double"` + V122 float32 `gorm:"type:double"` + V123 float32 `gorm:"type:double"` + V124 float32 `gorm:"type:double"` + V125 float32 `gorm:"type:double"` + V126 float32 `gorm:"type:double"` + V127 float32 `gorm:"type:double"` +} diff --git a/models/init.go b/models/init.go index d4a0b7b..507215c 100644 --- a/models/init.go +++ b/models/init.go @@ -18,6 +18,7 @@ func Init() { es = append(es, db.Instance.AutoMigrate(&AlbumAsset{})) es = append(es, db.Instance.AutoMigrate(&AlbumShare{})) es = append(es, db.Instance.AutoMigrate(&Asset{})) + es = append(es, db.Instance.AutoMigrate(&Face{})) es = append(es, db.Instance.AutoMigrate(&FavouriteAsset{})) es = append(es, db.Instance.AutoMigrate(&Grant{})) es = append(es, db.Instance.AutoMigrate(&Group{})) @@ -25,6 +26,7 @@ func Init() { es = append(es, db.Instance.AutoMigrate(&GroupUser{})) es = append(es, db.Instance.AutoMigrate(&Location{})) es = append(es, db.Instance.AutoMigrate(&Place{})) + es = append(es, db.Instance.AutoMigrate(&Person{})) es = append(es, db.Instance.AutoMigrate(&UploadRequest{})) es = append(es, db.Instance.AutoMigrate(&User{})) diff --git a/models/person.go b/models/person.go index f81dd12..b3ab91d 100644 --- a/models/person.go +++ b/models/person.go @@ -1,12 +1,14 @@ package models type Person struct { - ID uint64 `gorm:"primaryKey"` - AssetID uint64 - Asset Asset `gorm:"constraint:OnUpdate:CASCADE,OnDelete:CASCADE;"` - Vector []byte `gorm:"type:blob"` - RectX1 uint16 - RectY1 uint16 - RectX2 uint16 - RectY2 uint16 + ID uint64 `gorm:"primaryKey"` + CreatedAt int64 `gorm:""` + UserID uint64 `gorm:"index:uniq_user_person,unique;priority:1"` + User User `gorm:"constraint:OnUpdate:CASCADE,OnDelete:CASCADE;"` + Name string `gorm:"type:varchar(300);index:uniq_user_person,unique;priority:2"` +} + +// TableName overrides the table name +func (Person) TableName() string { + return "people" } diff --git a/processing/detectfaces.go b/processing/detectfaces.go new file mode 100644 index 0000000..75bb292 --- /dev/null +++ b/processing/detectfaces.go @@ -0,0 +1,76 @@ +package processing + +import ( + "log" + "reflect" + "server/config" + "server/db" + "server/faces" + "server/models" + "server/storage" + "strconv" +) + +type detectfaces struct{} + +func (t *detectfaces) shouldHandle(asset *models.Asset) bool { + return true +} + +func (t *detectfaces) requiresContent(asset *models.Asset) bool { + return true +} + +func (t *detectfaces) process(asset *models.Asset, storage storage.StorageAPI) (status int, clean func()) { + + if asset.ThumbPath == "" { + return Failed, nil + } + if storage.GetSize(asset.ThumbPath) <= 0 { + if storage.EnsureLocalFile(asset.ThumbPath) != nil { + return Failed, nil + } + } + clean = func() { + storage.ReleaseLocalFile(asset.ThumbPath) + } + // Extract faces + result, err := faces.Detect(storage.GetFullPath(asset.ThumbPath)) + if err != nil { + log.Printf("Error detecting faces for asset %d, path:%s: %s", asset.ID, asset.ThumbPath, err.Error()) + return Failed, nil + } + // Save faces' data to DB + for i, face := range result { + faceModel := models.Face{ + UserID: asset.UserID, + AssetID: asset.ID, + Num: i, + X1: face.Rectangle.Min.X, + Y1: face.Rectangle.Min.Y, + X2: face.Rectangle.Max.X, + Y2: face.Rectangle.Max.Y, + PersonID: nil, + } + // Use reflection to set Vx fields to the corresponding value from the array + for j, value := range face.Descriptor { + reflect.ValueOf(&faceModel).Elem().FieldByName("V" + strconv.Itoa(j)).SetFloat(float64(value)) + } + if err := db.Instance.Create(&faceModel).Error; err != nil { + log.Printf("Error saving face location for asset %d: %v", asset.ID, err) + return Failed, nil + } + // Find the face that is most similar (least distance) to this one and fetch it's person_id + db.Instance.Raw(`select t2.person_id, `+models.FacesVectorDistance+` as threshold + from faces t1 join faces t2 + where t1.id=? and t1.user_id=t2.user_id and t1.user_id=? and t2.person_id is not null and t1.id != t2.id + order by threshold limit 1`, faceModel.ID, asset.UserID).Row().Scan(&faceModel.PersonID, &faceModel.Distance) + log.Printf("Face %d, threshold: %f\n", faceModel.ID, faceModel.Distance) + if faceModel.PersonID != nil && faceModel.Distance <= config.FACE_MAX_DISTANCE_SQ { + // Update the current face with the found person_id + db.Instance.Exec("update faces set person_id=? where id=?", *faceModel.PersonID, faceModel.ID) + log.Printf("Updated face %d, person_id: %d\n", faceModel.ID, *faceModel.PersonID) + } + } + return Done, clean +} diff --git a/processing/processing.go b/processing/processing.go index db82369..3ac59f0 100644 --- a/processing/processing.go +++ b/processing/processing.go @@ -40,6 +40,7 @@ func Init() { tasks.register(&videoConvert{}) tasks.register(&metadata{}) tasks.register(&thumb{}) + tasks.register(&detectfaces{}) } func (ts *processingTasks) register(t processingTask) { @@ -87,7 +88,7 @@ func (ts *processingTasks) process(asset *models.Asset, assetStorage storage.Sto if cleanup != nil { cleanAll = append(cleanAll, cleanup) } - log.Printf("Task \"%s\", asset ID: %d, result: %d, time: %v", e.name, asset.ID, statusMap[e.name], timeConsumed) + log.Printf("Task \"%s\", asset ID: %d, result: %s, time: %v", e.name, asset.ID, statusConstMap[statusMap[e.name]], timeConsumed) } for _, clean := range cleanAll { clean() diff --git a/processing/processing_task.go b/processing/processing_task.go index 6cdfffa..6af4599 100644 --- a/processing/processing_task.go +++ b/processing/processing_task.go @@ -16,6 +16,18 @@ const ( FailedDB = 5 ) +var ( + // Map containing the status codes from above and their string representation + statusConstMap = map[int]string{ + Skipped: "Skipped", + UserSkipped: "UserSkipped", + Done: "Done", + Failed: "Failed", + FailedStorage: "FailedStorage", + FailedDB: "FailedDB", + } +) + type ProcessingTask struct { AssetID uint64 `gorm:"primaryKey"` Asset models.Asset `gorm:"constraint:OnUpdate:CASCADE,OnDelete:CASCADE;"` diff --git a/storage/storage.go b/storage/storage.go index 252d9db..e6cd12c 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -55,7 +55,7 @@ func Init() { } if len(buckets) == 0 { log.Printf("No Storage Buckets found") - // Create default bucket if MAIN_BUCKET_DIR is set + // Create default bucket if DEFAULT_BUCKET_DIR is set if config.DEFAULT_BUCKET_DIR != "" { log.Printf("Creating default bucket in directory: %s", config.DEFAULT_BUCKET_DIR) bucket := Bucket{