Merge pull request #9 from circled-me/face-recognition

Face recognition + use alpine as output image

Dockerfile

@@ -1,19 +1,26 @@
 FROM golang:1.22-alpine
-RUN apk add make gcc libc-dev mailcap
+RUN apk add dlib dlib-dev --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing/
+RUN apk add blas blas-dev cblas lapack lapack-dev libjpeg-turbo-dev cmake make gcc libc-dev g++ unzip libx11-dev pkgconf jpeg jpeg-dev libpng libpng-dev mailcap
 
 COPY go.mod /go/src/circled-server/
 COPY go.sum /go/src/circled-server/
 WORKDIR /go/src/circled-server/
 RUN go mod download
+RUN go build github.com/Kagami/go-face
 RUN CGO_CFLAGS="-D_LARGEFILE64_SOURCE" go build github.com/mattn/go-sqlite3
 COPY . /go/src/circled-server
 RUN CGO_ENABLED=1 CGO_CFLAGS="-D_LARGEFILE64_SOURCE" GOOS=linux go build -a -installsuffix cgo -o circled-server .
 
 # Final output image
 FROM alpine:3.20.1
-RUN apk --no-cache add ca-certificates exiftool tzdata ffmpeg
+RUN apk add dlib --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing/
+RUN apk --no-cache add ca-certificates exiftool tzdata blas cblas lapack libjpeg-turbo libstdc++ libgcc ffmpeg
 WORKDIR /opt/circled
 COPY --from=0 /etc/mime.types /etc/mime.types
 COPY --from=0 /go/src/circled-server/circled-server .
 COPY --from=0 /go/src/circled-server/templates ./templates
+# Use 68 landmarks model instead of 5 landmarks model
+ADD https://github.com/ageitgey/face_recognition_models/raw/master/face_recognition_models/models/shape_predictor_68_face_landmarks.dat ./models/shape_predictor_5_face_landmarks.dat
+ADD https://github.com/ageitgey/face_recognition_models/raw/master/face_recognition_models/models/dlib_face_recognition_resnet_model_v1.dat ./models/
+ADD https://github.com/ageitgey/face_recognition_models/raw/master/face_recognition_models/models/mmod_human_face_detector.dat ./models/
 ENTRYPOINT ["./circled-server"]

README.md

@@ -31,11 +31,12 @@ ___
   - Supports either locally mounted disks or
   - S3-compatible Services - this allows different users to use their own S3 bucket on the same server
 - Push notifications for new Album photos, etc
+- Face detection and tagging
 - Albums
   - Adding local server contributors and viewers
   - Sharing albums with anyone with a "secret" link
 - Chat with push notifications
-- Filtering photos by year, month, location, etc
+- Filtering photos by tagged person, year, month, location, etc
 - Moments - automatically grouping photos by time and location
 - Reverse geocoding for all assets
 - Automatic video conversion to web-compatible H.264 format
@@ -61,6 +62,9 @@ Current configuration environment variables:
 - `DEBUG_MODE` - currently defaults to `yes`
 - `DEFAULT_BUCKET_DIR` - a directory that will be used as default bucket if no other buckets exist (i.e. the first time you run the server)
 - `DEFAULT_ASSET_PATH_PATTERN` - the default path pattern to create subdirectories and file names based on asset info. Defaults to `<year>/<month>/<id>`
+- `PUSH_SERVER` - the push server URL. Defaults to `https://push.circled.me`
+- `FACE_DETECT_CNN` - use Convolutional Neural Network for face detection (as opposed to HOG). Much slower, but more accurate at different angles. Defaults to `no`
+- `FACE_MAX_DISTANCE_SQ` - squared distance between faces to consider them similar. Defaults to `0.11`
 
 ## docker-compose example
 ```yaml

config/config.go

@@ -2,6 +2,7 @@ package config
 
 import (
 	"os"
+	"strconv"
 	"strings"
 )
 
@@ -15,6 +16,8 @@ var (
 	TMP_DIR                    = "/tmp" // Used for temporary video conversion, etc (in case of S3 bucket)
 	DEFAULT_BUCKET_DIR         = ""     // Used for creating initial bucket
 	DEBUG_MODE                 = true
+	FACE_DETECT_CNN            = false // Use Convolutional Neural Network for face detection (as opposed to HOG). Much slower, supposedly more accurate at different angles
+	FACE_MAX_DISTANCE_SQ       = 0.11  // Squared distance between faces to consider them similar
 )
 
 func init() {
@@ -27,6 +30,8 @@ func init() {
 	readEnvString("DEFAULT_BUCKET_DIR", &DEFAULT_BUCKET_DIR)
 	readEnvString("DEFAULT_ASSET_PATH_PATTERN", &DEFAULT_ASSET_PATH_PATTERN)
 	readEnvBool("DEBUG_MODE", &DEBUG_MODE)
+	readEnvBool("FACE_DETECT_CNN", &FACE_DETECT_CNN)
+	readEnvFloat("FACE_MAX_DISTANCE_SQ", &FACE_MAX_DISTANCE_SQ)
 }
 
 func readEnvString(name string, value *string) {
@@ -45,3 +50,15 @@ func readEnvBool(name string, value *bool) {
 		*value = false
 	}
 }
+
+func readEnvFloat(name string, value *float64) {
+	v := os.Getenv(name)
+	if v == "" {
+		return
+	}
+	f, err := strconv.ParseFloat(v, 64)
+	if err != nil {
+		return
+	}
+	*value = f
+}

db/db.go

@@ -30,14 +30,10 @@ func Init() {
 		CreatedDateFunc = "date(from_unixtime(created_at))"
 	} else if config.SQLITE_FILE != "" {
 		// Sqlite setup
-		db, err = gorm.Open(sqlite.Open(config.SQLITE_FILE), &gorm.Config{})
+		db, err = gorm.Open(sqlite.Open(config.SQLITE_FILE+"?_foreign_keys=on"), &gorm.Config{})
 		if err != nil || db == nil {
 			log.Fatalf("SQLite DB error: %v", err)
 		}
-		db.Exec("PRAGMA foreign_keys = ON")
-		// if sqliteDB, err := db.DB(); err == nil && sqliteDB != nil {
-		// 	sqliteDB.SetMaxOpenConns(1)
-		// }
 		TimestampFunc = "strftime('%s', 'now')"
 		CreatedDateFunc = "date(created_at, 'unixepoch')"
 	} else {

faces/faces.go

@@ -0,0 +1,35 @@
+package faces
+
+import (
+	"log"
+	"path/filepath"
+	"server/config"
+
+	"github.com/Kagami/go-face"
+)
+
+var (
+	modelsDir = filepath.Join(".", "models")
+	rec       *face.Recognizer
+)
+
+func init() {
+	log.Println("Loading face recognition models...")
+	// Init the recognizer.
+	var err error
+	rec, err = face.NewRecognizer(modelsDir)
+	if err != nil {
+		log.Fatalf("Can't init face recognizer: %v", err)
+	}
+}
+
+func Detect(imgPath string) ([]face.Face, error) {
+	log.Printf("Detecting faces in %s", imgPath)
+	// Recognize faces on that image.
+	if !config.FACE_DETECT_CNN {
+		// HOG (Histogram of Oriented Gradients) based detection
+		return rec.RecognizeFile(imgPath)
+	}
+	// CNN (Convolutional Neural Network) based detection
+	return rec.RecognizeFileCNN(imgPath)
+}

faces/types.go

@@ -0,0 +1,37 @@
+package faces
+
+import "encoding/json"
+
+const (
+	IndexTop    = 0
+	IndexRight  = 1
+	IndexBottom = 2
+	IndexLeft   = 3
+)
+
+type (
+	FaceBoundaries      [4]int
+	FaceBoundariesList  []FaceBoundaries
+	FaceEncoding        [128]float64
+	FaceEncodingList    []FaceEncoding
+	FaceDetectionResult struct {
+		Locations FaceBoundariesList `json:"locations"`
+		Encodings FaceEncodingList   `json:"encodings"`
+	}
+)
+
+func toFacesResult(data []byte) (result FaceDetectionResult, err error) {
+	// Parse the string
+	err = json.Unmarshal(data, &result)
+	return result, err
+}
+
+func (l *FaceBoundaries) ToJSONString() string {
+	data, _ := json.Marshal(l)
+	return string(data)
+}
+
+func (e *FaceEncoding) ToJSONString() string {
+	data, _ := json.Marshal(e)
+	return string(data)
+}

go.mod

@@ -3,6 +3,7 @@ module server
 go 1.21
 
 require (
+	github.com/Kagami/go-face v0.0.0-20210630145111-0c14797b4d0e
 	github.com/aws/aws-sdk-go v1.45.25
 	github.com/gin-contrib/cors v1.6.0
 	github.com/gin-contrib/gzip v0.0.6

go.sum

@@ -1,4 +1,6 @@
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/Kagami/go-face v0.0.0-20210630145111-0c14797b4d0e h1:lqIUFzxaqyYqUn4MhzAvSAh4wIte/iLNcIEWxpT/qbc=
+github.com/Kagami/go-face v0.0.0-20210630145111-0c14797b4d0e/go.mod h1:9wdDJkRgo3SGTcFwbQ7elVIQhIr2bbBjecuY7VoqmPU=
 github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
 github.com/aws/aws-sdk-go v1.45.25 h1:c4fLlh5sLdK2DCRTY1z0hyuJZU4ygxX8m1FswL6/nF4=
 github.com/aws/aws-sdk-go v1.45.25/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=

handlers/asset.go

@@ -86,20 +86,28 @@ func LoadAssetsFromRows(c *gin.Context, rows *sql.Rows) *[]AssetInfo {
 }
 
 func AssetList(c *gin.Context, user *models.User) {
+	fr := AssetsForFaceRequest{}
+	_ = c.ShouldBindQuery(&fr)
+
 	// Modified depends on deleted assets as well, that's why the where condition is different
 	tx := db.Instance.
 		Table("assets").
 		Select("max(updated_at)").
 		Where("user_id=? AND size>0 AND thumb_size>0", user.ID)
-	if isNotModified(c, tx) {
+	if fr.FaceID == 0 && c.Query("reload") != "1" && isNotModified(c, tx) {
 		return
 	}
 	// TODO: For big sets maybe dynamically load asset info individually?
-	rows, err := db.Instance.
+	tmp := db.Instance.
 		Table("assets").
 		Select(AssetsSelectClause).
 		Joins("left join favourite_assets on favourite_assets.asset_id = assets.id").
-		Joins(LeftJoinForLocations).
+		Joins(LeftJoinForLocations)
+	if fr.FaceID > 0 {
+		// Find assets with faces similar to the given face or with the same person already assigned
+		tmp = tmp.Joins("join (select distinct t2.asset_id from faces t1 join faces t2 where t1.id=? and (t1.person_id = t2.person_id OR "+models.FacesVectorDistance+" <= ?)) f on f.asset_id = assets.id", fr.FaceID, fr.Threshold)
+	}
+	rows, err := tmp.
 		Where("assets.user_id=? and assets.deleted=0 and assets.size>0 and assets.thumb_size>0", user.ID).Order("assets.created_at DESC").Rows()
 	if err != nil {
 		c.JSON(http.StatusInternalServerError, DBError1Response)
@@ -217,23 +225,22 @@ func AssetDelete(c *gin.Context, user *models.User) {
 			log.Printf("Asset: %d, auth error", id)
 			continue
 		}
-		asset.Deleted = true
-		err = db.Instance.Save(&asset).Error
-		if err != nil {
+		// Delete asset record and rely on cascaded deletes
+		if db.Instance.Exec("delete from assets where id=?", id).Error != nil {
 			failed = append(failed, id)
-			log.Printf("Asset: %d, save error %s", id, err)
+			log.Printf("Asset: %d, delete error %s", id, err)
 			continue
 		}
-		// TODO: Delete record better (and rely on cascaded deletes) and reinsert with same RemoteID (to stop re-uploading)?
-		db.Instance.Exec("delete from album_assets where asset_id=?", id)
-		db.Instance.Exec("delete from favourite_assets where asset_id=?", id)
+		// Re-insert with same RemoteID to stop backing up the same asset
+		db.Instance.Exec("insert into assets (user_id, remote_id, updated_at, deleted) values (?, ?, ?, 1)", asset.UserID, asset.RemoteID, time.Now().Unix())
+
 		storage := storage.StorageFrom(&asset.Bucket)
 		if storage == nil {
 			log.Printf("Asset: %d, error: storage is nil", id)
 			failed = append(failed, id)
 			continue
 		}
-		// Finally delete
+		// Finally delete the files
 		if err = storage.Delete(asset.ThumbPath); err != nil {
 			log.Printf("Asset: %d, thumb delete error: %s", id, err.Error())
 		}