SQLite support + Other improvements and fixes

Dockerfile

@@ -4,14 +4,13 @@ COPY go.sum /go/src/circled-server/
 WORKDIR /go/src/circled-server/
 RUN go mod download
 COPY . /go/src/circled-server
-RUN go build -o circled-server .
-RUN apk add mailcap
-
+RUN apk add mailcap gcc musl-dev
+RUN CGO_ENABLED=1 go build -o circled-server .
 
 FROM jrottenberg/ffmpeg:6-alpine
 RUN apk --no-cache add ca-certificates exiftool tzdata
 WORKDIR /opt/circled
 COPY --from=0 /etc/mime.types /etc/mime.types
 COPY --from=0 /go/src/circled-server/circled-server .
 COPY --from=0 /go/src/circled-server/templates ./templates
-ENTRYPOINT ["./circled-server"]
+ENTRYPOINT ["./circled-server"]

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2023 Nikolay Dimitrov
+Copyright (c) 2024 Nikolay Dimitrov
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

config/config.go

@@ -9,7 +9,8 @@ var (
 	TLS_DOMAINS        = ""                    // e.g. "example.com,example2.com"
 	DEFAULT_ASSET_PATH = "<year>/<month>/<id>" // also available: <name>, <Month>
 	PUSH_SERVER        = "https://push.circled.me"
-	MYSQL_DSN          = "root:@tcp(127.0.0.1:3306)/circled?charset=utf8mb4&parseTime=True&loc=Local"
+	MYSQL_DSN          = "" // MySQL will be used if this is set
+	SQLITE_FILE        = "" // SQLite will be used if MYSQL_DSN is not configured and this is set
 	BIND_ADDRESS       = "0.0.0.0:8080"
 	TMP_DIR            = "/tmp" // Used for temporary video conversion, etc (in case of S3 bucket)
 	DEBUG_MODE         = true
@@ -19,6 +20,7 @@ func init() {
 	readEnvString("TLS_DOMAINS", &TLS_DOMAINS)
 	readEnvString("PUSH_SERVER", &PUSH_SERVER)
 	readEnvString("MYSQL_DSN", &MYSQL_DSN)
+	readEnvString("SQLITE_FILE", &SQLITE_FILE)
 	readEnvString("BIND_ADDRESS", &BIND_ADDRESS)
 	readEnvString("TMP_DIR", &TMP_DIR)
 	readEnvBool("DEBUG_MODE", &DEBUG_MODE)

db/db.go

@@ -5,17 +5,40 @@ import (
 	"server/config"
 
 	"gorm.io/driver/mysql"
+	"gorm.io/driver/sqlite"
 	"gorm.io/gorm"
 )
 
-var Instance *gorm.DB
+var (
+	Instance      *gorm.DB
+	TimestampFunc = ""
+)
 
 func Init() {
-	db, err := gorm.Open(mysql.Open(config.MYSQL_DSN), &gorm.Config{
-		PrepareStmt: true,
-	})
-	if err != nil || db == nil {
-		log.Fatalf("DB error: %v", err)
+	var db *gorm.DB
+	var err error
+	if config.MYSQL_DSN != "" {
+		// MySQL setup
+		db, err = gorm.Open(mysql.Open(config.MYSQL_DSN), &gorm.Config{
+			PrepareStmt: true,
+		})
+		if err != nil || db == nil {
+			log.Fatalf("MySQL DB error: %v", err)
+		}
+		TimestampFunc = "unix_timestamp()"
+	} else if config.SQLITE_FILE != "" {
+		// Sqlite setup
+		db, err = gorm.Open(sqlite.Open(config.SQLITE_FILE), &gorm.Config{})
+		if err != nil || db == nil {
+			log.Fatalf("SQLite DB error: %v", err)
+		}
+		db.Exec("PRAGMA foreign_keys = ON")
+		// if sqliteDB, err := db.DB(); err == nil && sqliteDB != nil {
+		// 	sqliteDB.SetMaxOpenConns(1)
+		// }
+		TimestampFunc = "strftime('%s', 'now')"
+	} else {
+		log.Fatal("No database configuration found")
 	}
 	Instance = db
 }

docker-compose-example.yaml

@@ -2,33 +2,11 @@ version: '2'
 services:
   circled-server:
     image: gubble/circled-server:latest
-    # build:
-    #  dockerfile: Dockerfile
     restart: always
-    depends_on:
-      mysql:
-        condition: service_healthy
     ports:
       - "8080:8080"
     environment:
-      MYSQL_DSN: "root:@tcp(mysql:3306)/circled?charset=utf8mb4&parseTime=True&loc=Local"
-      BIND_ADDRESS: 0.0.0.0:8080
+      SQLITE_FILE: "/mnt/data1/circled.db"
+      BIND_ADDRESS: "0.0.0.0:8080"
     volumes:
-      - ./asset-data:/mnt/data1
-
-  mysql:
-    image: mysql:5.7
-    command: --default-authentication-plugin=mysql_native_password
-    restart: always
-    volumes:
-      - ./mysql-data:/var/lib/mysql
-    environment:
-      MYSQL_DATABASE: circled
-      MYSQL_ALLOW_EMPTY_PASSWORD: "yes"
-      MYSQL_ROOT_HOST: "%"
-    healthcheck:
-      test: mysqladmin ping --silent
-      start_period: 5s
-      interval: 3s
-      timeout: 5s
-      retries: 20
+      - ./circled-data:/mnt/data1

docker-compose-mysql-example.yaml

@@ -0,0 +1,34 @@
+version: '2'
+services:
+  circled-server:
+    image: gubble/circled-server:latest
+    # build:
+    #  dockerfile: Dockerfile
+    restart: always
+    depends_on:
+      mysql:
+        condition: service_healthy
+    ports:
+      - "8080:8080"
+    environment:
+      MYSQL_DSN: "root:@tcp(mysql:3306)/circled?charset=utf8mb4&parseTime=True&loc=Local"
+      BIND_ADDRESS: 0.0.0.0:8080
+    volumes:
+      - ./asset-data:/mnt/data1
+
+  mysql:
+    image: mysql:5.7
+    command: --default-authentication-plugin=mysql_native_password
+    restart: always
+    volumes:
+      - ./mysql-data:/var/lib/mysql
+    environment:
+      MYSQL_DATABASE: circled
+      MYSQL_ALLOW_EMPTY_PASSWORD: "yes"
+      MYSQL_ROOT_HOST: "%"
+    healthcheck:
+      test: mysqladmin ping --silent
+      start_period: 5s
+      interval: 3s
+      timeout: 5s
+      retries: 20

go.mod

@@ -15,7 +15,7 @@ require (
 	github.com/orcaman/concurrent-map/v2 v2.0.1
 	github.com/zsefvlol/timezonemapper v1.0.0
 	golang.org/x/sys v0.15.0
-	gorm.io/driver/mysql v1.5.2
+	gorm.io/driver/sqlite v1.4.1
 	gorm.io/gorm v1.25.5
 )
 
@@ -39,6 +39,7 @@ require (
 	github.com/klauspost/cpuid/v2 v2.2.5 // indirect
 	github.com/leodido/go-urn v1.2.4 // indirect
 	github.com/mattn/go-isatty v0.0.19 // indirect
+	github.com/mattn/go-sqlite3 v2.0.3+incompatible // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/pelletier/go-toml/v2 v2.1.0 // indirect
@@ -50,7 +51,8 @@ require (
 	golang.org/x/net v0.17.0 // indirect
 	golang.org/x/sync v0.4.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
-	google.golang.org/protobuf v1.31.0 // indirect
+	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
+	gorm.io/driver/mysql v1.5.2 // indirect
 // github.com/Kagami/go-face v0.0.0-20210630145111-0c14797b4d0e // indirect
 )

go.sum

@@ -325,8 +325,8 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1N
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
-google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
-google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
+google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=

handlers/album.go

@@ -72,7 +72,7 @@ type AlbumShareResponse struct {
 
 func getFirstFavouriteAssetID(userID uint64) uint64 {
 	fav := models.FavouriteAsset{}
-	db.Instance.First(&fav, "user_id = ?", userID)
+	db.Instance.Order("asset_id DESC").Limit(1).Find(&fav, "user_id = ?", userID)
 	return fav.AssetID
 }
 
@@ -324,9 +324,9 @@ func AlbumAssets(c *gin.Context, user *models.User) {
 		rows, err = db.Instance.
 			Table("favourite_assets").
 			Select(AssetsSelectClause).
-			Where("favourite_assets.user_id = ?", user.ID).
 			Joins("join assets on favourite_assets.asset_id = assets.id").
-			Joins("left join locations ON locations.gps_lat = truncate(assets.gps_lat, 4) AND locations.gps_long = truncate(assets.gps_long, 4)").
+			Joins(LeftJoinForLocations).
+			Where("favourite_assets.user_id = ? and assets.deleted = 0", user.ID).
 			Order("assets.created_at DESC").Rows()
 	} else {
 		// Normal album - check for access (own album or as a contributor)
@@ -342,7 +342,7 @@ func AlbumAssets(c *gin.Context, user *models.User) {
 			Where("album_id = ?", r.AlbumID).
 			Joins("join assets on album_assets.asset_id = assets.id").
 			Joins("left join favourite_assets on favourite_assets.asset_id = assets.id").
-			Joins("left join locations ON locations.gps_lat = truncate(assets.gps_lat, 4) AND locations.gps_long = truncate(assets.gps_long, 4)").
+			Joins(LeftJoinForLocations).
 			Order("assets.created_at DESC").Rows()
 	}
 	if err != nil {

handlers/asset.go

@@ -43,7 +43,8 @@ type AssetInfo struct {
 
 const (
 	// created_at field is adjusted with time_offset so the time can be shown "as UTC"
-	AssetsSelectClause = "assets.id, assets.name, assets.user_id, assets.created_at+ifnull(time_offset,0), assets.remote_id, assets.mime_type, assets.gps_lat, assets.gps_long, locations.display, assets.size, assets.mime_type, favourite_assets.asset_id is not null as f"
+	AssetsSelectClause   = "assets.id, assets.name, assets.user_id, assets.created_at+ifnull(time_offset,0), assets.remote_id, assets.mime_type, assets.gps_lat, assets.gps_long, locations.display, assets.size, assets.mime_type, favourite_assets.asset_id is not null as f"
+	LeftJoinForLocations = "left join locations ON locations.gps_lat = round(assets.gps_lat*10000-0.5)/10000.0 AND locations.gps_long = round(assets.gps_long*10000-0.5)/10000.0"
 )
 
 type AssetDeleteRequest struct {
@@ -86,16 +87,19 @@ func LoadAssetsFromRows(c *gin.Context, rows *sql.Rows) *[]AssetInfo {
 
 func AssetList(c *gin.Context, user *models.User) {
 	// Modified depends on deleted assets as well, that's why the where condition is different
-	tx := db.Instance.Table("assets").Select("max(updated_at)").Where("user_id=? AND size>0 AND thumb_size>0", user.ID)
+	tx := db.Instance.
+		Table("assets").
+		Select("max(updated_at)").
+		Where("user_id=? AND size>0 AND thumb_size>0", user.ID)
 	if isNotModified(c, tx) {
 		return
 	}
-	// TODO: For big sets maybe dynamically load asset info individually
+	// TODO: For big sets maybe dynamically load asset info individually?
 	rows, err := db.Instance.
 		Table("assets").
 		Select(AssetsSelectClause).
 		Joins("left join favourite_assets on favourite_assets.asset_id = assets.id").
-		Joins("left join locations on locations.gps_lat = truncate(assets.gps_lat, 4) and locations.gps_long = truncate(assets.gps_long, 4)").
+		Joins(LeftJoinForLocations).
 		Where("assets.user_id=? and assets.deleted=0 and assets.size>0 and assets.thumb_size>0", user.ID).Order("assets.created_at DESC").Rows()
 	if err != nil {
 		c.JSON(http.StatusInternalServerError, DBError1Response)
@@ -220,7 +224,9 @@ func AssetDelete(c *gin.Context, user *models.User) {
 			log.Printf("Asset: %d, save error %s", id, err)
 			continue
 		}
+		// TODO: Delete record better (and rely on cascaded deletes) and reinsert with same RemoteID (to stop re-uploading)?
 		db.Instance.Exec("delete from album_assets where asset_id=?", id)
+		db.Instance.Exec("delete from favourite_assets where asset_id=?", id)
 		storage := storage.StorageFrom(&asset.Bucket)
 		if storage == nil {
 			log.Printf("Asset: %d, error: storage is nil", id)

handlers/moment.go

@@ -105,7 +105,7 @@ func MomentAssets(c *gin.Context, user *models.User) {
 		Table("assets").
 		Select(AssetsSelectClause).
 		Joins("left join favourite_assets on favourite_assets.asset_id = assets.id").
-		Joins("left join locations ON locations.gps_lat = truncate(assets.gps_lat, 4) AND locations.gps_long = truncate(assets.gps_long, 4)").
+		Joins(LeftJoinForLocations).
 		Where("assets.user_id = ? and place_id in (?) and assets.deleted=0 and assets.created_at>=? and assets.created_at<=?", user.ID, strings.Split(r.Places, ","), r.Start, r.End).
 		Order("assets.created_at DESC").Rows()
 

handlers/tag.go

@@ -74,7 +74,8 @@ func TagList(c *gin.Context, user *models.User) {
 	}
 	rows, err := db.Instance.Table("assets").Select("id, mime_type, favourite, created_at, locations.gps_lat, locations.gps_long, area, city, country").
 		Where("user_id=? AND deleted=0 AND size>0 AND thumb_size>0", user.ID).
-		Joins("left join locations ON locations.gps_lat = truncate(assets.gps_lat, 4) AND locations.gps_long = truncate(assets.gps_long, 4)").Order("created_at DESC").
+		Joins(LeftJoinForLocations).
+		Order("created_at DESC").
 		Rows()
 	if err != nil {
 		c.JSON(http.StatusInternalServerError, DBError1Response)

processing/processing.go

@@ -21,6 +21,12 @@ type processingTasksElement struct {
 	task processingTask
 }
 
+type pendingTask struct {
+	assetID  uint64
+	status   string
+	recordID *uint64
+}
+
 type processingTasks []processingTasksElement
 
 var tasks = processingTasks{}
@@ -98,30 +104,43 @@ func processPending() {
 		Select("assets.id, IFNULL(processing_tasks.status, ''), processing_tasks.asset_id").
 		Where("assets.deleted=0 AND "+
 			"assets.size>0 AND "+
-			"unix_timestamp()-assets.updated_at>30 AND "+
+			db.TimestampFunc+"-assets.updated_at>30 AND "+
 			"(processing_tasks.status IS NULL OR "+
 			"  LENGTH(processing_tasks.status)-LENGTH(REPLACE(processing_tasks.status, ',', ''))+1 < ?)", len(tasks)).
 		Order("assets.created_at").Rows()
 	if err != nil {
 		log.Printf("processPending error: %v", err)
 		return
 	}
-	defer rows.Close()
+	// Create an array of temp structs to hold the fetched data
+	pendingTasks := []pendingTask{}
 	for rows.Next() {
-		asset := models.Asset{}
+		assetID := uint64(0)
 		status := ""
 		var recordId *uint64
-		if err = rows.Scan(&asset.ID, &status, &recordId); err != nil {
+		if err = rows.Scan(&assetID, &status, &recordId); err != nil {
 			log.Printf("processPending row error: %v", err)
 			break
 		}
+		pendingTasks = append(pendingTasks, pendingTask{
+			assetID:  assetID,
+			status:   status,
+			recordID: recordId,
+		})
+	}
+	rows.Close()
+	// Above was needed as sqlite3 was locking
+	for _, task := range pendingTasks {
+		asset := models.Asset{
+			ID: task.assetID,
+		}
 		if err = db.Instance.Preload("Bucket").Preload("User").First(&asset).Error; err != nil {
 			log.Printf("processPending load asset error: %v, asset: %d", err, asset.ID)
 			break
 		}
 		current := ProcessingTask{
 			AssetID: asset.ID,
-			Status:  status,
+			Status:  task.status,
 		}
 		var assetStorage storage.StorageAPI
 		if tasks.requireContent(&asset) {
@@ -142,7 +161,7 @@ func processPending() {
 		statusMap := current.statusToMap()
 		tasks.process(&asset, assetStorage, statusMap)
 		current.updateWith(statusMap)
-		if recordId == nil {
+		if task.recordID == nil {
 			// This is a new record
 			err = db.Instance.Create(&current).Error
 		} else {