chore: trigger gc if disk at minimum free space limit

config/datastore.go

@@ -22,6 +22,7 @@ const (
 
 // Datastore tracks the configuration of the datastore.
 type Datastore struct {
+	DiskMinFreePercent float64
 	StorageMax         string // in B, kB, kiB, MB, ...
 	StorageGCWatermark int64  // in percentage to multiply on StorageMax
 	GCPeriod           string // in ns, us, ms, s, m, h

config/init.go

@@ -130,6 +130,7 @@ func addressesConfig() Addresses {
 // DefaultDatastoreConfig is an internal function exported to aid in testing.
 func DefaultDatastoreConfig() Datastore {
 	return Datastore{
+		DiskMinFreePercent: 95,
 		StorageMax:         "10GB",
 		StorageGCWatermark: 90, // 90%
 		GCPeriod:           "1h",

core/corerepo/gc.go

@@ -4,29 +4,39 @@ import (
 	"bytes"
 	"context"
 	"errors"
+	"fmt"
 	"time"
 
-	"github.com/ipfs/kubo/core"
-	"github.com/ipfs/kubo/gc"
-	"github.com/ipfs/kubo/repo"
-
 	"github.com/dustin/go-humanize"
+	"github.com/gammazero/fsutil/disk"
 	"github.com/ipfs/boxo/mfs"
 	"github.com/ipfs/go-cid"
 	logging "github.com/ipfs/go-log"
+	"github.com/ipfs/kubo/core"
+	"github.com/ipfs/kubo/gc"
+	"github.com/ipfs/kubo/repo"
 )
 
 var log = logging.Logger("corerepo")
 
 var ErrMaxStorageExceeded = errors.New("maximum storage limit exceeded. Try to unpin some files")
 
+// Datastore default values.
+const (
+	defaultDiskMinFreePercent = 95
+	defaultStorageGCWatermark = 90
+	defaultStorageMax         = "10GB"
+)
+
 type GC struct {
 	Node       *core.IpfsNode
 	Repo       repo.Repo
 	StorageMax uint64
 	StorageGC  uint64
 	SlackGB    uint64
 	Storage    uint64
+
+	diskMinFreePercent float64
 }
 
 func NewGC(n *core.IpfsNode) (*GC, error) {
@@ -40,16 +50,22 @@ func NewGC(n *core.IpfsNode) (*GC, error) {
 	// TODO: there should be a general check for all of the cfg fields
 	// maybe distinguish between user config file and default struct?
 	if cfg.Datastore.StorageMax == "" {
-		if err := r.SetConfigKey("Datastore.StorageMax", "10GB"); err != nil {
+		if err := r.SetConfigKey("Datastore.StorageMax", defaultStorageMax); err != nil {
 			return nil, err
 		}
-		cfg.Datastore.StorageMax = "10GB"
+		cfg.Datastore.StorageMax = defaultStorageMax
 	}
 	if cfg.Datastore.StorageGCWatermark == 0 {
-		if err := r.SetConfigKey("Datastore.StorageGCWatermark", 90); err != nil {
+		if err := r.SetConfigKey("Datastore.StorageGCWatermark", defaultStorageGCWatermark); err != nil {
+			return nil, err
+		}
+		cfg.Datastore.StorageGCWatermark = defaultStorageGCWatermark
+	}
+	if cfg.Datastore.DiskMinFreePercent == 0 {
+		if err := r.SetConfigKey("Datastore.DiskMinFreePercent", defaultDiskMinFreePercent); err != nil {
 			return nil, err
 		}
-		cfg.Datastore.StorageGCWatermark = 90
+		cfg.Datastore.DiskMinFreePercent = defaultDiskMinFreePercent
 	}
 
 	storageMax, err := humanize.ParseBytes(cfg.Datastore.StorageMax)
@@ -71,6 +87,8 @@ func NewGC(n *core.IpfsNode) (*GC, error) {
 		StorageMax: storageMax,
 		StorageGC:  storageGC,
 		SlackGB:    slackGB,
+
+		diskMinFreePercent: cfg.Datastore.DiskMinFreePercent,
 	}, nil
 }
 
@@ -205,6 +223,14 @@ func ConditionalGC(ctx context.Context, node *core.IpfsNode, offset uint64) erro
 }
 
 func (gc *GC) maybeGC(ctx context.Context, offset uint64) error {
+	full, err := checkDiskFull(gc.Repo.Path(), gc.diskMinFreePercent)
+	if err != nil {
+		return err
+	}
+	if full {
+		return gc.doGC(ctx)
+	}
+
 	storage, err := gc.Repo.GetStorageUsage(ctx)
 	if err != nil {
 		return err
@@ -217,11 +243,33 @@ func (gc *GC) maybeGC(ctx context.Context, offset uint64) error {
 
 		// Do GC here
 		log.Info("Watermark exceeded. Starting repo GC...")
+		return gc.doGC(ctx)
+	}
 
-		if err := GarbageCollect(gc.Node, ctx); err != nil {
-			return err
-		}
-		log.Infof("Repo GC done. See `ipfs repo stat` to see how much space got freed.\n")
+	return nil
+}
+
+func (gc *GC) doGC(ctx context.Context) error {
+	if err := GarbageCollect(gc.Node, ctx); err != nil {
+		return err
 	}
+	log.Infof("Repo GC done. See `ipfs repo stat` to see how much space got freed.\n")
 	return nil
 }
+
+func checkDiskFull(repoPath string, diskMinFreePercent float64) (bool, error) {
+	if diskMinFreePercent < 0 || diskMinFreePercent > 100 {
+		return false, nil
+	}
+	du, err := disk.Usage(repoPath)
+	if err != nil {
+		return false, fmt.Errorf("cannot get disk usage at path %q: %w", repoPath, err)
+	}
+
+	if du.Percent >= diskMinFreePercent {
+		log.Warnf("Disk usage CRITICAL (%.2f%%), starting repo GC", du.Percent)
+		return true, nil
+	}
+	return false, nil
+
+}

core/corerepo/gc_test.go

@@ -0,0 +1,22 @@
+package corerepo
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestCheckDiskFull(t *testing.T) {
+	repoDir := t.TempDir()
+
+	full, err := checkDiskFull(repoDir, 99.9)
+	require.NoError(t, err)
+	require.False(t, full)
+
+	full, err = checkDiskFull(repoDir, 0.01)
+	require.NoError(t, err)
+	require.True(t, full)
+
+	_, err = checkDiskFull("/no/such/directory", 90)
+	require.Error(t, err)
+}

go.mod

@@ -17,6 +17,7 @@ require (
 	github.com/elgris/jsondiff v0.0.0-20160530203242-765b5c24c302
 	github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5
 	github.com/fsnotify/fsnotify v1.7.0
+	github.com/gammazero/fsutil v0.1.1
 	github.com/google/uuid v1.6.0
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/hashicorp/go-version v1.7.0

go.sum

@@ -204,6 +204,8 @@ github.com/gammazero/chanqueue v1.0.0 h1:FER/sMailGFA3DDvFooEkipAMU+3c9Bg3bheloP
 github.com/gammazero/chanqueue v1.0.0/go.mod h1:fMwpwEiuUgpab0sH4VHiVcEoji1pSi+EIzeG4TPeKPc=
 github.com/gammazero/deque v1.0.0 h1:LTmimT8H7bXkkCy6gZX7zNLtkbz4NdS2z8LZuor3j34=
 github.com/gammazero/deque v1.0.0/go.mod h1:iflpYvtGfM3U8S8j+sZEKIak3SAKYpA5/SQewgfXDKo=
+github.com/gammazero/fsutil v0.1.1 h1:sWMlUs9BhBIqnsV77B3eS1ZAedoJhCuTmLiF/qrLNlU=
+github.com/gammazero/fsutil v0.1.1/go.mod h1:HYJutEsW337gztmm4HTN4XrlQI6WRNWSOjcpcAhttME=
 github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps=
 github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=