diff --git a/config/datastore.go b/config/datastore.go index 665e0364798..b1c274a63e3 100644 --- a/config/datastore.go +++ b/config/datastore.go @@ -22,6 +22,7 @@ const ( // Datastore tracks the configuration of the datastore. type Datastore struct { + DiskMinFreePercent float64 StorageMax string // in B, kB, kiB, MB, ... StorageGCWatermark int64 // in percentage to multiply on StorageMax GCPeriod string // in ns, us, ms, s, m, h diff --git a/config/init.go b/config/init.go index f5217f413f1..e2450602804 100644 --- a/config/init.go +++ b/config/init.go @@ -130,6 +130,7 @@ func addressesConfig() Addresses { // DefaultDatastoreConfig is an internal function exported to aid in testing. func DefaultDatastoreConfig() Datastore { return Datastore{ + DiskMinFreePercent: 95, StorageMax: "10GB", StorageGCWatermark: 90, // 90% GCPeriod: "1h", diff --git a/core/corerepo/gc.go b/core/corerepo/gc.go index cf89587d66f..b7869b5f877 100644 --- a/core/corerepo/gc.go +++ b/core/corerepo/gc.go @@ -4,22 +4,30 @@ import ( "bytes" "context" "errors" + "fmt" "time" - "github.com/ipfs/kubo/core" - "github.com/ipfs/kubo/gc" - "github.com/ipfs/kubo/repo" - "github.com/dustin/go-humanize" + "github.com/gammazero/fsutil/disk" "github.com/ipfs/boxo/mfs" "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log" + "github.com/ipfs/kubo/core" + "github.com/ipfs/kubo/gc" + "github.com/ipfs/kubo/repo" ) var log = logging.Logger("corerepo") var ErrMaxStorageExceeded = errors.New("maximum storage limit exceeded. Try to unpin some files") +// Datastore default values. +const ( + defaultDiskMinFreePercent = 95 + defaultStorageGCWatermark = 90 + defaultStorageMax = "10GB" +) + type GC struct { Node *core.IpfsNode Repo repo.Repo @@ -27,6 +35,8 @@ type GC struct { StorageGC uint64 SlackGB uint64 Storage uint64 + + diskMinFreePercent float64 } func NewGC(n *core.IpfsNode) (*GC, error) { @@ -40,16 +50,22 @@ func NewGC(n *core.IpfsNode) (*GC, error) { // TODO: there should be a general check for all of the cfg fields // maybe distinguish between user config file and default struct? if cfg.Datastore.StorageMax == "" { - if err := r.SetConfigKey("Datastore.StorageMax", "10GB"); err != nil { + if err := r.SetConfigKey("Datastore.StorageMax", defaultStorageMax); err != nil { return nil, err } - cfg.Datastore.StorageMax = "10GB" + cfg.Datastore.StorageMax = defaultStorageMax } if cfg.Datastore.StorageGCWatermark == 0 { - if err := r.SetConfigKey("Datastore.StorageGCWatermark", 90); err != nil { + if err := r.SetConfigKey("Datastore.StorageGCWatermark", defaultStorageGCWatermark); err != nil { + return nil, err + } + cfg.Datastore.StorageGCWatermark = defaultStorageGCWatermark + } + if cfg.Datastore.DiskMinFreePercent == 0 { + if err := r.SetConfigKey("Datastore.DiskMinFreePercent", defaultDiskMinFreePercent); err != nil { return nil, err } - cfg.Datastore.StorageGCWatermark = 90 + cfg.Datastore.DiskMinFreePercent = defaultDiskMinFreePercent } storageMax, err := humanize.ParseBytes(cfg.Datastore.StorageMax) @@ -71,6 +87,8 @@ func NewGC(n *core.IpfsNode) (*GC, error) { StorageMax: storageMax, StorageGC: storageGC, SlackGB: slackGB, + + diskMinFreePercent: cfg.Datastore.DiskMinFreePercent, }, nil } @@ -205,6 +223,14 @@ func ConditionalGC(ctx context.Context, node *core.IpfsNode, offset uint64) erro } func (gc *GC) maybeGC(ctx context.Context, offset uint64) error { + full, err := checkDiskFull(gc.Repo.Path(), gc.diskMinFreePercent) + if err != nil { + return err + } + if full { + return gc.doGC(ctx) + } + storage, err := gc.Repo.GetStorageUsage(ctx) if err != nil { return err @@ -217,11 +243,33 @@ func (gc *GC) maybeGC(ctx context.Context, offset uint64) error { // Do GC here log.Info("Watermark exceeded. Starting repo GC...") + return gc.doGC(ctx) + } - if err := GarbageCollect(gc.Node, ctx); err != nil { - return err - } - log.Infof("Repo GC done. See `ipfs repo stat` to see how much space got freed.\n") + return nil +} + +func (gc *GC) doGC(ctx context.Context) error { + if err := GarbageCollect(gc.Node, ctx); err != nil { + return err } + log.Infof("Repo GC done. See `ipfs repo stat` to see how much space got freed.\n") return nil } + +func checkDiskFull(repoPath string, diskMinFreePercent float64) (bool, error) { + if diskMinFreePercent < 0 || diskMinFreePercent > 100 { + return false, nil + } + du, err := disk.Usage(repoPath) + if err != nil { + return false, fmt.Errorf("cannot get disk usage at path %q: %w", repoPath, err) + } + + if du.Percent >= diskMinFreePercent { + log.Warnf("Disk usage CRITICAL (%.2f%%), starting repo GC", du.Percent) + return true, nil + } + return false, nil + +} diff --git a/core/corerepo/gc_test.go b/core/corerepo/gc_test.go new file mode 100644 index 00000000000..6b548e8d705 --- /dev/null +++ b/core/corerepo/gc_test.go @@ -0,0 +1,22 @@ +package corerepo + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestCheckDiskFull(t *testing.T) { + repoDir := t.TempDir() + + full, err := checkDiskFull(repoDir, 99.9) + require.NoError(t, err) + require.False(t, full) + + full, err = checkDiskFull(repoDir, 0.01) + require.NoError(t, err) + require.True(t, full) + + _, err = checkDiskFull("/no/such/directory", 90) + require.Error(t, err) +} diff --git a/go.mod b/go.mod index e4f5ce41daf..295d9663b26 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( github.com/elgris/jsondiff v0.0.0-20160530203242-765b5c24c302 github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5 github.com/fsnotify/fsnotify v1.7.0 + github.com/gammazero/fsutil v0.1.1 github.com/google/uuid v1.6.0 github.com/hashicorp/go-multierror v1.1.1 github.com/hashicorp/go-version v1.7.0 diff --git a/go.sum b/go.sum index 39ced23ab40..51f606097dd 100644 --- a/go.sum +++ b/go.sum @@ -204,6 +204,8 @@ github.com/gammazero/chanqueue v1.0.0 h1:FER/sMailGFA3DDvFooEkipAMU+3c9Bg3bheloP github.com/gammazero/chanqueue v1.0.0/go.mod h1:fMwpwEiuUgpab0sH4VHiVcEoji1pSi+EIzeG4TPeKPc= github.com/gammazero/deque v1.0.0 h1:LTmimT8H7bXkkCy6gZX7zNLtkbz4NdS2z8LZuor3j34= github.com/gammazero/deque v1.0.0/go.mod h1:iflpYvtGfM3U8S8j+sZEKIak3SAKYpA5/SQewgfXDKo= +github.com/gammazero/fsutil v0.1.1 h1:sWMlUs9BhBIqnsV77B3eS1ZAedoJhCuTmLiF/qrLNlU= +github.com/gammazero/fsutil v0.1.1/go.mod h1:HYJutEsW337gztmm4HTN4XrlQI6WRNWSOjcpcAhttME= github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps= github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=