Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(cmd/fsck): refactor and add path for data check #5019

Open
wants to merge 5 commits into
base: main
Choose a base branch
from
+335 −224
Open

feat(cmd/fsck): refactor and add path for data check #5019

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
8b585ce
feat: refactor fsck
jiefenghuang Jul 23, 2024
141bf3f
fix
jiefenghuang Jul 23, 2024
050d039
fix
jiefenghuang Jul 23, 2024
37795de
remove unused hint
jiefenghuang Jul 23, 2024
98ffd33
set recursive as default
jiefenghuang Jul 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
271 changes: 151 additions & 120 deletions cmd/fsck.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,172 +51,203 @@ $ juicefs fsck redis://localhost --path /d1/d2 --repair
# recursively check
$ juicefs fsck redis://localhost --path /d1/d2 --recursive`,
Flags: []cli.Flag{
&cli.StringFlag{
Name: "check",
Value: "both",
Usage: `check "meta", "data" or "both"`,
},
&cli.StringFlag{
Name: "path",
Value: "/",
Usage: "absolute path within JuiceFS to check",
},
&cli.BoolFlag{
Name: "repair",
Usage: "repair specified path if it's broken",
},
&cli.BoolFlag{
Name: "recursive",
Aliases: []string{"r"},
Value: true,
Usage: "recursively check or repair",
},
&cli.BoolFlag{
Name: "repair",
Usage: "repair specified path if it's broken (Note: repair attr and stat only)",
},
&cli.BoolFlag{
Name: "sync-dir-stat",
Usage: "sync stat of all directories, even if they are existed and not broken (NOTE: it may take a long time for huge trees)",
},
&cli.UintFlag{
Name: "threads",
Value: 20,
Usage: "number of concurrent threads",
},
},
}
}

const (
checkMetaFlag = 1 << iota
checkDataFlag
)

func fsck(ctx *cli.Context) error {
setup(ctx, 1)
removePassword(ctx.Args().Get(0))

var flag int
switch ctx.String("check") {
case "meta":
flag = checkMetaFlag
case "data":
flag = checkDataFlag
case "both":
flag = checkMetaFlag | checkDataFlag
default:
logger.Fatalf("invalid check flag: %s", ctx.String("check"))
}

if ctx.Bool("repair") && ctx.String("path") == "" {
logger.Fatalf("Please provide the path to repair with `--path` option")
}
removePassword(ctx.Args().Get(0))

path := ctx.String("path")
if path == "" {
path = "/"
}
if !strings.HasPrefix(path, "/") {
logger.Fatalf("File path should be the absolute path within JuiceFS")
}
defer logger.Infof("Check %s [%s] finished, check log for more details", path, ctx.String("check"))

m := meta.NewClient(ctx.Args().Get(0), nil)
format, err := m.Load(true)
if err != nil {
logger.Fatalf("load setting: %s", err)
}
var c = meta.NewContext(0, 0, []uint32{0})
if p := ctx.String("path"); p != "" {
if !strings.HasPrefix(p, "/") {
logger.Fatalf("File path should be the absolute path within JuiceFS")
}
return m.Check(c, p, ctx.Bool("repair"), ctx.Bool("recursive"), ctx.Bool("sync-dir-stat"))
}

chunkConf := chunk.Config{
BlockSize: format.BlockSize * 1024,
Compress: format.Compression,
GetTimeout: time.Second * 60,
PutTimeout: time.Second * 60,
MaxUpload: 20,
BufferSize: 300 << 20,
CacheDir: "memory",
if ctx.Bool("sync-dir-stat") && !format.DirStats {
logger.Warn("dir stats is disabled, flag '--sync-dir-stat' will be ignored")
}

blob, err := createStorage(*format)
if err != nil {
logger.Fatalf("object storage: %s", err)
}
logger.Infof("Data use %s", blob)
blob = object.WithPrefix(blob, "chunks/")
objs, err := osync.ListAll(blob, "", "", "", true)
if err != nil {
logger.Fatalf("list all blocks: %s", err)
threads := ctx.Uint("threads")
if threads == 0 {
threads = 20
logger.Warnf("threads number set to %d", threads)
}

// Find all blocks in object storage
var c = meta.NewContext(0, 0, []uint32{0})
progress := utils.NewProgress(false)
blockDSpin := progress.AddDoubleSpinner("Found blocks")
var blocks = make(map[string]int64)
for obj := range objs {
if obj == nil {
break // failed listing
}
if obj.IsDir() {
continue
}

logger.Debugf("found block %s", obj.Key())
parts := strings.Split(obj.Key(), "/")
if len(parts) != 3 {
continue
}
name := parts[2]
blocks[name] = obj.Size()
blockDSpin.IncrInt64(obj.Size())
}
blockDSpin.Done()
if progress.Quiet {
c, b := blockDSpin.Current()
logger.Infof("Found %d blocks (%d bytes)", c, b)
}

// List all slices in metadata engine
sliceCSpin := progress.AddCountSpinner("Listed slices")
slices := make(map[meta.Ino][]meta.Slice)
r := m.ListSlices(c, slices, false, sliceCSpin.Increment)
if r != 0 {
logger.Fatalf("list all slices: %s", r)
}
sliceCSpin.Done()
delfilesSpin := progress.AddCountSpinner("Deleted files")
delfiles := make(map[meta.Ino]bool)
err = m.ScanDeletedObject(c, nil, nil, nil, func(ino meta.Ino, size uint64, ts int64) (clean bool, err error) {
delfilesSpin.Increment()
delfiles[ino] = true
return false, nil
})
if err != nil {
logger.Warnf("scan deleted objects: %s", err)
var metaChecker func(meta.Ino, string, *meta.Attr) error
if flag&checkMetaFlag != 0 {
metaChecker = m.GetMetaChecker(c, ctx.Bool("repair"), ctx.Bool("sync-dir-stat"))
}
delfilesSpin.Done()

// Scan all slices to find lost blocks
skippedSlices := progress.AddCountSpinner("Skipped slices")
sliceCBar := progress.AddCountBar("Scanned slices", sliceCSpin.Current())
sliceBSpin := progress.AddByteSpinner("Scanned slices")
lostDSpin := progress.AddDoubleSpinner("Lost blocks")
var dataChecker func(meta.Ino, []meta.Slice) error
brokens := make(map[meta.Ino]string)
for inode, ss := range slices {
if delfiles[inode] {
skippedSlices.IncrBy(len(ss))
continue
if flag&checkDataFlag != 0 {
sliceCBar := progress.AddCountBar("Scanned slices", 0)
sliceBSpin := progress.AddByteSpinner("Scanned slices")
lostDSpin := progress.AddDoubleSpinner("Lost blocks")
defer func() {
if progress.Quiet {
logger.Infof("Used by %d slices (%d bytes)", sliceCBar.Current(), sliceBSpin.Current())
}
if lc, lb := lostDSpin.Current(); lc > 0 {
msg := fmt.Sprintf("%d objects are lost (%d bytes), %d broken files:\n", lc, lb, len(brokens))
msg += fmt.Sprintf("%13s: PATH\n", "INODE")
var fileList []string
for i, p := range brokens {
fileList = append(fileList, fmt.Sprintf("%13d: %s", i, p))
}
sort.Strings(fileList)
msg += strings.Join(fileList, "\n")
logger.Error(msg)
}
}()

chunkConf := chunk.Config{
BlockSize: format.BlockSize * 1024,
Compress: format.Compression,
GetTimeout: time.Second * 60,
PutTimeout: time.Second * 60,
MaxUpload: 20,
BufferSize: 300 << 20,
CacheDir: "memory",
}
for _, s := range ss {
n := (s.Size - 1) / uint32(chunkConf.BlockSize)
for i := uint32(0); i <= n; i++ {
sz := chunkConf.BlockSize
if i == n {
sz = int(s.Size) - int(i)*chunkConf.BlockSize

blob, err := createStorage(*format)
if err != nil {
logger.Fatalf("object storage: %s", err)
}
logger.Infof("Data use %s", blob)
blob = object.WithPrefix(blob, "chunks/")

var blocks = make(map[string]int64)
if path != "/" {
objs, err := osync.ListAll(blob, "", "", "", true)
if err != nil {
logger.Fatalf("list all blocks: %s", err)
}

// Find all blocks in object storage
blockDSpin := progress.AddDoubleSpinner("Found blocks")
for obj := range objs {
if obj == nil {
break // failed listing
}
if obj.IsDir() {
continue
}

parts := strings.Split(obj.Key(), "/")
if len(parts) != 3 {
continue
}
key := fmt.Sprintf("%d_%d_%d", s.Id, i, sz)
if _, ok := blocks[key]; !ok {
var objKey string
if format.HashPrefix {
objKey = fmt.Sprintf("%02X/%v/%s", s.Id%256, s.Id/1000/1000, key)
} else {
objKey = fmt.Sprintf("%v/%v/%s", s.Id/1000/1000, s.Id/1000, key)
name := parts[2]
blocks[name] = obj.Size()
blockDSpin.IncrInt64(obj.Size())
}
blockDSpin.Done()
}

dataChecker = func(inode meta.Ino, ss []meta.Slice) error {
sliceCBar.IncrTotal(int64(len(ss)))
for _, s := range ss {
n := (s.Size - 1) / uint32(chunkConf.BlockSize)
for i := uint32(0); i <= n; i++ {
sz := chunkConf.BlockSize
if i == n {
sz = int(s.Size) - int(i)*chunkConf.BlockSize
}
if _, err := blob.Head(objKey); err != nil {
if _, ok := brokens[inode]; !ok {
if ps := m.GetPaths(meta.Background, inode); len(ps) > 0 {
brokens[inode] = ps[0]
} else {
brokens[inode] = fmt.Sprintf("inode:%d", inode)
key := fmt.Sprintf("%d_%d_%d", s.Id, i, sz)
if _, ok := blocks[key]; !ok {
var objKey string
if format.HashPrefix {
objKey = fmt.Sprintf("%02X/%v/%s", s.Id%256, s.Id/1000/1000, key)
} else {
objKey = fmt.Sprintf("%v/%v/%s", s.Id/1000/1000, s.Id/1000, key)
}
if _, err := blob.Head(objKey); err != nil {
if _, ok := brokens[inode]; !ok {
if ps := m.GetPaths(meta.Background, inode); len(ps) > 0 {
brokens[inode] = ps[0]
} else {
brokens[inode] = fmt.Sprintf("inode:%d", inode)
}
}
logger.Errorf("can't find block %s for file %s: %s", objKey, brokens[inode], err)
lostDSpin.IncrInt64(int64(sz))
}
logger.Errorf("can't find block %s for file %s: %s", objKey, brokens[inode], err)
lostDSpin.IncrInt64(int64(sz))
}
}
sliceCBar.Increment()
sliceBSpin.IncrInt64(int64(s.Size))
}
sliceCBar.Increment()
sliceBSpin.IncrInt64(int64(s.Size))
}
}
progress.Done()
if progress.Quiet {
logger.Infof("Used by %d slices (%d bytes)", sliceCBar.Current(), sliceBSpin.Current())
}
if lc, lb := lostDSpin.Current(); lc > 0 {
msg := fmt.Sprintf("%d objects are lost (%d bytes), %d broken files:\n", lc, lb, len(brokens))
msg += fmt.Sprintf("%13s: PATH\n", "INODE")
var fileList []string
for i, p := range brokens {
fileList = append(fileList, fmt.Sprintf("%13d: %s", i, p))
return nil
}
sort.Strings(fileList)
msg += strings.Join(fileList, "\n")
logger.Fatal(msg)
}

return nil
err = m.Check(c, path, ctx.Bool("recursive"), threads, metaChecker, dataChecker, progress)
progress.Done()
return err
}
Loading
Loading