diff --git a/.travis.yml b/.travis.yml index cd68e5f..1ab8b38 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,7 @@ install: true script: - make deps - gx-go rewrite - - go test -race -coverprofile=unittest.coverprofile -covermode=atomic ./... + - go test -race -coverprofile=unittest.coverprofile -covermode=atomic . after_success: diff --git a/convert.go b/convert.go new file mode 100644 index 0000000..b4dbd7c --- /dev/null +++ b/convert.go @@ -0,0 +1,173 @@ +// Package flatfs is a Datastore implementation that stores all +// objects in a two-level directory structure in the local file +// system, regardless of the hierarchy of the keys. +package flatfs + +import ( + "fmt" + "io" + "os" + "path/filepath" + "time" + + "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/query" + "github.com/jbenet/go-os-rename" +) + +func UpgradeV0toV1(path string, prefixLen int) error { + fun := Prefix(prefixLen) + err := WriteShardFunc(path, fun) + if err != nil { + return err + } + err = WriteReadme(path, fun) + if err != nil { + return err + } + return nil +} + +func DowngradeV1toV0(path string) error { + err := os.Remove(filepath.Join(path, SHARDING_FN)) + if err != nil { + return err + } + err = os.Remove(filepath.Join(path, README_FN)) + if err != nil && !os.IsNotExist(err) { + return err + } + return nil +} + +func Move(oldPath string, newPath string, out io.Writer) error { + oldDS, err := Open(oldPath, false) + if err != nil { + return fmt.Errorf("%s: %v", oldPath, err) + } + newDS, err := Open(newPath, false) + if err != nil { + return fmt.Errorf("%s: %v", newPath, err) + } + + if out != nil { + fmt.Fprintf(out, "Getting Keys...\n") + } + res, err := oldDS.Query(query.Query{KeysOnly: true}) + if err != nil { + return err + } + entries, err := res.Rest() + if err != nil { + return err + } + prog := newProgress(len(entries), out) + + if out != nil { + fmt.Fprintf(out, "Moving Keys...\n") + } + + // first move the keys + for _, e := range entries { + err := moveKey(oldDS, newDS, datastore.RawKey(e.Key)) + if err != nil { + return err + } + prog.Next() + } + + if out != nil { + fmt.Fprintf(out, "\nCleaning Up...\n") + } + + // now walk the old top-level directory + dir, err := os.Open(oldDS.path) + if err != nil { + return err + } + defer dir.Close() + names, err := dir.Readdirnames(-1) + if err != nil { + return err + } + for _, fn := range names { + if fn == "." || fn == ".." { + continue + } + oldPath := filepath.Join(oldDS.path, fn) + inf, err := os.Stat(oldPath) + if err != nil { + return err + } + if inf.IsDir() || fn == "SHARDING" || fn == "_README" { + // if we are a director or generated file just remove it + err := os.Remove(oldPath) + if err != nil { + return err + } + } else { + // else move it + newPath := filepath.Join(newDS.path, fn) + err := osrename.Rename(oldPath, newPath) + if err != nil { + return err + } + } + } + + if out != nil { + fmt.Fprintf(out, "All Done.\n") + } + + return nil +} + +func moveKey(oldDS *Datastore, newDS *Datastore, key datastore.Key) error { + _, oldPath := oldDS.encode(key) + dir, newPath := newDS.encode(key) + err := newDS.makeDirNoSync(dir) + if err != nil { + return err + } + err = osrename.Rename(oldPath, newPath) + if err != nil { + return err + } + return nil +} + +type progress struct { + total int + current int + + out io.Writer + + start time.Time +} + +func newProgress(total int, out io.Writer) *progress { + return &progress{ + total: total, + start: time.Now(), + out: out, + } +} + +func (p *progress) Next() { + p.current++ + if p.out == nil { + return + } + if p.current%10 == 0 || p.current == p.total { + fmt.Fprintf(p.out, "\r[%d / %d]", p.current, p.total) + } + + if p.current%100 == 0 || p.current == p.total { + took := time.Now().Sub(p.start) + av := took / time.Duration(p.current) + estim := av * time.Duration(p.total-p.current) + //est := strings.Split(estim.String(), ".")[0] + + fmt.Fprintf(p.out, " Approx time remaining: %s ", estim) + } +} diff --git a/convert_test.go b/convert_test.go new file mode 100644 index 0000000..dbf0e76 --- /dev/null +++ b/convert_test.go @@ -0,0 +1,90 @@ +package flatfs_test + +import ( + "bytes" + "encoding/hex" + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/ipfs/go-datastore" + "github.com/ipfs/go-ds-flatfs" + //"gx/ipfs/QmRWDav6mzWseLWeYfVd5fvUKiVe9xNH29YfMF438fG364/go-datastore/query" + + rand "github.com/dustin/randbo" +) + +func TestMove(t *testing.T) { + tempdir, cleanup := tempdir(t) + defer cleanup() + + v1dir := filepath.Join(tempdir, "v1") + err := flatfs.Create(v1dir, flatfs.Prefix(3)) + if err != nil { + t.Fatalf("Create fail: %v\n", err) + } + err = ioutil.WriteFile(filepath.Join(v1dir, "README_ALSO"), []byte("something"), 0666) + if err != nil { + t.Fatalf("WriteFile fail: %v\n", err) + } + v1, err := flatfs.Open(v1dir, false) + if err != nil { + t.Fatalf("New fail: %v\n", err) + } + + r := rand.New() + var blocks [][]byte + var keys []datastore.Key + for i := 0; i < 256; i++ { + blk := make([]byte, 1000) + r.Read(blk) + blocks = append(blocks, blk) + + key := "x" + hex.EncodeToString(blk[:8]) + keys = append(keys, datastore.NewKey(key)) + err := v1.Put(keys[i], blocks[i]) + if err != nil { + t.Fatalf("Put fail: %v\n", err) + } + } + + v2dir := filepath.Join(tempdir, "v2") + err = flatfs.Create(v2dir, flatfs.NextToLast(2)) + if err != nil { + t.Fatalf("Mkdir fail: %v\n", err) + } + flatfs.Move(v1dir, v2dir, nil) + + // This should fail if the directory is not empty + err = os.Remove(v1dir) + if err != nil { + t.Fatalf("Remove fail: %v\n", err) + } + + // Make sure the README file moved + _, err = os.Stat(filepath.Join(v2dir, "README_ALSO")) + if err != nil { + t.Fatalf(err.Error()) + } + + v2, err := flatfs.Open(v2dir, false) + if err != nil { + t.Fatalf("New fail: %v\n", err) + } + + // Sanity check, make sure we can retrieve a new key + data, err := v2.Get(keys[0]) + if err != nil { + t.Fatalf("Get fail: %v\n", err) + } + if !bytes.Equal(data.([]byte), blocks[0]) { + t.Fatalf("block context differ for key %s\n", keys[0].String()) + } + + shard := filepath.Join(v2dir, flatfs.NextToLast(2).Func()(keys[0].String())) + _, err = os.Stat(shard) + if err != nil { + t.Fatalf(err.Error()) + } +} diff --git a/flatfs/main.go b/flatfs/main.go new file mode 100644 index 0000000..2f76281 --- /dev/null +++ b/flatfs/main.go @@ -0,0 +1,96 @@ +package main + +import ( + "fmt" + "os" + "strconv" + + "github.com/ipfs/go-ds-flatfs" +) + +// To convert from the old format to a new format with a different +// sharding function use: +// flatfs upgrade blocks 5 +// flatfs create blocks-new v1/next-to-last/2 +// flatfs move blocks blocks-new +// rmdir blocks +// mv blocks-new blocks +// to do the reverse +// flatfs create blocks-new v1/prefix/5 +// flatfs move blocks blocks-new +// rmdir blocks +// mv blocks-new blocks +// flatfs downgrade blocks + +func usage() { + fmt.Fprintf(os.Stderr, "usage: %s create DIR SHARDFUN | upgrade DIR PREFIXLEN | downgrade DIR | move OLDDIR NEWDIR\n", os.Args[0]) + os.Exit(1) +} + +func fail(err error) { + fmt.Fprintf(os.Stderr, "%v\n", err) + os.Exit(1) +} + +func main() { + if len(os.Args) < 2 { + usage() + } + + switch os.Args[1] { + case "create": + if len(os.Args) != 4 { + usage() + } + dir := os.Args[2] + funStr := os.Args[3] + if funStr[0] != '/' { + if funStr[0] != 'v' { // and version if not provided + funStr = "v1/" + funStr + } + funStr = flatfs.PREFIX + funStr + } + fun, err := flatfs.ParseShardFunc(funStr) + if err != nil { + fail(err) + } + err = flatfs.Create(dir, fun) + if err != nil { + fail(err) + } + case "upgrade": + if len(os.Args) != 4 { + usage() + } + dir := os.Args[2] + prefixLen, err := strconv.Atoi(os.Args[3]) + if err != nil { + fail(err) + } + err = flatfs.UpgradeV0toV1(dir, prefixLen) + if err != nil { + fail(err) + } + case "downgrade": + if len(os.Args) != 3 { + usage() + } + dir := os.Args[2] + err := flatfs.DowngradeV1toV0(dir) + if err != nil { + fail(err) + } + case "move": + if len(os.Args) != 4 { + usage() + } + oldDir := os.Args[2] + newDir := os.Args[3] + err := flatfs.Move(oldDir, newDir, os.Stderr) + if err != nil { + fail(err) + } + default: + usage() + } +}