diff --git a/flatfs.go b/flatfs.go index 06d3ef0..af7ce79 100644 --- a/flatfs.go +++ b/flatfs.go @@ -7,7 +7,6 @@ import ( "encoding/json" "errors" "fmt" - "io/ioutil" "math" "math/rand" "os" @@ -608,7 +607,7 @@ func (fs *Datastore) Get(key datastore.Key) (value []byte, err error) { } _, path := fs.encode(key) - data, err := ioutil.ReadFile(path) + data, err := readFile(path) if err != nil { if os.IsNotExist(err) { return nil, datastore.ErrNotFound @@ -1013,7 +1012,7 @@ func (fs *Datastore) writeDiskUsageFile(du int64, doSync bool) { // readDiskUsageFile is only safe to call in Open() func (fs *Datastore) readDiskUsageFile() int64 { fpath := filepath.Join(fs.path, DiskUsageFile) - duB, err := ioutil.ReadFile(fpath) + duB, err := readFile(fpath) if err != nil { return 0 } @@ -1050,7 +1049,7 @@ func (fs *Datastore) Accuracy() string { } func (fs *Datastore) tempFile() (*os.File, error) { - file, err := ioutil.TempFile(fs.tempPath, "temp-") + file, err := tempFile(fs.tempPath, "temp-") return file, err } @@ -1093,7 +1092,7 @@ func (fs *Datastore) walk(path string, qrb *query.ResultBuilder) error { var result query.Result result.Key = key.String() if !qrb.Query.KeysOnly { - value, err := ioutil.ReadFile(filepath.Join(path, fn)) + value, err := readFile(filepath.Join(path, fn)) if err != nil { result.Error = err } else { diff --git a/go.mod b/go.mod index f7927f4..86e5ab4 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,7 @@ module github.com/ipfs/go-ds-flatfs require ( + github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 // indirect github.com/ipfs/go-datastore v0.4.4 github.com/ipfs/go-log v1.0.3 github.com/jbenet/goprocess v0.1.4 diff --git a/go.sum b/go.sum index 6f02fd7..315861c 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 h1:iW0a5ljuFxkLGPNem5Ui+KBjFJzKg4Fv2fnxe4dvzpM= +github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5/go.mod h1:Y2QMoi1vgtOIfc+6DhrMOGkLoGzqSV2rKp4Sm+opsyA= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= diff --git a/util_unix.go b/util_unix.go new file mode 100644 index 0000000..f05ac36 --- /dev/null +++ b/util_unix.go @@ -0,0 +1,16 @@ +// +build !windows + +package flatfs + +import ( + "io/ioutil" + "os" +) + +func tempFile(dir, pattern string) (f *os.File, err error) { + return ioutil.TempFile(dir, pattern) +} + +func readFile(filename string) ([]byte, error) { + return ioutil.ReadFile(filename) +} diff --git a/util_windows.go b/util_windows.go new file mode 100644 index 0000000..988fa58 --- /dev/null +++ b/util_windows.go @@ -0,0 +1,93 @@ +// +build windows + +package flatfs + +import ( + "bytes" + "io/ioutil" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "time" + + goissue34681 "github.com/alexbrainman/goissue34681" +) + +var tmpRand uint32 +var randmu sync.Mutex + +func reseed() uint32 { + return uint32(time.Now().UnixNano() + int64(os.Getpid())) +} + +func nextRandom() string { + randmu.Lock() + r := tmpRand + if r == 0 { + r = reseed() + } + r = r*1664525 + 1013904223 // constants from Numerical Recipes + tmpRand = r + randmu.Unlock() + return strconv.Itoa(int(1e9 + r%1e9))[1:] +} + +func prefixAndSuffix(pattern string) (prefix, suffix string) { + if pos := strings.LastIndex(pattern, "*"); pos != -1 { + prefix, suffix = pattern[:pos], pattern[pos+1:] + } else { + prefix = pattern + } + return +} + +func tempFile(dir, pattern string) (f *os.File, err error) { + if dir == "" { + dir = os.TempDir() + } + + prefix, suffix := prefixAndSuffix(pattern) + + nconflict := 0 + for i := 0; i < 10000; i++ { + name := filepath.Join(dir, prefix+nextRandom()+suffix) + f, err = goissue34681.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0600) + if os.IsExist(err) { + if nconflict++; nconflict > 10 { + randmu.Lock() + tmpRand = reseed() + randmu.Unlock() + } + continue + } + break + } + return +} + +func readFile(filename string) ([]byte, error) { + f, err := goissue34681.Open(filename) + if err != nil { + return nil, err + } + defer f.Close() + // It's a good but not certain bet that FileInfo will tell us exactly how much to + // read, so let's try it but be prepared for the answer to be wrong. + var n int64 = bytes.MinRead + + if fi, err := f.Stat(); err == nil { + // As initial capacity for readAll, use Size + a little extra in case Size + // is zero, and to avoid another allocation after Read has filled the + // buffer. The readAll call will read into its allocated internal buffer + // cheaply. If the size was wrong, we'll either waste some space off the end + // or reallocate as needed, but in the overwhelmingly common case we'll get + // it just right. + if size := fi.Size() + bytes.MinRead; size > n { + n = size + } + } + + return ioutil.ReadAll(f) +}