diff --git a/mmap_stub.go b/mmap_stub.go new file mode 100644 index 0000000..06883ce --- /dev/null +++ b/mmap_stub.go @@ -0,0 +1,25 @@ +//go:build appengine || plan9 || js || wasip1 || wasi + +package maxminddb + +import ( + "errors" +) + +type mmapUnsupportedError struct{} + +func (mmapUnsupportedError) Error() string { + return "mmap is not supported on this platform" +} + +func (mmapUnsupportedError) Is(target error) bool { + return target == errors.ErrUnsupported +} + +func mmap(_, _ int) (data []byte, err error) { + return nil, mmapUnsupportedError{} +} + +func munmap(_ []byte) (err error) { + return mmapUnsupportedError{} +} diff --git a/mmap_unix.go b/mmap_unix.go index 48b2e40..3e2235c 100644 --- a/mmap_unix.go +++ b/mmap_unix.go @@ -4,13 +4,36 @@ package maxminddb import ( + "errors" + "os" + "golang.org/x/sys/unix" ) +type mmapENODEVError struct{} + +func (mmapENODEVError) Error() string { + return "mmap: the underlying filesystem of the specified file does not support memory mapping" +} + +func (mmapENODEVError) Is(target error) bool { + return target == errors.ErrUnsupported +} + func mmap(fd, length int) (data []byte, err error) { - return unix.Mmap(fd, 0, length, unix.PROT_READ, unix.MAP_SHARED) + data, err = unix.Mmap(fd, 0, length, unix.PROT_READ, unix.MAP_SHARED) + if err != nil { + if err == unix.ENODEV { + return nil, mmapENODEVError{} + } + return nil, os.NewSyscallError("mmap", err) + } + return data, nil } func munmap(b []byte) (err error) { - return unix.Munmap(b) + if err = unix.Munmap(b); err != nil { + return os.NewSyscallError("munmap", err) + } + return nil } diff --git a/reader.go b/reader.go index 6bc4d47..ae18794 100644 --- a/reader.go +++ b/reader.go @@ -5,8 +5,11 @@ import ( "bytes" "errors" "fmt" + "io" "net/netip" + "os" "reflect" + "runtime" ) const dataSectionSeparatorSize = 16 @@ -45,6 +48,78 @@ type Metadata struct { RecordSize uint `maxminddb:"record_size"` } +// Open takes a string path to a MaxMind DB file and returns a Reader +// structure or an error. The database file is opened using a memory map +// on supported platforms. On platforms without memory map support, such +// as WebAssembly or Google App Engine, or if the memory map attempt fails +// due to lack of support from the filesystem, the database is loaded into memory. +// Use the Close method on the Reader object to return the resources to the system. +func Open(file string) (*Reader, error) { + mapFile, err := os.Open(file) + if err != nil { + return nil, err + } + defer mapFile.Close() + + stats, err := mapFile.Stat() + if err != nil { + return nil, err + } + + size64 := stats.Size() + // mmapping an empty file returns -EINVAL on Unix platforms, + // and ERROR_FILE_INVALID on Windows. + if size64 == 0 { + return nil, errors.New("file is empty") + } + + size := int(size64) + // Check for overflow. + if int64(size) != size64 { + return nil, errors.New("file too large") + } + + data, err := mmap(int(mapFile.Fd()), size) + if err != nil { + if errors.Is(err, errors.ErrUnsupported) { + data, err = openFallback(mapFile, size) + if err != nil { + return nil, err + } + return FromBytes(data) + } + return nil, err + } + + reader, err := FromBytes(data) + if err != nil { + _ = munmap(data) + return nil, err + } + + reader.hasMappedFile = true + runtime.SetFinalizer(reader, (*Reader).Close) + return reader, nil +} + +func openFallback(f *os.File, size int) (data []byte, err error) { + data = make([]byte, size) + _, err = io.ReadFull(f, data) + return data, err +} + +// Close returns the resources used by the database to the system. +func (r *Reader) Close() error { + var err error + if r.hasMappedFile { + runtime.SetFinalizer(r, nil) + r.hasMappedFile = false + err = munmap(r.buffer) + } + r.buffer = nil + return err +} + // FromBytes takes a byte slice corresponding to a MaxMind DB file and returns // a Reader structure or an error. func FromBytes(buffer []byte) (*Reader, error) { diff --git a/reader_memory.go b/reader_memory.go deleted file mode 100644 index 4ebb347..0000000 --- a/reader_memory.go +++ /dev/null @@ -1,26 +0,0 @@ -//go:build appengine || plan9 || js || wasip1 || wasi -// +build appengine plan9 js wasip1 wasi - -package maxminddb - -import "io/ioutil" - -// Open takes a string path to a MaxMind DB file and returns a Reader -// structure or an error. The database file is opened using a memory map -// on supported platforms. On platforms without memory map support, such -// as WebAssembly or Google App Engine, the database is loaded into memory. -// Use the Close method on the Reader object to return the resources to the system. -func Open(file string) (*Reader, error) { - bytes, err := ioutil.ReadFile(file) - if err != nil { - return nil, err - } - - return FromBytes(bytes) -} - -// Close returns the resources used by the database to the system. -func (r *Reader) Close() error { - r.buffer = nil - return nil -} diff --git a/reader_mmap.go b/reader_mmap.go deleted file mode 100644 index 1d08301..0000000 --- a/reader_mmap.go +++ /dev/null @@ -1,64 +0,0 @@ -//go:build !appengine && !plan9 && !js && !wasip1 && !wasi -// +build !appengine,!plan9,!js,!wasip1,!wasi - -package maxminddb - -import ( - "os" - "runtime" -) - -// Open takes a string path to a MaxMind DB file and returns a Reader -// structure or an error. The database file is opened using a memory map -// on supported platforms. On platforms without memory map support, such -// as WebAssembly or Google App Engine, the database is loaded into memory. -// Use the Close method on the Reader object to return the resources to the system. -func Open(file string) (*Reader, error) { - mapFile, err := os.Open(file) - if err != nil { - _ = mapFile.Close() - return nil, err - } - - stats, err := mapFile.Stat() - if err != nil { - _ = mapFile.Close() - return nil, err - } - - fileSize := int(stats.Size()) - mmap, err := mmap(int(mapFile.Fd()), fileSize) - if err != nil { - _ = mapFile.Close() - return nil, err - } - - if err := mapFile.Close(); err != nil { - //nolint:errcheck // we prefer to return the original error - munmap(mmap) - return nil, err - } - - reader, err := FromBytes(mmap) - if err != nil { - //nolint:errcheck // we prefer to return the original error - munmap(mmap) - return nil, err - } - - reader.hasMappedFile = true - runtime.SetFinalizer(reader, (*Reader).Close) - return reader, nil -} - -// Close returns the resources used by the database to the system. -func (r *Reader) Close() error { - var err error - if r.hasMappedFile { - runtime.SetFinalizer(r, nil) - r.hasMappedFile = false - err = munmap(r.buffer) - } - r.buffer = nil - return err -}