Skip to content

Commit

Permalink
Direct IO fix for Mac
Browse files Browse the repository at this point in the history 
Summary:
O_DIRECT is not available in Mac as a flag for open. The fix is to make
use of fctl after the file is opened

Test Plan: Run the tests on mac and Linux

Reviewers: sdong

Subscribers: andrewkr, dhruba, leveldb

Differential Revision: https://reviews.facebook.net/D58665
  • Loading branch information
krad committed May 23, 2016
1 parent 99765ed commit 21f847e
Show file tree
Hide file tree
Showing 2 changed files with 40 additions and 5 deletions.
30 changes: 30 additions & 0 deletions util/env_posix.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,12 +153,22 @@ class PosixEnv : public Env {
*result = nullptr;
return IOError(fname, errno);
} else if (options.use_direct_reads && !options.use_mmap_writes) {
#ifdef OS_MACOSX
int flags = O_RDONLY;
#else
int flags = O_RDONLY | O_DIRECT;
TEST_SYNC_POINT_CALLBACK("NewSequentialFile:O_DIRECT", &flags);
#endif
int fd = open(fname.c_str(), flags, 0644);
if (fd < 0) {
return IOError(fname, errno);
}
#ifdef OS_MACOSX
if (fcntl(fd, F_NOCACHE, 1) == -1) {
close(fd);
return IOError(fname, errno);
}
#endif
std::unique_ptr<PosixDirectIOSequentialFile> file(
new PosixDirectIOSequentialFile(fname, fd));
*result = std::move(file);
Expand Down Expand Up @@ -201,8 +211,12 @@ class PosixEnv : public Env {
}
close(fd);
} else if (options.use_direct_reads) {
#ifdef OS_MACOSX
int flags = O_RDONLY;
#else
int flags = O_RDONLY | O_DIRECT;
TEST_SYNC_POINT_CALLBACK("NewRandomAccessFile:O_DIRECT", &flags);
#endif
fd = open(fname.c_str(), flags, 0644);
if (fd < 0) {
s = IOError(fname, errno);
Expand All @@ -211,6 +225,12 @@ class PosixEnv : public Env {
new PosixDirectIORandomAccessFile(fname, fd));
*result = std::move(file);
s = Status::OK();
#ifdef OS_MACOSX
if (fcntl(fd, F_NOCACHE, 1) == -1) {
close(fd);
s = IOError(fname, errno);
}
#endif
}
} else {
result->reset(new PosixRandomAccessFile(fname, fd, options));
Expand Down Expand Up @@ -245,7 +265,11 @@ class PosixEnv : public Env {
if (options.use_mmap_writes && !forceMmapOff) {
result->reset(new PosixMmapFile(fname, fd, page_size_, options));
} else if (options.use_direct_writes) {
#ifdef OS_MACOSX
int flags = O_WRONLY | O_APPEND | O_TRUNC | O_CREAT;
#else
int flags = O_WRONLY | O_APPEND | O_TRUNC | O_CREAT | O_DIRECT;
#endif
TEST_SYNC_POINT_CALLBACK("NewWritableFile:O_DIRECT", &flags);
fd = open(fname.c_str(), flags, 0644);
if (fd < 0) {
Expand All @@ -255,6 +279,12 @@ class PosixEnv : public Env {
new PosixDirectIOWritableFile(fname, fd));
*result = std::move(file);
s = Status::OK();
#ifdef OS_MACOSX
if (fcntl(fd, F_NOCACHE, 1) == -1) {
close(fd);
s = IOError(fname, errno);
}
#endif
}
} else {
// disable mmap writes
Expand Down
15 changes: 10 additions & 5 deletions util/env_test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -794,14 +794,15 @@ TEST_P(EnvPosixTestWithParam, InvalidateCache) {
// Create file.
{
unique_ptr<WritableFile> wfile;
#ifndef OS_MACOSX
if (soptions.use_direct_writes) {
rocksdb::SyncPoint::GetInstance()->SetCallBack(
"NewWritableFile:O_DIRECT", [&](void* arg) {
int* val = static_cast<int*>(arg);
*val &= ~O_DIRECT;
});
}

#endif
ASSERT_OK(env_->NewWritableFile(fname, &wfile, soptions));
ASSERT_OK(wfile.get()->Append(slice));
ASSERT_OK(wfile.get()->InvalidateCache(0, 0));
Expand All @@ -813,14 +814,15 @@ TEST_P(EnvPosixTestWithParam, InvalidateCache) {
unique_ptr<RandomAccessFile> file;
char scratch[kSectorSize];
Slice result;
#ifndef OS_MACOSX
if (soptions.use_direct_reads) {
rocksdb::SyncPoint::GetInstance()->SetCallBack(
"NewRandomAccessFile:O_DIRECT", [&](void* arg) {
int* val = static_cast<int*>(arg);
*val &= ~O_DIRECT;
});
}

#endif
ASSERT_OK(env_->NewRandomAccessFile(fname, &file, soptions));
ASSERT_OK(file.get()->Read(0, kSectorSize, &result, scratch));
ASSERT_EQ(memcmp(scratch, data.get(), kSectorSize), 0);
Expand All @@ -833,14 +835,15 @@ TEST_P(EnvPosixTestWithParam, InvalidateCache) {
unique_ptr<SequentialFile> file;
char scratch[kSectorSize];
Slice result;
#ifndef OS_MACOSX
if (soptions.use_direct_reads) {
rocksdb::SyncPoint::GetInstance()->SetCallBack(
"NewSequentialFile:O_DIRECT", [&](void* arg) {
int* val = static_cast<int*>(arg);
*val &= ~O_DIRECT;
});
}

#endif
ASSERT_OK(env_->NewSequentialFile(fname, &file, soptions));
ASSERT_OK(file.get()->Read(kSectorSize, &result, scratch));
ASSERT_EQ(memcmp(scratch, data.get(), kSectorSize), 0);
Expand Down Expand Up @@ -978,14 +981,15 @@ TEST_P(EnvPosixTestWithParam, Preallocation) {
unique_ptr<WritableFile> srcfile;
EnvOptions soptions;
soptions.use_direct_reads = soptions.use_direct_writes = directio;
#ifndef OS_MACOSX
if (soptions.use_direct_writes) {
rocksdb::SyncPoint::GetInstance()->SetCallBack(
"NewWritableFile:O_DIRECT", [&](void* arg) {
int* val = static_cast<int*>(arg);
*val &= ~O_DIRECT;
});
}

#endif
ASSERT_OK(env_->NewWritableFile(src, &srcfile, soptions));
srcfile->SetPreallocationBlockSize(1024 * 1024);

Expand Down Expand Up @@ -1041,14 +1045,15 @@ TEST_P(EnvPosixTestWithParam, ConsistentChildrenAttributes) {
oss << test::TmpDir(env_) << "/testfile_" << i;
const std::string path = oss.str();
unique_ptr<WritableFile> file;
#ifndef OS_MACOSX
if (soptions.use_direct_writes) {
rocksdb::SyncPoint::GetInstance()->SetCallBack(
"NewWritableFile:O_DIRECT", [&](void* arg) {
int* val = static_cast<int*>(arg);
*val &= ~O_DIRECT;
});
}

#endif
ASSERT_OK(env_->NewWritableFile(path, &file, soptions));
auto buf_ptr = NewAligned(data.size(), 'T');
Slice buf(buf_ptr.get(), data.size());
Expand Down

0 comments on commit 21f847e

Please sign in to comment.