Skip to content

Commit fa8879c

Browse files
vinser52vinser52
authored
NUMA bindigs support for private memory (#82)
1 parent ab98751 commit fa8879c

14 files changed

+236
-87
lines changed

‎cachelib/allocator/CMakeLists.txt

+1
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ add_library (cachelib_allocator
5555
PoolOptimizeStrategy.cpp
5656
PoolRebalancer.cpp
5757
PoolResizer.cpp
58+
PrivateMemoryManager.cpp
5859
RebalanceStrategy.cpp
5960
SlabReleaseStats.cpp
6061
TempShmMapping.cpp

‎cachelib/allocator/CacheAllocator-inl.h

+31-11
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,9 @@ CacheAllocator<CacheTrait>::CacheAllocator(
6060
tempShm_(type == InitMemType::kNone && isOnShm_
6161
? std::make_unique<TempShmMapping>(config_.getCacheSize())
6262
: nullptr),
63+
privMemManager_(type == InitMemType::kNone && !isOnShm_
64+
? std::make_unique<PrivateMemoryManager>()
65+
: nullptr),
6366
shmManager_(type != InitMemType::kNone
6467
? std::make_unique<ShmManager>(config_.cacheDir,
6568
config_.isUsingPosixShm())
@@ -123,6 +126,16 @@ ShmSegmentOpts CacheAllocator<CacheTrait>::createShmCacheOpts(TierId tid) {
123126
return opts;
124127
}
125128

129+
template <typename CacheTrait>
130+
PrivateSegmentOpts CacheAllocator<CacheTrait>::createPrivateSegmentOpts(TierId tid) {
131+
PrivateSegmentOpts opts;
132+
opts.alignment = sizeof(Slab);
133+
auto memoryTierConfigs = config_.getMemoryTierConfigs();
134+
opts.memBindNumaNodes = memoryTierConfigs[tid].getMemBind();
135+
136+
return opts;
137+
}
138+
126139
template <typename CacheTrait>
127140
size_t CacheAllocator<CacheTrait>::memoryTierSize(TierId tid) const {
128141
auto partitions = std::accumulate(memoryTierConfigs.begin(), memoryTierConfigs.end(), 0UL,
@@ -134,21 +147,18 @@ size_t CacheAllocator<CacheTrait>::memoryTierSize(TierId tid) const {
134147
}
135148

136149
template <typename CacheTrait>
137-
std::vector<std::unique_ptr<MemoryAllocator>>
138-
CacheAllocator<CacheTrait>::createPrivateAllocator() {
139-
std::vector<std::unique_ptr<MemoryAllocator>> allocators;
140-
150+
std::unique_ptr<MemoryAllocator>
151+
CacheAllocator<CacheTrait>::createPrivateAllocator(TierId tid) {
141152
if (isOnShm_)
142-
allocators.emplace_back(std::make_unique<MemoryAllocator>(
153+
return std::make_unique<MemoryAllocator>(
143154
getAllocatorConfig(config_),
144155
tempShm_->getAddr(),
145-
config_.getCacheSize()));
156+
memoryTierSize(tid));
146157
else
147-
allocators.emplace_back(std::make_unique<MemoryAllocator>(
158+
return std::make_unique<MemoryAllocator>(
148159
getAllocatorConfig(config_),
149-
config_.getCacheSize()));
150-
151-
return allocators;
160+
privMemManager_->createMapping(config_.size, createPrivateSegmentOpts(tid)),
161+
memoryTierSize(tid));
152162
}
153163

154164
template <typename CacheTrait>
@@ -177,6 +187,16 @@ CacheAllocator<CacheTrait>::restoreMemoryAllocator(TierId tid) {
177187
config_.disableFullCoredump);
178188
}
179189

190+
template <typename CacheTrait>
191+
std::vector<std::unique_ptr<MemoryAllocator>>
192+
CacheAllocator<CacheTrait>::createPrivateAllocators() {
193+
std::vector<std::unique_ptr<MemoryAllocator>> allocators;
194+
for (int tid = 0; tid < getNumTiers(); tid++) {
195+
allocators.emplace_back(createPrivateAllocator(tid));
196+
}
197+
return allocators;
198+
}
199+
180200
template <typename CacheTrait>
181201
std::vector<std::unique_ptr<MemoryAllocator>>
182202
CacheAllocator<CacheTrait>::createAllocators() {
@@ -310,7 +330,7 @@ std::vector<std::unique_ptr<MemoryAllocator>>
310330
CacheAllocator<CacheTrait>::initAllocator(
311331
InitMemType type) {
312332
if (type == InitMemType::kNone) {
313-
return createPrivateAllocator();
333+
return createPrivateAllocators();
314334
} else if (type == InitMemType::kMemNew) {
315335
return createAllocators();
316336
} else if (type == InitMemType::kMemAttach) {

‎cachelib/allocator/CacheAllocator.h

+6-1
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@
6060
#include "cachelib/allocator/PoolOptimizer.h"
6161
#include "cachelib/allocator/PoolRebalancer.h"
6262
#include "cachelib/allocator/PoolResizer.h"
63+
#include "cachelib/allocator/PrivateMemoryManager.h"
6364
#include "cachelib/allocator/ReadOnlySharedCacheView.h"
6465
#include "cachelib/allocator/Reaper.h"
6566
#include "cachelib/allocator/RebalanceStrategy.h"
@@ -2155,6 +2156,8 @@ auto& mmContainer = getMMContainer(tid, pid, cid);
21552156
std::chrono::seconds timeout = std::chrono::seconds{0});
21562157

21572158
ShmSegmentOpts createShmCacheOpts(TierId tid);
2159+
PrivateSegmentOpts createPrivateSegmentOpts(TierId tid);
2160+
std::unique_ptr<MemoryAllocator> createPrivateAllocator(TierId tid);
21582161
std::unique_ptr<MemoryAllocator> createNewMemoryAllocator(TierId tid);
21592162
std::unique_ptr<MemoryAllocator> restoreMemoryAllocator(TierId tid);
21602163
std::unique_ptr<CCacheManager> restoreCCacheManager(TierId tid);
@@ -2204,7 +2207,7 @@ auto& mmContainer = getMMContainer(tid, pid, cid);
22042207
// @throw std::runtime_error if type is invalid
22052208
std::vector<std::unique_ptr<MemoryAllocator>> initAllocator(InitMemType type);
22062209

2207-
std::vector<std::unique_ptr<MemoryAllocator>> createPrivateAllocator();
2210+
std::vector<std::unique_ptr<MemoryAllocator>> createPrivateAllocators();
22082211
std::vector<std::unique_ptr<MemoryAllocator>> createAllocators();
22092212
std::vector<std::unique_ptr<MemoryAllocator>> restoreAllocators();
22102213

@@ -2373,6 +2376,8 @@ auto& mmContainer = getMMContainer(tid, pid, cid);
23732376
// is not persisted when cache process exits.
23742377
std::unique_ptr<TempShmMapping> tempShm_;
23752378

2379+
std::unique_ptr<PrivateMemoryManager> privMemManager_;
2380+
23762381
std::unique_ptr<ShmManager> shmManager_;
23772382

23782383
// Deserialize data to restore cache allocator. Used only while attaching to

‎cachelib/allocator/MemoryTierCacheConfig.h

+6-3
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,14 @@
1616

1717
#pragma once
1818

19+
#include "cachelib/common/Utils.h"
1920
#include "cachelib/shm/ShmCommon.h"
2021

2122
namespace facebook {
2223
namespace cachelib {
2324
class MemoryTierCacheConfig {
25+
using bitmask_type = util::NumaBitMask;
26+
2427
public:
2528
// Creates instance of MemoryTierCacheConfig for Posix/SysV Shared memory.
2629
static MemoryTierCacheConfig fromShm() { return MemoryTierCacheConfig(); }
@@ -39,12 +42,12 @@ class MemoryTierCacheConfig {
3942
size_t getRatio() const noexcept { return ratio; }
4043

4144
// Allocate memory only from specified NUMA nodes
42-
MemoryTierCacheConfig& setMemBind(const NumaBitMask& _numaNodes) {
45+
MemoryTierCacheConfig& setMemBind(const bitmask_type& _numaNodes) {
4346
numaNodes = _numaNodes;
4447
return *this;
4548
}
4649

47-
const NumaBitMask& getMemBind() const noexcept { return numaNodes; }
50+
const bitmask_type& getMemBind() const noexcept { return numaNodes; }
4851

4952
size_t calculateTierSize(size_t totalCacheSize, size_t partitionNum) const {
5053
// TODO: Call this method when tiers are enabled in allocator
@@ -71,7 +74,7 @@ class MemoryTierCacheConfig {
7174
size_t ratio{1};
7275

7376
// Numa node(s) to bind the tier
74-
NumaBitMask numaNodes;
77+
bitmask_type numaNodes;
7578

7679
// TODO: introduce a container for tier settings when adding support for
7780
// file-mapped memory

‎cachelib/allocator/PrivateMemoryManager.cpp

+50
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
/*
2+
* Copyright (c) Meta Platforms, Inc. and affiliates.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
#include "cachelib/allocator/PrivateMemoryManager.h"
18+
19+
#include <folly/ScopeGuard.h>
20+
21+
namespace facebook {
22+
namespace cachelib {
23+
24+
PrivateMemoryManager::~PrivateMemoryManager() {
25+
for (auto& entry : mappings) {
26+
util::munmapMemory(entry.first, entry.second);
27+
}
28+
}
29+
30+
void* PrivateMemoryManager::createMapping(size_t size,
31+
PrivateSegmentOpts opts) {
32+
void* addr = util::mmapAlignedZeroedMemory(opts.alignment, size);
33+
auto guard = folly::makeGuard([&]() {
34+
util::munmapMemory(addr, size);
35+
mappings.erase(addr);
36+
});
37+
38+
XDCHECK_EQ(reinterpret_cast<uint64_t>(addr) & (opts.alignment - 1), 0ULL);
39+
40+
if (!opts.memBindNumaNodes.empty()) {
41+
util::mbindMemory(addr, size, MPOL_BIND, opts.memBindNumaNodes, 0);
42+
}
43+
44+
mappings.emplace(addr, size);
45+
46+
guard.dismiss();
47+
return addr;
48+
}
49+
} // namespace cachelib
50+
} // namespace facebook

‎cachelib/allocator/PrivateMemoryManager.h

+44
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
/*
2+
* Copyright (c) Meta Platforms, Inc. and affiliates.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
#pragma once
18+
19+
#include <cstddef>
20+
#include <unordered_map>
21+
22+
#include "cachelib/common/Utils.h"
23+
24+
namespace facebook {
25+
namespace cachelib {
26+
27+
struct PrivateSegmentOpts {
28+
size_t alignment{1}; // alignment for mapping.
29+
util::NumaBitMask memBindNumaNodes;
30+
};
31+
32+
class PrivateMemoryManager {
33+
public:
34+
PrivateMemoryManager() {}
35+
~PrivateMemoryManager();
36+
37+
void* createMapping(size_t size, PrivateSegmentOpts opts);
38+
39+
private:
40+
std::unordered_map<void*, size_t> mappings;
41+
};
42+
43+
} // namespace cachelib
44+
} // namespace facebook

‎cachelib/cachebench/util/CacheConfig.h

+1-1
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ struct MemoryTierConfig : public JSONConfig {
5252
MemoryTierCacheConfig getMemoryTierCacheConfig() {
5353
MemoryTierCacheConfig config = MemoryTierCacheConfig::fromShm();
5454
config.setRatio(ratio);
55-
config.setMemBind(NumaBitMask(memBindNodes));
55+
config.setMemBind(util::NumaBitMask(memBindNodes));
5656
return config;
5757
}
5858

‎cachelib/common/CMakeLists.txt

+1
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ target_link_libraries(cachelib_common PUBLIC
3939
Folly::folly_exception_tracer
4040
Folly::folly_exception_tracer_base
4141
Folly::folly_exception_counter
42+
numa
4243
)
4344

4445
install(TARGETS cachelib_common

‎cachelib/common/Utils.cpp

+17
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616

1717
#include <dirent.h>
1818
#include <folly/experimental/exception_tracer/ExceptionTracer.h>
19+
#include <numaif.h>
1920
#include <sys/mman.h>
2021
#include <sys/resource.h>
2122
#include <sys/shm.h>
@@ -181,6 +182,22 @@ void* mmapAlignedZeroedMemory(size_t alignment,
181182
throw std::system_error(errno, std::system_category(), "Cannot mmap");
182183
}
183184

185+
void munmapMemory(void* addr, size_t size) { munmap(addr, size); }
186+
187+
void mbindMemory(void* addr,
188+
unsigned long len,
189+
int mode,
190+
const NumaBitMask& mask,
191+
unsigned int flags) {
192+
auto nodesMask = mask.getNativeBitmask();
193+
194+
long ret = mbind(addr, len, mode, nodesMask->maskp, nodesMask->size, flags);
195+
if (ret != 0) {
196+
util::throwSystemError(
197+
errno, folly::sformat("mbind() failed: {}", std::strerror(errno)));
198+
}
199+
}
200+
184201
void setMaxLockMemory(uint64_t bytes) {
185202
struct rlimit rlim {
186203
bytes, bytes

0 commit comments

Comments
 (0)