Skip to content

Commit c6ae3f0

Browse files
vinser52vinser52
committed
NUMA bindigs support for private memory (#82)
1 parent 2377c45 commit c6ae3f0

14 files changed

+236
-87
lines changed

‎cachelib/allocator/CMakeLists.txt

+1
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ add_library (cachelib_allocator
5555
PoolOptimizeStrategy.cpp
5656
PoolRebalancer.cpp
5757
PoolResizer.cpp
58+
PrivateMemoryManager.cpp
5859
RebalanceStrategy.cpp
5960
SlabReleaseStats.cpp
6061
TempShmMapping.cpp

‎cachelib/allocator/CacheAllocator-inl.h

+31-11
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,9 @@ CacheAllocator<CacheTrait>::CacheAllocator(
5959
tempShm_(type == InitMemType::kNone && isOnShm_
6060
? std::make_unique<TempShmMapping>(config_.getCacheSize())
6161
: nullptr),
62+
privMemManager_(type == InitMemType::kNone && !isOnShm_
63+
? std::make_unique<PrivateMemoryManager>()
64+
: nullptr),
6265
shmManager_(type != InitMemType::kNone
6366
? std::make_unique<ShmManager>(config_.cacheDir,
6467
config_.isUsingPosixShm())
@@ -121,6 +124,16 @@ ShmSegmentOpts CacheAllocator<CacheTrait>::createShmCacheOpts(TierId tid) {
121124
return opts;
122125
}
123126

127+
template <typename CacheTrait>
128+
PrivateSegmentOpts CacheAllocator<CacheTrait>::createPrivateSegmentOpts(TierId tid) {
129+
PrivateSegmentOpts opts;
130+
opts.alignment = sizeof(Slab);
131+
auto memoryTierConfigs = config_.getMemoryTierConfigs();
132+
opts.memBindNumaNodes = memoryTierConfigs[tid].getMemBind();
133+
134+
return opts;
135+
}
136+
124137
template <typename CacheTrait>
125138
size_t CacheAllocator<CacheTrait>::memoryTierSize(TierId tid) const {
126139
auto& memoryTierConfigs = config_.memoryTierConfigs;
@@ -133,21 +146,18 @@ size_t CacheAllocator<CacheTrait>::memoryTierSize(TierId tid) const {
133146
}
134147

135148
template <typename CacheTrait>
136-
std::vector<std::unique_ptr<MemoryAllocator>>
137-
CacheAllocator<CacheTrait>::createPrivateAllocator() {
138-
std::vector<std::unique_ptr<MemoryAllocator>> allocators;
139-
149+
std::unique_ptr<MemoryAllocator>
150+
CacheAllocator<CacheTrait>::createPrivateAllocator(TierId tid) {
140151
if (isOnShm_)
141-
allocators.emplace_back(std::make_unique<MemoryAllocator>(
152+
return std::make_unique<MemoryAllocator>(
142153
getAllocatorConfig(config_),
143154
tempShm_->getAddr(),
144-
config_.getCacheSize()));
155+
memoryTierSize(tid));
145156
else
146-
allocators.emplace_back(std::make_unique<MemoryAllocator>(
157+
return std::make_unique<MemoryAllocator>(
147158
getAllocatorConfig(config_),
148-
config_.getCacheSize()));
149-
150-
return allocators;
159+
privMemManager_->createMapping(config_.size, createPrivateSegmentOpts(tid)),
160+
memoryTierSize(tid));
151161
}
152162

153163
template <typename CacheTrait>
@@ -176,6 +186,16 @@ CacheAllocator<CacheTrait>::restoreMemoryAllocator(TierId tid) {
176186
config_.disableFullCoredump);
177187
}
178188

189+
template <typename CacheTrait>
190+
std::vector<std::unique_ptr<MemoryAllocator>>
191+
CacheAllocator<CacheTrait>::createPrivateAllocators() {
192+
std::vector<std::unique_ptr<MemoryAllocator>> allocators;
193+
for (int tid = 0; tid < getNumTiers(); tid++) {
194+
allocators.emplace_back(createPrivateAllocator(tid));
195+
}
196+
return allocators;
197+
}
198+
179199
template <typename CacheTrait>
180200
std::vector<std::unique_ptr<MemoryAllocator>>
181201
CacheAllocator<CacheTrait>::createAllocators() {
@@ -309,7 +329,7 @@ std::vector<std::unique_ptr<MemoryAllocator>>
309329
CacheAllocator<CacheTrait>::initAllocator(
310330
InitMemType type) {
311331
if (type == InitMemType::kNone) {
312-
return createPrivateAllocator();
332+
return createPrivateAllocators();
313333
} else if (type == InitMemType::kMemNew) {
314334
return createAllocators();
315335
} else if (type == InitMemType::kMemAttach) {

‎cachelib/allocator/CacheAllocator.h

+6-1
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@
6161
#include "cachelib/allocator/PoolOptimizer.h"
6262
#include "cachelib/allocator/PoolRebalancer.h"
6363
#include "cachelib/allocator/PoolResizer.h"
64+
#include "cachelib/allocator/PrivateMemoryManager.h"
6465
#include "cachelib/allocator/ReadOnlySharedCacheView.h"
6566
#include "cachelib/allocator/Reaper.h"
6667
#include "cachelib/allocator/RebalanceStrategy.h"
@@ -2192,6 +2193,8 @@ class CacheAllocator : public CacheBase {
21922193
std::chrono::seconds timeout = std::chrono::seconds{0});
21932194

21942195
ShmSegmentOpts createShmCacheOpts(TierId tid);
2196+
PrivateSegmentOpts createPrivateSegmentOpts(TierId tid);
2197+
std::unique_ptr<MemoryAllocator> createPrivateAllocator(TierId tid);
21952198
std::unique_ptr<MemoryAllocator> createNewMemoryAllocator(TierId tid);
21962199
std::unique_ptr<MemoryAllocator> restoreMemoryAllocator(TierId tid);
21972200
std::unique_ptr<CCacheManager> restoreCCacheManager(TierId tid);
@@ -2241,7 +2244,7 @@ class CacheAllocator : public CacheBase {
22412244
// @throw std::runtime_error if type is invalid
22422245
std::vector<std::unique_ptr<MemoryAllocator>> initAllocator(InitMemType type);
22432246

2244-
std::vector<std::unique_ptr<MemoryAllocator>> createPrivateAllocator();
2247+
std::vector<std::unique_ptr<MemoryAllocator>> createPrivateAllocators();
22452248
std::vector<std::unique_ptr<MemoryAllocator>> createAllocators();
22462249
std::vector<std::unique_ptr<MemoryAllocator>> restoreAllocators();
22472250

@@ -2407,6 +2410,8 @@ class CacheAllocator : public CacheBase {
24072410
// is not persisted when cache process exits.
24082411
std::unique_ptr<TempShmMapping> tempShm_;
24092412

2413+
std::unique_ptr<PrivateMemoryManager> privMemManager_;
2414+
24102415
std::unique_ptr<ShmManager> shmManager_;
24112416

24122417
// Deserialize data to restore cache allocator. Used only while attaching to

‎cachelib/allocator/MemoryTierCacheConfig.h

+6-3
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,14 @@
1616

1717
#pragma once
1818

19+
#include "cachelib/common/Utils.h"
1920
#include "cachelib/shm/ShmCommon.h"
2021

2122
namespace facebook {
2223
namespace cachelib {
2324
class MemoryTierCacheConfig {
25+
using bitmask_type = util::NumaBitMask;
26+
2427
public:
2528
// Creates instance of MemoryTierCacheConfig for Posix/SysV Shared memory.
2629
static MemoryTierCacheConfig fromShm() { return MemoryTierCacheConfig(); }
@@ -39,12 +42,12 @@ class MemoryTierCacheConfig {
3942
size_t getRatio() const noexcept { return ratio; }
4043

4144
// Allocate memory only from specified NUMA nodes
42-
MemoryTierCacheConfig& setMemBind(const NumaBitMask& _numaNodes) {
45+
MemoryTierCacheConfig& setMemBind(const bitmask_type& _numaNodes) {
4346
numaNodes = _numaNodes;
4447
return *this;
4548
}
4649

47-
const NumaBitMask& getMemBind() const noexcept { return numaNodes; }
50+
const bitmask_type& getMemBind() const noexcept { return numaNodes; }
4851

4952
size_t calculateTierSize(size_t totalCacheSize, size_t partitionNum) const {
5053
// TODO: Call this method when tiers are enabled in allocator
@@ -71,7 +74,7 @@ class MemoryTierCacheConfig {
7174
size_t ratio{1};
7275

7376
// Numa node(s) to bind the tier
74-
NumaBitMask numaNodes;
77+
bitmask_type numaNodes;
7578

7679
// TODO: introduce a container for tier settings when adding support for
7780
// file-mapped memory

‎cachelib/allocator/PrivateMemoryManager.cpp

+50
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
/*
2+
* Copyright (c) Meta Platforms, Inc. and affiliates.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
#include "cachelib/allocator/PrivateMemoryManager.h"
18+
19+
#include <folly/ScopeGuard.h>
20+
21+
namespace facebook {
22+
namespace cachelib {
23+
24+
PrivateMemoryManager::~PrivateMemoryManager() {
25+
for (auto& entry : mappings) {
26+
util::munmapMemory(entry.first, entry.second);
27+
}
28+
}
29+
30+
void* PrivateMemoryManager::createMapping(size_t size,
31+
PrivateSegmentOpts opts) {
32+
void* addr = util::mmapAlignedZeroedMemory(opts.alignment, size);
33+
auto guard = folly::makeGuard([&]() {
34+
util::munmapMemory(addr, size);
35+
mappings.erase(addr);
36+
});
37+
38+
XDCHECK_EQ(reinterpret_cast<uint64_t>(addr) & (opts.alignment - 1), 0ULL);
39+
40+
if (!opts.memBindNumaNodes.empty()) {
41+
util::mbindMemory(addr, size, MPOL_BIND, opts.memBindNumaNodes, 0);
42+
}
43+
44+
mappings.emplace(addr, size);
45+
46+
guard.dismiss();
47+
return addr;
48+
}
49+
} // namespace cachelib
50+
} // namespace facebook

‎cachelib/allocator/PrivateMemoryManager.h

+44
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
/*
2+
* Copyright (c) Meta Platforms, Inc. and affiliates.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
#pragma once
18+
19+
#include <cstddef>
20+
#include <unordered_map>
21+
22+
#include "cachelib/common/Utils.h"
23+
24+
namespace facebook {
25+
namespace cachelib {
26+
27+
struct PrivateSegmentOpts {
28+
size_t alignment{1}; // alignment for mapping.
29+
util::NumaBitMask memBindNumaNodes;
30+
};
31+
32+
class PrivateMemoryManager {
33+
public:
34+
PrivateMemoryManager() {}
35+
~PrivateMemoryManager();
36+
37+
void* createMapping(size_t size, PrivateSegmentOpts opts);
38+
39+
private:
40+
std::unordered_map<void*, size_t> mappings;
41+
};
42+
43+
} // namespace cachelib
44+
} // namespace facebook

‎cachelib/cachebench/util/CacheConfig.h

+1-1
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ struct MemoryTierConfig : public JSONConfig {
5252
MemoryTierCacheConfig getMemoryTierCacheConfig() {
5353
MemoryTierCacheConfig config = MemoryTierCacheConfig::fromShm();
5454
config.setRatio(ratio);
55-
config.setMemBind(NumaBitMask(memBindNodes));
55+
config.setMemBind(util::NumaBitMask(memBindNodes));
5656
return config;
5757
}
5858

‎cachelib/common/CMakeLists.txt

+1
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ target_link_libraries(cachelib_common PUBLIC
4040
Folly::folly_exception_tracer
4141
Folly::folly_exception_tracer_base
4242
Folly::folly_exception_counter
43+
numa
4344
)
4445

4546
install(TARGETS cachelib_common

‎cachelib/common/Utils.cpp

+17
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616

1717
#include <dirent.h>
1818
#include <folly/experimental/exception_tracer/ExceptionTracer.h>
19+
#include <numaif.h>
1920
#include <sys/mman.h>
2021
#include <sys/resource.h>
2122
#include <sys/shm.h>
@@ -181,6 +182,22 @@ void* mmapAlignedZeroedMemory(size_t alignment,
181182
throw std::system_error(errno, std::system_category(), "Cannot mmap");
182183
}
183184

185+
void munmapMemory(void* addr, size_t size) { munmap(addr, size); }
186+
187+
void mbindMemory(void* addr,
188+
unsigned long len,
189+
int mode,
190+
const NumaBitMask& mask,
191+
unsigned int flags) {
192+
auto nodesMask = mask.getNativeBitmask();
193+
194+
long ret = mbind(addr, len, mode, nodesMask->maskp, nodesMask->size, flags);
195+
if (ret != 0) {
196+
util::throwSystemError(
197+
errno, folly::sformat("mbind() failed: {}", std::strerror(errno)));
198+
}
199+
}
200+
184201
void setMaxLockMemory(uint64_t bytes) {
185202
struct rlimit rlim {
186203
bytes, bytes

0 commit comments

Comments
 (0)