Skip to content

Commit 48c386a

Browse files
y123456yzy123456yz
committed
storage analyze
1 parent f050a35 commit 48c386a

18 files changed

+102
-14
lines changed

‎mongo/compile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
代码高亮网页转换：http://www.codeinword.com/
22

3-
3+
mtools:分析mongodb日志等 https://github.com/rueckstiess/mtools
44

55
c++17可以参考https://github.com/MeouSker77/Cpp17
66

‎mongo/src/mongo/db/catalog/coll_mod.cpp

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -297,6 +297,14 @@ void setCollectionOptionFlag(OperationContext* opCtx,
297297
invariant(newOptions.flagsSet);
298298
}
299299

300+
/*
301+
该方法给集合添加一个标识，来修改集合的行为。 标识包含usePowerOf2Sizes和index。
302+
303+
命令格式为：
304+
305+
db.runCommand({"collMod":<collection>,"<flag>":<value>})
306+
307+
*/
300308
Status _collModInternal(OperationContext* opCtx,
301309
const NamespaceString& nss,
302310
const BSONObj& cmdObj,
@@ -690,3 +698,4 @@ Status updateUUIDSchemaVersionNonReplicated(OperationContext* opCtx, bool upgrad
690698
return Status::OK();
691699
}
692700
} // namespace mongo
701+

‎mongo/src/mongo/db/catalog/coll_mod.h

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,15 @@ class Collection;
3737
class NamespaceString;
3838
class OperationContext;
3939

40+
/*
41+
该方法给集合添加一个标识，来修改集合的行为。 标识包含usePowerOf2Sizes和index。
42+
43+
命令格式为：
44+
45+
db.runCommand({"collMod":<collection>,"<flag>":<value>})
46+
47+
*/
48+
4049
/**
4150
* If upgrade is true, adds UUIDs to all collections of all databases. If upgrade is false, removes
4251
* UUIDs from all collections of all databases. It updates non-replicated collections by indirectly

‎mongo/src/mongo/db/catalog/collection_impl.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -626,6 +626,7 @@ void CollectionImpl::deleteDocument(OperationContext* opCtx,
626626
/* check if any cursors point to us. if so, advance them. */
627627
_cursorManager.invalidateDocument(opCtx, loc, INVALIDATION_DELETION);
628628

629+
//删除对应索引KV
629630
int64_t keysDeleted;
630631
_indexCatalog.unindexRecord(opCtx, doc.value(), loc, noWarn, &keysDeleted);
631632
if (opDebug) {

‎mongo/src/mongo/db/catalog/collection_info_cache_impl.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -256,7 +256,7 @@ void CollectionInfoCacheImpl::init(OperationContext* opCtx) {
256256

257257
//通过类似_collection->infoCache()->addedIndex(_opCtx, descriptorPtr);方式调用，参考IndexCatalogImpl::IndexBuildBlock::init()
258258
//IndexCatalogImpl::IndexBuildBlock::init()调用
259-
//新索引注册到_indexUsageTracker
259+
//新索引注册到_indexUsageTracker，表示该表新增了索引信息
260260
void CollectionInfoCacheImpl::addedIndex(OperationContext* opCtx, const IndexDescriptor* desc) {
261261
// Requires exclusive collection lock.
262262
invariant(opCtx->lockState()->isCollectionLockedForMode(_collection->ns().ns(), MODE_X));

‎mongo/src/mongo/db/catalog/index_catalog.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@ struct InsertDeleteOptions;
6666
class IndexCatalog {
6767
public:
6868
//遍历所有索引信息可以参考fillOutPlannerParams
69+
//IndexIteratorImpl集成该迭代器，迭代器使用参考IndexCatalogImpl::findIdIndex
6970
class IndexIterator {
7071
public:
7172
class Impl {

‎mongo/src/mongo/db/catalog/index_catalog_entry_impl.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,7 @@ IndexCatalogEntryImpl::IndexCatalogEntryImpl(IndexCatalogEntry* const this_,
128128
_collator = std::move(statusWithCollator.getValue());
129129
}
130130

131+
////基于age列创建大于25岁的部分索引 db.persons.createIndex({country:1},{partialFilterExpression: {age: {$gt:25}}})
131132
if (BSONElement filterElement = _descriptor->getInfoElement("partialFilterExpression")) {
132133
invariant(filterElement.isABSONObj());
133134
BSONObj filter = filterElement.Obj();

‎mongo/src/mongo/db/catalog/index_catalog_entry_impl.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,7 @@ class IndexCatalogEntryImpl : public IndexCatalogEntry::Impl {
9595
return _ordering;
9696
}
9797

98+
//基于age列创建大于25岁的部分索引 db.persons.createIndex({country:1},{partialFilterExpression: {age: {$gt:25}}})
9899
const MatchExpression* getFilterExpression() const final {
99100
return _filterExpression.get();
100101
}
@@ -201,6 +202,7 @@ class IndexCatalogEntryImpl : public IndexCatalogEntry::Impl {
201202
// Owned here.
202203
std::unique_ptr<HeadManager> _headManager;
203204
std::unique_ptr<CollatorInterface> _collator;
205+
//例如基于age列创建大于25岁的部分索引 db.persons.createIndex({country:1},{partialFilterExpression: {age: {$gt:25}}})
204206
std::unique_ptr<MatchExpression> _filterExpression;
205207

206208
// cached stuff

‎mongo/src/mongo/db/catalog/index_catalog_impl.cpp

Lines changed: 55 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -252,6 +252,7 @@ Status IndexCatalogImpl::checkUnfinished() const {
252252
<< _collection->ns().ns());
253253
}
254254

255+
//版本兼容相关，IndexCatalogImpl::_getAccessMethodName中调用，先忽略
255256
bool IndexCatalogImpl::_shouldOverridePlugin(OperationContext* opCtx,
256257
const BSONObj& keyPattern) const {
257258
string pluginName = IndexNames::findPluginName(keyPattern);
@@ -298,6 +299,7 @@ string IndexCatalogImpl::_getAccessMethodName(OperationContext* opCtx,
298299

299300
// ---------------------------
300301

302+
//忽略，版本兼容性相关的
301303
Status IndexCatalogImpl::_upgradeDatabaseMinorVersionIfNeeded(OperationContext* opCtx,
302304
const string& newPluginName) {
303305
// first check if requested index requires pdfile minor version to be bumped
@@ -307,6 +309,7 @@ Status IndexCatalogImpl::_upgradeDatabaseMinorVersionIfNeeded(OperationContext*
307309

308310
DatabaseCatalogEntry* dbce = _collection->dbce();
309311

312+
//
310313
if (!dbce->isOlderThan24(opCtx)) {
311314
return Status::OK(); // these checks have already been done
312315
}
@@ -373,7 +376,7 @@ db/catalog/database_impl.cpp: fullIdIndexSpec = uassertStatusOK(i
373376
374377
*/
375378
//DatabaseImpl::createCollection createSystemIndexes CollectionImpl::truncate中调用执行
376-
//空表上面建索引
379+
//空表上面建索引,一般是手动创建索引或者第一次给某个表写数据的时候调用
377380
StatusWith<BSONObj> IndexCatalogImpl::createIndexOnEmptyCollection(OperationContext* opCtx,
378381
BSONObj spec) {
379382
invariant(opCtx->lockState()->isCollectionLockedForMode(_collection->ns().toString(), MODE_X));
@@ -386,6 +389,7 @@ StatusWith<BSONObj> IndexCatalogImpl::createIndexOnEmptyCollection(OperationCont
386389
return status;
387390

388391
//索引个数 索引冲突 索引名冲突等检查失败，例如索引名有冲突 索引达到上限等
392+
//然后返回对应的索引spec
389393
StatusWith<BSONObj> statusWithSpec = prepareSpecForCreate(opCtx, spec);
390394
status = statusWithSpec.getStatus();
391395
if (!status.isOK())
@@ -403,20 +407,26 @@ StatusWith<BSONObj> IndexCatalogImpl::createIndexOnEmptyCollection(OperationCont
403407
// now going to touch disk
404408
IndexBuildBlock indexBuildBlock(opCtx, _collection, spec);
405409
//IndexCatalogImpl::IndexBuildBlock::init
406-
status = indexBuildBlock.init(); //建索引
410+
//该索引初始化
411+
status = indexBuildBlock.init();
407412
if (!status.isOK())
408413
return status;
409414

410415
// sanity checks, etc...
416+
//获取该索引对应IndexCatalogEntryImpl
411417
IndexCatalogEntry* entry = indexBuildBlock.getEntry();
412418
invariant(entry);
419+
//获取该索引descriptor
413420
IndexDescriptor* descriptor = entry->descriptor();
414421
invariant(descriptor);
415422
invariant(entry == _entries.find(descriptor));
416423

424+
//啥也没做IndexAccessMethod::initializeAsEmpty->WiredTigerIndex::initAsEmpty
417425
status = entry->accessMethod()->initializeAsEmpty(opCtx);
418426
if (!status.isOK())
419427
return status;
428+
429+
//IndexBuildBlock::success
420430
indexBuildBlock.success();
421431

422432
// sanity check
@@ -425,14 +435,17 @@ StatusWith<BSONObj> IndexCatalogImpl::createIndexOnEmptyCollection(OperationCont
425435
return spec;
426436
}
427437

428-
//index_create_impl.cpp中的MultiIndexBlockImpl::init new该类
438+
//IndexCatalogImpl::createIndexOnEmptyCollection中第一次使用空表的时候(第一次写数据或者手动建表)
439+
//index_create_impl.cpp中的MultiIndexBlockImpl::init new该类,也就是建索引的时候
429440
IndexCatalogImpl::IndexBuildBlock::IndexBuildBlock(OperationContext* opCtx,
430441
Collection* collection,
431442
const BSONObj& spec)
432443
: _collection(collection),
433444
_catalog(collection->getIndexCatalog()),
434445
_ns(_catalog->_getCollection()->ns().ns()),
435446
_spec(spec.getOwned()),
447+
//真正赋值见IndexCatalogImpl::_setupInMemoryStructures
448+
//对应IndexCatalogEntryImpl
436449
_entry(nullptr),
437450
_opCtx(opCtx) {
438451
invariant(collection);
@@ -442,10 +455,11 @@ IndexCatalogImpl::IndexBuildBlock::IndexBuildBlock(OperationContext* opCtx,
442455
//创建集合的时候或者程序重启的时候建索引:DatabaseImpl::createCollection->IndexCatalogImpl::createIndexOnEmptyCollection->IndexCatalogImpl::IndexBuildBlock::init
443456
//MultiIndexBlockImpl::init->IndexCatalogImpl::IndexBuildBlock::init 程序运行过程中，并且集合已经存在的时候建索引
444457

445-
//IndexCatalogImpl::createIndexOnEmptyCollection 调用
446-
//CmdCreateIndex::errmsgRun->MultiIndexBlockImpl::init调用
458+
//IndexCatalogImpl::createIndexOnEmptyCollection 调用(第一次创建空表或者第一次往空表写入数据)
459+
//CmdCreateIndex::errmsgRun->MultiIndexBlockImpl::init调用(收到createIndex建索引)
447460

448461
////获取descriptor该索引对应的IndexCatalogEntryImpl添加到_entries数组
462+
//对IndexBuildBlock对应的索引进行初始化
449463
Status IndexCatalogImpl::IndexBuildBlock::init() {
450464
// need this first for names, etc...
451465
//获取所有信息的key，可以db.collection.getIndexes()获取
@@ -474,7 +488,7 @@ Status IndexCatalogImpl::IndexBuildBlock::init() {
474488
/// ---------- setup in memory structures ----------------
475489
const bool initFromDisk = false;
476490
//获取descriptor该索引对应的IndexCatalogEntryImpl添加到_entries数组
477-
//一个索引对应一个IndexCatalogImpl
491+
//一个索引对应一个IndexCatalogEntryImpl
478492
_entry = IndexCatalogImpl::_setupInMemoryStructures(
479493
_catalog, _opCtx, std::move(descriptor), initFromDisk);
480494

@@ -491,6 +505,7 @@ IndexCatalogImpl::IndexBuildBlock::~IndexBuildBlock() {
491505
// Don't need to call fail() here, as rollback will clean everything up for us.
492506
}
493507

508+
//失败处理
494509
void IndexCatalogImpl::IndexBuildBlock::fail() {
495510
fassert(17204, _catalog->_getCollection()->ok()); // defensive
496511

@@ -506,19 +521,26 @@ void IndexCatalogImpl::IndexBuildBlock::fail() {
506521

507522
//IndexCatalogImpl::createIndexOnEmptyCollection调用 检查索引是否创建成功
508523
void IndexCatalogImpl::IndexBuildBlock::success() {
524+
//获取该索引对应的表信息
509525
Collection* collection = _catalog->_getCollection();
510526
fassert(17207, collection->ok());
511527
NamespaceString ns(_indexNamespace);
512528
invariant(_opCtx->lockState()->isDbLockedForMode(ns.db(), MODE_X));
513529

530+
//KVCollectionCatalogEntry::indexBuildSuccess
531+
//更新元数据"_mdb_catalog.wt"信息中的索引
514532
collection->getCatalogEntry()->indexBuildSuccess(_opCtx, _indexName);
515533

534+
//IndexCatalogImpl::findIndexByName
535+
//查找该索引desc
516536
IndexDescriptor* desc = _catalog->findIndexByName(_opCtx, _indexName, true);
517537
fassert(17330, desc);
538+
//IndexCatalogImpl::_getEntries从数组中查找内存中是否有该索引desc
518539
IndexCatalogEntry* entry = _catalog->_getEntries().find(desc);
519540
fassert(17331, entry && entry == _entry);
520541

521542
OperationContext* opCtx = _opCtx;
543+
//例如id索引对应打印marking index _id_ as ready in snapshot id 85053
522544
LOG(2) << "marking index " << _indexName << " as ready in snapshot id "
523545
<< opCtx->recoveryUnit()->getSnapshotId();
524546
_opCtx->recoveryUnit()->onCommit([opCtx, entry, collection] {
@@ -1170,6 +1192,9 @@ void IndexCatalogImpl::_deleteIndexFromDisk(OperationContext* opCtx,
11701192
// "Leftover" means they were unfinished when a mongod shut down.
11711193
// Certain operations are prohibited until someone fixes.
11721194
// Retrieve by calling getAndClearUnfinishedIndexes().
1195+
1196+
//restartInProgressIndexesFromLastShutdown->checkNS调用
1197+
//当真正运行添加某个索引得时候，如果这时候实例挂掉，当实例重启后需要调用该接口先删除索引,然后再checkNS重构索引
11731198
vector<BSONObj> IndexCatalogImpl::getAndClearUnfinishedIndexes(OperationContext* opCtx) {
11741199
vector<BSONObj> toReturn = _unfinishedIndexes;
11751200
_unfinishedIndexes.clear();
@@ -1453,7 +1478,15 @@ const IndexCatalogEntry* IndexCatalogImpl::getEntry(const IndexDescriptor* desc)
14531478
return entry;
14541479
}
14551480

1481+
/*
1482+
该方法给集合添加一个标识，来修改集合的行为。 标识包含usePowerOf2Sizes和index。
1483+
1484+
命令格式为：
1485+
1486+
db.runCommand({"collMod":<collection>,"<flag>":<value>})
14561487
1488+
*/
1489+
//_collModInternal中调用， 先跳过
14571490
const IndexDescriptor* IndexCatalogImpl::refreshEntry(OperationContext* opCtx,
14581491
const IndexDescriptor* oldDesc) {
14591492
invariant(opCtx->lockState()->isCollectionLockedForMode(_collection->ns().ns(), MODE_X));
@@ -1471,6 +1504,7 @@ const IndexDescriptor* IndexCatalogImpl::refreshEntry(OperationContext* opCtx,
14711504

14721505
// Delete the IndexCatalogEntry that owns this descriptor. After deletion, 'oldDesc' is
14731506
// invalid and should not be dereferenced.
1507+
//从_entries数组清除oldDesc
14741508
IndexCatalogEntry* oldEntry = _entries.release(oldDesc);
14751509
opCtx->recoveryUnit()->registerChange(
14761510
new IndexRemoveChange(opCtx, _collection, &_entries, oldEntry));
@@ -1492,7 +1526,8 @@ const IndexDescriptor* IndexCatalogImpl::refreshEntry(OperationContext* opCtx,
14921526
}
14931527

14941528
// ---------------------------
1495-
//IndexCatalogImpl::_indexRecords
1529+
//IndexCatalogImpl::_indexRecords调用，
1530+
//把bsonRecords对应的索引KV数据写入存储引擎
14961531
Status IndexCatalogImpl::_indexFilteredRecords(OperationContext* opCtx,
14971532
IndexCatalogEntry* index,
14981533
const std::vector<BsonRecord>& bsonRecords,
@@ -1518,23 +1553,29 @@ Status IndexCatalogImpl::_indexFilteredRecords(OperationContext* opCtx,
15181553
}
15191554

15201555
//IndexCatalogImpl::indexRecords
1556+
//把bsonRecords对应的索引KV(如果需要过滤，则提前过滤)数据写入存储引擎
15211557
Status IndexCatalogImpl::_indexRecords(OperationContext* opCtx,
15221558
IndexCatalogEntry* index,
15231559
const std::vector<BsonRecord>& bsonRecords,
15241560
int64_t* keysInsertedOut) {
1525-
const MatchExpression* filter = index->getFilterExpression();
1526-
if (!filter)
1561+
//基于age列创建大于25岁的部分索引 db.persons.createIndex({country:1},{partialFilterExpression: {age: {$gt:25}}})
1562+
const MatchExpression* filter = index->getFilterExpression();
1563+
if (!filter) //普通索引，也就是不带partialFilterExpression
15271564
return _indexFilteredRecords(opCtx, index, bsonRecords, keysInsertedOut);
15281565

1566+
//携带有partialFilterExpression参数的索引
15291567
std::vector<BsonRecord> filteredBsonRecords;
15301568
for (auto bsonRecord : bsonRecords) {
1569+
//只把满足partialFilterExpression条件的doc添加到filteredBsonRecords，然后写入存储引擎
15311570
if (filter->matchesBSON(*(bsonRecord.docPtr)))
15321571
filteredBsonRecords.push_back(bsonRecord);
15331572
}
15341573

15351574
return _indexFilteredRecords(opCtx, index, filteredBsonRecords, keysInsertedOut);
15361575
}
15371576

1577+
//删除loc对应的index索引数据
1578+
//IndexCatalogImpl::unindexRecord调用
15381579
Status IndexCatalogImpl::_unindexRecord(OperationContext* opCtx,
15391580
IndexCatalogEntry* index,
15401581
const BSONObj& obj,
@@ -1588,6 +1629,7 @@ Status IndexCatalogImpl::indexRecords(OperationContext* opCtx,
15881629
return Status::OK();
15891630
}
15901631

1632+
//CollectionImpl::deleteDocument调用，删除数据loc对应的索引KV数据
15911633
void IndexCatalogImpl::unindexRecord(OperationContext* opCtx,
15921634
const BSONObj& obj,
15931635
const RecordId& loc,
@@ -1597,16 +1639,19 @@ void IndexCatalogImpl::unindexRecord(OperationContext* opCtx,
15971639
*keysDeletedOut = 0;
15981640
}
15991641

1642+
//删除该数据对应的所有索引KV
16001643
for (IndexCatalogEntryContainer::const_iterator i = _entries.begin(); i != _entries.end();
16011644
++i) {
16021645
IndexCatalogEntry* entry = i->get();
16031646

16041647
// If it's a background index, we DO NOT want to log anything.
16051648
bool logIfError = entry->isReady(opCtx) ? !noWarn : false;
1649+
//删除loc对应的index索引数据
16061650
_unindexRecord(opCtx, entry, obj, loc, logIfError, keysDeletedOut).transitional_ignore();
16071651
}
16081652
}
16091653

1654+
//IndexCatalogImpl::_fixIndexSpec中调用
16101655
BSONObj IndexCatalogImpl::fixIndexKey(const BSONObj& key) {
16111656
if (IndexDescriptor::isIdIndexPattern(key)) {
16121657
return _idObj;
@@ -1639,7 +1684,7 @@ void IndexCatalogImpl::prepareInsertDeleteOptions(OperationContext* opCtx,
16391684
}
16401685
}
16411686

1642-
//
1687+
//从spec中获取索引信息,然后以bson格式返回
16431688
StatusWith<BSONObj> IndexCatalogImpl::_fixIndexSpec(OperationContext* opCtx,
16441689
Collection* collection,
16451690
const BSONObj& spec) {

‎mongo/src/mongo/db/catalog/index_catalog_impl.h

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -282,6 +282,9 @@ class IndexCatalogImpl : public IndexCatalog::Impl {
282282
* 3) indexes entry in .ns file
283283
* 4) system.namespaces entry for index ns
284284
*/
285+
//IndexCatalogImpl::createIndexOnEmptyCollection中第一次使用空表的时候(第一次写数据或者手动建表)
286+
//index_create_impl.cpp中的MultiIndexBlockImpl::init new该类
287+
285288
//MultiIndexBlockImpl.IndexToBuild.block为该类型
286289
class IndexBuildBlock {
287290
MONGO_DISALLOW_COPYING(IndexBuildBlock);
@@ -315,6 +318,7 @@ class IndexCatalogImpl : public IndexCatalog::Impl {
315318
std::string _indexName;
316319
std::string _indexNamespace;
317320

321+
//对应IndexCatalogEntryImpl
318322
IndexCatalogEntry* _entry;
319323
bool _inProgress;
320324

@@ -477,3 +481,4 @@ class IndexCatalogImpl : public IndexCatalog::Impl {
477481
}
478482
};
479483
} // namespace mongo
484+

‎mongo/src/mongo/db/catalog/index_create_impl.cpp

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -261,6 +261,10 @@ db.runCommand(
261261

262262
//IndexCatalogImpl::createIndexOnEmptyCollection中调用
263263

264+
//当某个实例上面真正构建索引，突然实例挂了，mongod重启后调用:
265+
// restartInProgressIndexesFromLastShutdown->checkNS调用
266+
267+
264268
//加索引和重做索引会走到这里面来
265269
//CmdCreateIndex::errmsgRun CmdReIndex::errmsgRun调用
266270
//建索引的一些初始化工作

‎mongo/src/mongo/db/index/index_access_method.cpp

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -267,8 +267,11 @@ Status IndexAccessMethod::remove(OperationContext* opCtx,
267267
return Status::OK();
268268
}
269269

270-
//MultiIndexBlockImpl::init调用
270+
//MultiIndexBlockImpl::init(手动加索引) IndexCatalogImpl::createIndexOnEmptyCollection(空表写数据或者建表)调用
271271
Status IndexAccessMethod::initializeAsEmpty(OperationContext* opCtx) {
272+
//唯一索引WiredTigerIndexUnique 普通索引WiredTigerIndexStandard
273+
274+
//WiredTigerIndex::initAsEmpty，啥也没做这里面
272275
return _newInterface->initAsEmpty(opCtx);
273276
}
274277

0 commit comments

Comments
 (0)