Skip to content

Commit 53b30d3

Browse files
y123456yzy123456yz
committed
kv analyze
1 parent 2c595f0 commit 53b30d3

File tree

6 files changed

+229
-6
lines changed

6 files changed

+229
-6
lines changed

‎mongo/src/mongo/db/storage/bson_collection_catalog_entry.cpp

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -289,6 +289,14 @@ bool BSONCollectionCatalogEntry::MetaData::eraseIndex(StringData name) {
289289
return true;
290290
}
291291

292+
//建一个新表的元数据打印:db.createCollection("mycol", { capped : true, autoIndexId : true, size : 6142800, max : 10000 } )
293+
//{ md: { ns: "test.mycol", options: { uuid: UUID("75591c22-bd0f-4a56-ac95-ef90224cf3df"), capped: true, size: 6142976, max: 10000, autoIndexId: true },
294+
//indexes: [ { spec: { v: 2, key: { _id: 1 }, name: "_id_", ns: "test.mycol" }, ready: false, multikey: false,
295+
//multikeyPaths: { _id: BinData(0, 00) }, head: 0, prefix: -1 } ], prefix: -1 }, idxIdent: { _id_: "test/index/2--6948813758302814892" },
296+
//ns: "test.mycol", ident: "test/collection/1--6948813758302814892" }
297+
298+
//原表重命名，则md中的索引也需要修改，例如上面的indexes中的test.mycol，对应库test，表mycol，如果表mycol
299+
//重命名为mycol2，则indexes中所有的mycol需要修改为mycol2
292300
void BSONCollectionCatalogEntry::MetaData::rename(StringData toNS) {
293301
ns = toNS.toString();
294302
for (size_t i = 0; i < indexes.size(); i++) {

‎mongo/src/mongo/db/storage/kv/kv_catalog.cpp

Lines changed: 126 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -184,6 +184,113 @@
184184
├── WiredTiger.turtle
185185
└── WiredTiger.wt
186186
187+
{
188+
md: {
189+
ns: "test.mycol",
190+
options: {
191+
uuid: UUID("75591c22-bd0f-4a56-ac95-ef90224cf3df"),
192+
capped: true,
193+
size: 6142976,
194+
max: 10000,
195+
autoIndexId: true
196+
},
197+
indexes: [{
198+
spec: {
199+
v: 2,
200+
key: {
201+
_id: 1
202+
},
203+
name: "_id_",
204+
ns: "test.mycol"
205+
},
206+
ready: false,
207+
multikey: false,
208+
multikeyPaths: {
209+
_id: BinData(0, 00)
210+
},
211+
head: 0,
212+
prefix: -1
213+
}],
214+
prefix: -1
215+
},
216+
idxIdent: {
217+
_id_: "test/index/2--6948813758302814892"
218+
},
219+
ns: "test.mycol",
220+
ident: "test/collection/1--6948813758302814892"
221+
}
222+
223+
{
224+
md: {
225+
ns: "test.test",
226+
options: {
227+
uuid: UUID("520904ec-0432-4c00-a15d-788e2f5d707b")
228+
//建表参数都在这里
229+
},
230+
indexes: [{
231+
spec: {
232+
v: 2,
233+
key: {
234+
_id: 1
235+
},
236+
name: "_id_",
237+
ns: "test.test"
238+
},
239+
ready: true,
240+
multikey: false,
241+
multikeyPaths: {
242+
_id: BinData(0, 00)
243+
},
244+
head: 0,
245+
prefix: -1
246+
}, {
247+
spec: {
248+
v: 2,
249+
key: {
250+
name: 1.0,
251+
age: 1.0
252+
},
253+
name: "name_1_age_1",
254+
ns: "test.test",
255+
background: true
256+
},
257+
ready: true,
258+
multikey: false,
259+
multikeyPaths: {
260+
name: BinData(0, 00),
261+
age: BinData(0, 00)
262+
},
263+
head: 0,
264+
prefix: -1
265+
}, {
266+
spec: {
267+
v: 2,
268+
key: {
269+
zipcode: 1.0
270+
},
271+
name: "zipcode_1",
272+
ns: "test.test",
273+
background: true
274+
},
275+
ready: true,
276+
multikey: false,
277+
multikeyPaths: {
278+
zipcode: BinData(0, 00)
279+
},
280+
head: 0,
281+
prefix: -1
282+
}],
283+
prefix: -1
284+
},
285+
idxIdent: {
286+
_id_: "test/index/8-380857198902467499",
287+
name_1_age_1: "test/index/0--6948813758302814892",
288+
zipcode_1: "test/index/3--6948813758302814892"
289+
},
290+
ns: "test.test",
291+
ident: "test/collection/7-380857198902467499"
292+
}
293+
187294
*/
188295
#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kStorage
189296

@@ -647,6 +754,7 @@ const BSONCollectionCatalogEntry::MetaData KVCatalog::getMetaData(OperationConte
647754
return md;
648755
}
649756

757+
650758
//集合相关的元数据信息记录到_mdb_catalog.wt 如创建某个集合对应的数据文件在哪里，索引文件在哪里
651759

652760
//KVCollectionCatalogEntry类的如下相关接口完成对MetaData的更新:updateValidator updateFlags setIsTemp removeUUID addUUID updateTTLSetting indexBuildSuccess
@@ -670,14 +778,23 @@ void KVCatalog::putMetaData(OperationContext* opCtx,
670778
oldIdentMap = obj["idxIdent"].Obj();
671779

672780
// fix ident map
781+
/* 例如已经有_id_索引，和name_1_age_1索引，新键db.test.createIndex( { zipcode: 1}, {background: true} )
782+
idxIdent: {
783+
_id_: "test/index/8-380857198902467499", 注意test对应的是库
784+
name_1_age_1: "test/index/0--6948813758302814892",
785+
zipcode_1: "test/index/3--6948813758302814892"
786+
},
787+
*/
673788
for (size_t i = 0; i < md.indexes.size(); i++) {
674789
string name = md.indexes[i].name();
675790
BSONElement e = oldIdentMap[name];
791+
//先把原有的找出来存入newIdentMap
676792
if (e.type() == String) {
677793
newIdentMap.append(e);
678794
continue;
679795
}
680796
// missing, create new
797+
//新加的索引也追加进去
681798
newIdentMap.append(name, _newUniqueIdent(ns, "index"));
682799
}
683800
b.append("idxIdent", newIdentMap.obj());
@@ -704,28 +821,35 @@ void KVCatalog::putMetaData(OperationContext* opCtx,
704821
fassert(28521, status.isOK());
705822
}
706823

707-
//表重命名后元数据也要更新
824+
//表重命名后元数据也要更新，包括表名 表ident 索引也修改
825+
//KVDatabaseCatalogEntryBase::renameCollection调用
708826
Status KVCatalog::renameCollection(OperationContext* opCtx,
709827
StringData fromNS,
710828
StringData toNS,
711829
bool stayTemp) {
712830
RecordId loc;
713831
BSONObj old = _findEntry(opCtx, fromNS, &loc).getOwned();
714832
{
833+
//存储新表元数据信息
715834
BSONObjBuilder b;
716835

836+
//新表名
717837
b.append("ns", toNS);
718838

719839
BSONCollectionCatalogEntry::MetaData md;
840+
//解析处原表的md信息
720841
md.parse(old["md"].Obj());
842+
//BSONCollectionCatalogEntry::MetaData::rename 表对应索引元数据也需要修改
721843
md.rename(toNS);
722844
if (!stayTemp)
723845
md.options.temp = false;
846+
//把修改后的md组装到b中
724847
b.append("md", md.toBSON());
725848

726849
b.appendElementsUnique(old);
727850

728851
BSONObj obj = b.obj();
852+
//把表对应元数据信息更新为替换后的新的
729853
Status status = _rs->updateRecord(opCtx, loc, obj.objdata(), obj.objsize(), false, NULL);
730854
fassert(28522, status.isOK());
731855
}
@@ -790,7 +914,7 @@ std::vector<std::string> KVCatalog::getAllIdentsForDB(StringData db) const {
790914
// 参考KVStorageEngine::reconcileCatalogAndIdents
791915

792916
//KVStorageEngine::reconcileCatalogAndIdents调用
793-
//获取集群所有的ident信息
917+
//获取集群所有的ident信息，包括表的，也包括index的
794918
std::vector<std::string> KVCatalog::getAllIdents(OperationContext* opCtx) const {
795919
std::vector<std::string> v;
796920

‎mongo/src/mongo/db/storage/kv/kv_database_catalog_entry_base.cpp

Lines changed: 92 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,78 @@
2525
* exception statement from all source files in the program, then also delete
2626
* it in the license file.
2727
*/
28+
/* _mdb_catalog.wt内容
29+
{
30+
ns: "test.test1",
31+
md: {
32+
ns: "test.test1",
33+
options: {
34+
uuid: UUID("520904ec-0432-4c00-a15d-788e2f5d707b")
35+
},
36+
indexes: [{
37+
spec: {
38+
v: 2,
39+
key: {
40+
_id: 1
41+
},
42+
name: "_id_",
43+
ns: "test.test1"
44+
},
45+
ready: true,
46+
multikey: false,
47+
multikeyPaths: {
48+
_id: BinData(0, 00)
49+
},
50+
head: 0,
51+
prefix: -1
52+
}, {
53+
spec: {
54+
v: 2,
55+
key: {
56+
name: 1.0,
57+
age: 1.0
58+
},
59+
name: "name_1_age_1",
60+
ns: "test.test1",
61+
background: true
62+
},
63+
ready: true,
64+
multikey: false,
65+
multikeyPaths: {
66+
name: BinData(0, 00),
67+
age: BinData(0, 00)
68+
},
69+
head: 0,
70+
prefix: -1
71+
}, {
72+
spec: {
73+
v: 2,
74+
key: {
75+
zipcode: 1.0
76+
},
77+
name: "zipcode_1",
78+
ns: "test.test1",
79+
background: true
80+
},
81+
ready: true,
82+
multikey: false,
83+
multikeyPaths: {
84+
zipcode: BinData(0, 00)
85+
},
86+
head: 0,
87+
prefix: -1
88+
}],
89+
prefix: -1
90+
},
91+
idxIdent: {
92+
_id_: "test/index/8-380857198902467499",
93+
name_1_age_1: "test/index/0--6948813758302814892",
94+
zipcode_1: "test/index/3--6948813758302814892"
95+
},
96+
ident: "test/collection/7-380857198902467499"
97+
}
2898
99+
*/
29100
#include "mongo/platform/basic.h"
30101

31102
#include <memory>
@@ -228,7 +299,7 @@ CollectionCatalogEntry* KVDatabaseCatalogEntryBase::getCollectionCatalogEntry(St
228299
return it->second;
229300
}
230301

231-
//获取对该表进行底层KV操作的KVCollectionCatalogEntry
302+
//获取对该表进行底层KV操作的WiredTigerRecordStore
232303
RecordStore* KVDatabaseCatalogEntryBase::getRecordStore(StringData ns) const {
233304
CollectionMap::const_iterator it = _collections.find(ns.toString());
234305
if (it == _collections.end()) {
@@ -371,6 +442,12 @@ void KVDatabaseCatalogEntryBase::reinitCollectionAfterRepair(OperationContext* o
371442
}
372443

373444
//DatabaseImpl::renameCollection调用，集合重命名
445+
//从该接口可以看出，一个表操作需要包含以下信息：
446+
// 1. 更新sizeStorer.wt size元数据文件中对应的表，因为表名已经改变了
447+
// 2. 表名修改后，_mdb_catalog.wt元数据也需要更新，包括表名和ident等，ident是通过表名生成的，表名改了，因此ident也需要修改
448+
// 3. 表名改了后，ident也变了，因此操作该表的WiredTigerRecordStore也需要改变，需要重新生成
449+
// 4. 在cache中根据上面新的表名、新的ident、新的WiredTigerRecordStore生成新的KVCollectionCatalogEntry，该entry在内存cache中缓存起来
450+
// 疑问？为何没有对idxIdent(name_1_age_1: "test/index/0--6948813758302814892")改名，原因是idxIdent中对应的test是库，没有表信息
374451
Status KVDatabaseCatalogEntryBase::renameCollection(OperationContext* opCtx,
375452
StringData fromNS,
376453
StringData toNS,
@@ -379,34 +456,45 @@ Status KVDatabaseCatalogEntryBase::renameCollection(OperationContext* opCtx,
379456

380457
RecordStore* originalRS = NULL;
381458

459+
//获取表信息
382460
CollectionMap::const_iterator it = _collections.find(fromNS.toString());
383461
if (it == _collections.end()) {
384462
return Status(ErrorCodes::NamespaceNotFound, "rename cannot find collection");
385463
}
386464

465+
//获取操作该表的WiredTigerRecordStore
387466
originalRS = it->second->getRecordStore();
388467

468+
//目的表明是否已经存在，存在说明重复了，直接报错
389469
it = _collections.find(toNS.toString());
390470
if (it != _collections.end()) {
391471
return Status(ErrorCodes::NamespaceExists, "for rename to already exists");
392472
}
393473

474+
//原表对应ident
394475
const std::string identFrom = _engine->getCatalog()->getCollectionIdent(fromNS);
395476

477+
//WiredTigerKVEngine::okToRename
478+
//cache中记录的表数据大小，表重命名后，记录数元数据文件sizeStorer.wt也需要对应修改
396479
Status status = _engine->getEngine()->okToRename(opCtx, fromNS, toNS, identFrom, originalRS);
397480
if (!status.isOK())
398481
return status;
399482

400-
status = _engine->getCatalog()->renameCollection(opCtx, fromNS, toNS, stayTemp);
483+
//_mdb_catalog.wt元数据文件中的表名需要更新，元数据也需要更新，现在_mdb_catalog.wt中是新表的元数据信息了
484+
//KVCatalog::renameCollection
485+
status = _engine->getCatalog()->renameCollection(opCtx, fromNS, toNS, stayTemp);
401486
if (!status.isOK())
402487
return status;
403488

489+
//源表名对应的ident也需要改名，因为表名对应ident必须根据指定算法生成，表名改了，indent肯定也就不一样了
404490
const std::string identTo = _engine->getCatalog()->getCollectionIdent(toNS);
405491

406492
invariant(identFrom == identTo);
407493

494+
//获取新表的元数据信息文件_mdb_catalog.wt中的元数据信息md
408495
BSONCollectionCatalogEntry::MetaData md = _engine->getCatalog()->getMetaData(opCtx, toNS);
409496

497+
//清除原表的内存cache信息
410498
const CollectionMap::iterator itFrom = _collections.find(fromNS.toString());
411499
invariant(itFrom != _collections.end());
412500
opCtx->recoveryUnit()->registerChange(
@@ -416,9 +504,11 @@ Status KVDatabaseCatalogEntryBase::renameCollection(OperationContext* opCtx,
416504
opCtx->recoveryUnit()->registerChange(
417505
new AddCollectionChange(opCtx, this, toNS, identTo, false));
418506

507+
//获取操作新表的WiredTigerRecordStore
419508
auto rs =
420509
_engine->getEngine()->getGroupedRecordStore(opCtx, toNS, identTo, md.options, md.prefix);
421510

511+
//新表生成对应新的KVCollectionCatalogEntry cache信息
422512
_collections[toNS.toString()] = new KVCollectionCatalogEntry(
423513
_engine->getEngine(), _engine->getCatalog(), toNS, identTo, std::move(rs));
424514

‎mongo/src/mongo/db/storage/kv/kv_storage_engine.cpp

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -194,7 +194,6 @@ KVStorageEngine::KVStorageEngine(
194194
会有一份对应的WiredTiger需要的元数据维护在WiredTiger.wt表中。因此，事实上这里有两份数据表的列表，
195195
并且在某些情况下可能会存在不一致，比如，异常宕机的场景。因此MongoDB在启动过程中，会对这两份数据
196196
进行一致性检查，如果是异常宕机启动过程，会以WiredTiger.wt表中的数据为准，对_mdb_catalog表中的记录进行修正。这个过程会需要遍历WiredTiger.wt表得到所有数据表的列表。
197-
???? 这里说法有误，应该是以_mdb_catalog.wt为准--yangyazhou todo xxx 感觉有点问题
198197
199198
综上，可以看到，在MongoDB启动过程中，有多处涉及到需要从WiredTiger.wt表中读取数据表的元数据。
200199
对这种需求，WiredTiger专门提供了一类特殊的『metadata』类型的cursor。

‎mongo/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -661,6 +661,8 @@ void WiredTigerKVEngine::cleanShutdown() {
661661
}
662662
}
663663

664+
//KVDatabaseCatalogEntryBase::renameCollection 修改表名的时候走到这里
665+
//cache中记录的表数据大小，表重命名后，记录数元数据文件sizeStorer.wt也需要对应修改
664666
Status WiredTigerKVEngine::okToRename(OperationContext* opCtx,
665667
StringData fromNS,
666668
StringData toNS,

‎mongo/src/mongo/db/storage/wiredtiger/wiredtiger_size_storer.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -174,7 +174,7 @@ void WiredTigerSizeStorer::onDestroy(WiredTigerRecordStore* rs) {
174174
entry.rs = NULL;
175175
}
176176

177-
//WiredTigerRecordStore::_increaseDataSize
177+
//WiredTigerRecordStore::_increaseDataSize WiredTigerKVEngine::okToRename
178178
//修改_entries[uri]的值，也就是修改内存中的值中调用
179179
//WiredTigerSizeStorer::storeToCache和WiredTigerSizeStorer::loadFromCache对应
180180
void WiredTigerSizeStorer::storeToCache(StringData uri, long long numRecords, long long dataSize) {

0 commit comments

Comments
 (0)