如何解决带有索引的简单 Mongo 查找的高负载
我有一个 mongoDB，我正在使用 NodeJS（运行 mongoose）进行查询。
在这种特殊情况下，我正在查询一堆集合并将数据作为 CSV 传输到 archiverjs 以创建一个 zip 文件。所以我有一个传入请求，使用 mongoose 和 mongo 游标查询数据，通过管道输送到管道中，该管道将分别以 archiverjs 结束，http 响应将 zip 文件传送给用户。
async function getSortedQueryCursor(...) {
...
const query = MODEL_LOOKUP[fileType]
.find(reducer)
.sort({ [idString]: 'asc' });
return query.cursor();
}
async function getData(...) {
const cursor = await getSortedQueryCursor(...);
return cursor
.pipe(filter1Stream)
.pipe(filter2Stream)
.pipe(filter3Stream)
.pipe(csvStringifyStream);
}
router.post('/:scenarioId',async (request,response) => {
...
const archive = Archiver(...);
archive.pipe(response);
const result = await getData(...);
archive.append(stream,{ name: filepath });
return archive.finalize();
}
只要某个特定集合在游戏中（该集合包含大约 4000 万个文档），查询就会持续很长时间（>15 秒），我可以看到在此期间 100% CPU 上的 mongo 进程。更令人惊讶的是结果集是空的（没有与查询匹配的文档）。
这是一个相当简单的查询：
items.find({ scenarioId: 'ckqf5ulg38gu208eecxlf95fc' },{ sort: { dataId: 1 }
我在 scenarioId 和 dataId 上有索引。如果我在 shell 上运行查询，它会在 30ms 后返回。
explain() 导致：
[
{
"queryPlanner": {
"plannerVersion": 1,"namespace": "data.items","indexFilterSet": false,"parsedQuery": {
"scenarioId": {
"$eq": "ckqf5ulg38gu208eecxlf95fc"
}
},"winningPlan": {
"stage": "SORT","sortPattern": {
"itemId": 1
},"memLimit": 104857600,"type": "simple","inputStage": {
"stage": "FETCH","inputStage": {
"stage": "IXSCAN","keyPattern": {
"scenarioId": 1
},"indexName": "scenarioId_1","isMultiKey": false,"multiKeyPaths": {
"scenarioId": []
},"isUnique": false,"isSparse": false,"isPartial": false,"indexVersion": 2,"direction": "forward","indexBounds": {
"scenarioId": [
"[\"ckqf5ulg38gu208eecxlf95fc\",\"ckqf5ulg38gu208eecxlf95fc\"]"
]
}
}
}
},"rejectedPlans": [
...
]
},"executionStats": {
"executionSuccess": true,"nReturned": 0,"executionTimeMillis": 0,"totalKeysExamined": 0,"totalDocsExamined": 0,"executionStages": {
"stage": "SORT","executionTimeMillisEstimate": 0,"works": 3,"advanced": 0,"needTime": 1,"needYield": 0,"saveState": 0,"restoreState": 0,"isEOF": 1,"sortPattern": {
"dataId": 1
},"totalDataSizeSorted": 0,"usedDisk": false,"works": 1,"needTime": 0,"docsExamined": 0,"alreadyHasObj": 0,\"ckqf5ulg38gu208eecxlf95fc\"]"
]
},"keysExamined": 0,"seeks": 1,"dupsTested": 0,"dupsDropped": 0
}
}
},...
},"serverInfo": {
...
"version": "4.4.6","gitVersion": "72e66213c2c3eab37d9358d5e78ad7f5c1d0d0d7"
},...
}
]
它告诉我（我在解释这些结果方面不是很有经验）查询非常便宜："executionTimeMillisEstimate": 0, 因为它没有运行文档扫描 "docsExamined": 0,。
接下来我连接到 mongo 服务器并运行 db.currentOp({"secs_running": {$gte: 5}}) 以从这边获取一些信息：
{
"type" : "op",...
"clientMetadata" : {
"driver" : {
"name" : "nodejs|Mongoose","version" : "3.6.5"
},"os" : {
"type" : "Linux","name" : "linux","architecture" : "x64","version" : "5.8.0-50-generic"
},"platform" : "'Node.js v14.17.0,LE (unified)","version" : "3.6.5|5.12.3"
},"active" : true,"secs_running" : NumberLong(16),"microsecs_running" : NumberLong(16661409),"op" : "query","ns" : "data.items","command" : {
"find" : "items","filter" : {
"scenarioId" : "ckqf5ulg38gu208eecxlf95fc"
},"sort" : {
"itemId" : 1
},"projection" : {
},"returnKey" : false,"showRecordId" : false,"lsid" : {
"id" : UUID("be3ce18b-5365-4680-b734-543d06418301")
},"$clusterTime" : {
"clusterTime" : Timestamp(1625498044,1),"signature" : {
"hash" : BinData(0,"AAAAAAAAAAAAAAAAAAAAAAAAAAA="),"keyId" : 0
}
},"$db" : "data","$readPreference" : {
"mode" : "primaryPreferred"
}
},"numYields" : 14701,"locks" : {
"ReplicationStateTransition" : "w","Global" : "r","Database" : "r","Collection" : "r"
},"waitingForLock" : false,"lockStats" : {
"ReplicationStateTransition" : {
"acquireCount" : {
"w" : NumberLong(14702)
}
},"Global" : {
"acquireCount" : {
"r" : NumberLong(14702)
}
},"Database" : {
"acquireCount" : {
"r" : NumberLong(14702)
}
},"Collection" : {
"acquireCount" : {
"r" : NumberLong(14702)
}
},"Mutex" : {
"acquireCount" : {
"r" : NumberLong(1)
}
}
},"waitingForFlowControl" : false,"flowControlStats" : {
}
}
任何想法如何提高性能或找到我的应用程序中的瓶颈？由于 mongo 方面的负载很高，并且没有找到/传递给应用程序的文件，我猜是 mongo 遇到了问题......
编辑：我已经使用 db.setProfilingLevel(2) 和 db.system.profile.find().pretty() 从数据库端记录了整个过程。在这里我们可以看到整个集合（或者我误解了 "docsExamined" : 39612167？）被查询：
{
"op" : "query","sort" : {
"dataId" : 1
},"projection" : {
},...
"$db" : "data","keysExamined" : 39612167,"docsExamined" : 39612167,"cursorExhausted" : true,"numYield" : 39613,"nreturned" : 0,"queryHash" : "B7F40289","planCacheKey" : "BADED068","locks" : {
"ReplicationStateTransition" : {
"acquireCount" : {
"w" : NumberLong(39615)
}
},"Global" : {
"acquireCount" : {
"r" : NumberLong(39615)
}
},"Database" : {
"acquireCount" : {
"r" : NumberLong(39614)
}
},"Collection" : {
"acquireCount" : {
"r" : NumberLong(39614)
}
},"flowControl" : {
},"storage" : {
},"responseLength" : 242,"protocol" : "op_msg","millis" : 48401,"planSummary" : "IXSCAN { dataId: 1 }","execStats" : {
"stage" : "CACHED_PLAN","nReturned" : 0,"executionTimeMillisEstimate" : 48401,"works" : 1,"advanced" : 0,"needTime" : 0,"needYield" : 0,"saveState" : 39613,"restoreState" : 39613,"isEOF" : 1,"inputStage" : {
"stage" : "FETCH","filter" : {
"scenarioId" : {
"$eq" : "ckqf5ulg38gu208eecxlf95fc"
}
},"executionTimeMillisEstimate" : 6270,"works" : 39612168,"needTime" : 39612167,"alreadyHasObj" : 0,"inputStage" : {
"stage" : "IXSCAN","nReturned" : 39612167,"executionTimeMillisEstimate" : 2151,"advanced" : 39612167,"keyPattern" : {
"dataId" : 1
},"indexName" : "dataId_1","isMultiKey" : false,"multiKeyPaths" : {
"dataId" : [ ]
},"isUnique" : false,"isSparse" : false,"isPartial" : false,"indexVersion" : 2,"direction" : "forward","indexBounds" : {
"dataId" : [
"[MinKey,MaxKey]"
]
},"seeks" : 1,"dupsTested" : 0,"dupsDropped" : 0
}
}
}
解决方法
（像往常一样）索引似乎没有正确设置。我创建了一个新的（二级？）索引：
{
"dataId" : 1,"scenarioId": 1
}
现在查询在几毫秒内返回......
编辑：仍然让我感到疑惑的是，shell 查询以毫秒为单位返回，而 mongoose 查询花费了很长时间。尽管查询似乎相同（从我的角度来看）mongo 对它们的处理方式不同。
版权声明：本文内容由互联网用户自发贡献，该文观点与技术仅代表作者本人。本站仅提供信息存储空间服务，不拥有所有权，不承担相关法律责任。如发现本站有涉嫌侵权/违法违规的内容， 请发送邮件至 dio@foxmail.com 举报，一经查实，本站将立刻删除。
