Summary
In this post, I discuss the symptoms and workaround for an atomicity bug in MongoDB. This error manifests itself when performing collection updates from multiple threads, with upsert:true, on a unique key. The bug has been open since 2014 and still unresolved. https://jira.mongodb.org/browse/SERVER-14322
Bug Set Up
/*jshint esversion: 6 */
'use strict';
'use esversion 6';
const MongoClient = require('mongodb').MongoClient;
const CONNECTION_URL = 'mongodb://admin:mongo@localhost:27017';
const DB_NAME = 'testDB';
const COLLECTION = 'testCollection';
let connection;
let promises = [];
function update(i, db, query, value, upsrt) {
return db.collection(COLLECTION).updateOne(query, value, {'upsert' : upsrt})
.then((result) => {
return new Promise((resolve, reject) => { resolve('i: ' + i + ' time: ' + new Date().getTime());});
})
.catch(err => {
throw err;
});
}
MongoClient.connect(CONNECTION_URL, {useNewUrlParser : true})
.then((result) => {
connection = result;
const db = connection.db(DB_NAME);
for (let i=0; i<5 .then="" :="" addtoset="" db="" funds="" i="" id="" indices="" johndoe="" let="" p="" promise.all="" promises.push="" promises="" query="" results="" return="" set="" true="" value=""> {
results.forEach((result) => {
console.log('result - ' + result);
});
})
.catch(err => {
console.log(err);
})
.finally(() => {
console.log('closing db connection');
connection.close();
});
Explanation
The Node.js code above creates an array of promises, each of which attempts to perform a db update with upsert equal to 'true'. The promises sent thru Promise.all to execute them all as a batch.
Results
The duplicate key error is generated, as expected.
{ MongoError: E11000 duplicate key error collection: testDB.testCollection index: _id_ dup key: { : "johnDoe" }
at Function.create (/nas/archive/dev/workspace/blackjack/node_modules/mongodb-core/lib/error.js:43:12)
at toError (/nas/archive/dev/workspace/blackjack/node_modules/mongodb/lib/utils.js:149:22)
at coll.s.topology.update (/nas/archive/dev/workspace/blackjack/node_modules/mongodb/lib/operations/collection_ops.js:1399:39)
at /nas/archive/dev/workspace/blackjack/node_modules/mongodb-core/lib/connection/pool.js:532:18
at process.internalTickCallback (internal/process/next_tick.js:70:11)
driver: true,
name: 'MongoError',
index: 0,
code: 11000,
errmsg:
'E11000 duplicate key error collection: testDB.testCollection index: _id_ dup key: { : "johnDoe" }',
[Symbol(mongoErrorContextSymbol)]: {} }
closing db connection
Below is the output of a read of this collection. One update (#4 index) completed before the error was thrown and halted the execution.> db.testCollection.find({});
{ "_id" : "johnDoe", "funds" : 400, "indices" : [ 4 ] }
Workaround
function updateWithRetry(i, db, query, value, upsrt) {
return db.collection(COLLECTION).updateOne(query, value, {'upsert' : upsrt})
.then((result) => {
return new Promise((resolve, reject) => { resolve('i: ' + i + ' time: ' + new Date().getTime());});
})
.catch(err => {
if (err.code === 11000){
console.log('i: ' + i + ' 11000 error');
return updateWithRetry(i, db, query, value, false);
}
else {
throw err;
}
});
}
Explanation
In this revised update block, I added a check for the 11000 error code. If that's, in fact, the source of the error, I retry the update with a recursive call.
Results
i: 0 11000 error i: 2 11000 error i: 3 11000 error i: 4 11000 error result - i: 0 time: 1546649403149 result - i: 1 time: 1546649403146 result - i: 2 time: 1546649403149 result - i: 3 time: 1546649403149 result - i: 4 time: 1546649403149 closing db connection
> db.testCollection.find({});
{ "_id" : "johnDoe", "funds" : 400, "indices" : [ 1, 0, 2, 3, 4 ] }
Source
Copyright ©1993-2024 Joey E Whelan, All rights reserved.
