MongoDB - E11000 duplicate key error

Summary

In this post, I discuss the symptoms and workaround for an atomicity bug in MongoDB.  This error manifests itself when performing collection updates from multiple threads, with upsert:true, on a unique key.  The bug has been open since 2014 and still unresolved.  https://jira.mongodb.org/browse/SERVER-14322

Bug Set Up

/*jshint esversion: 6 */
 
'use strict';
'use esversion 6';
 
const MongoClient = require('mongodb').MongoClient;
const CONNECTION_URL = 'mongodb://admin:mongo@localhost:27017';
const DB_NAME = 'testDB';
const COLLECTION = 'testCollection';
 
let connection;
let promises = [];
 
function update(i, db,  query, value, upsrt) {
 return db.collection(COLLECTION).updateOne(query, value, {'upsert' : upsrt})
 .then((result) => {
  return new Promise((resolve, reject) => { resolve('i: ' + i + ' time: ' + new Date().getTime());});
 })
 .catch(err => {
  throw err;
 });
}
 
MongoClient.connect(CONNECTION_URL, {useNewUrlParser : true})
.then((result) => {
 connection = result;
 const db = connection.db(DB_NAME);
 
 for (let i=0; i<5 .then="" :="" addtoset="" db="" funds="" i="" id="" indices="" johndoe="" let="" p="" promise.all="" promises.push="" promises="" query="" results="" return="" set="" true="" value=""> {
 results.forEach((result) => {
  console.log('result - ' + result);
 });
})
.catch(err => {
 console.log(err); 
})
.finally(() => {
 console.log('closing db connection');
 connection.close();
});

Explanation

The Node.js code above creates an array of promises, each of which attempts to perform a db update with upsert equal to 'true'.  The promises sent thru Promise.all to execute them all as a batch.

Results

The duplicate key error is generated, as expected.

{ MongoError: E11000 duplicate key error collection: testDB.testCollection index: _id_ dup key: { : "johnDoe" }
    at Function.create (/nas/archive/dev/workspace/blackjack/node_modules/mongodb-core/lib/error.js:43:12)
    at toError (/nas/archive/dev/workspace/blackjack/node_modules/mongodb/lib/utils.js:149:22)
    at coll.s.topology.update (/nas/archive/dev/workspace/blackjack/node_modules/mongodb/lib/operations/collection_ops.js:1399:39)
    at /nas/archive/dev/workspace/blackjack/node_modules/mongodb-core/lib/connection/pool.js:532:18
    at process.internalTickCallback (internal/process/next_tick.js:70:11)
  driver: true,
  name: 'MongoError',
  index: 0,
  code: 11000,
  errmsg:
   'E11000 duplicate key error collection: testDB.testCollection index: _id_ dup key: { : "johnDoe" }',
  [Symbol(mongoErrorContextSymbol)]: {} }
closing db connection

Below is the output of a read of this collection.  One update (#4 index) completed before the error was thrown and halted the execution.

> db.testCollection.find({});
{ "_id" : "johnDoe", "funds" : 400, "indices" : [ 4 ] }

Workaround

function updateWithRetry(i, db,  query, value, upsrt) {
 return db.collection(COLLECTION).updateOne(query, value, {'upsert' : upsrt})
 .then((result) => {
  return new Promise((resolve, reject) => { resolve('i: ' + i + ' time: ' + new Date().getTime());});
 })
 .catch(err => {
  if (err.code === 11000){
   console.log('i: ' + i + ' 11000 error');
   return updateWithRetry(i, db, query, value, false);
  }
  else {
   throw err;
  }
 });
}

Explanation

In this revised update block, I added a check for the 11000 error code.  If that's, in fact, the source of the error, I retry the update with a recursive call.

Results

i: 0 11000 error
i: 2 11000 error
i: 3 11000 error
i: 4 11000 error
result - i: 0 time: 1546649403149
result - i: 1 time: 1546649403146
result - i: 2 time: 1546649403149
result - i: 3 time: 1546649403149
result - i: 4 time: 1546649403149
closing db connection

> db.testCollection.find({});
{ "_id" : "johnDoe", "funds" : 400, "indices" : [ 1, 0, 2, 3, 4 ] }

Source

https://gist.github.com/joeywhelan/6e7acac5cebb79c7bef93234ffa13190

Copyright ©1993-2024 Joey E Whelan, All rights reserved.