Views returning invalid document ids that cause couchbase to hang trying to retrieve them

Couchbase Server
1

We use views a lot to query for data. In the past month our development server views started returning ids that do not exist in the actual database. Not all ids returned from a view do not exist, just some of them. Not only do they not exist, but they cause the server to hang trying to retrieve them. If I search in the couchbase console for an id that just never existed in the db I immediately get the error: “document does not exist”. If I search for one of these problem ids
returned by a view it just hangs for a while and eventually times out. We use the couchbase nodejs sdk to retrieve documents for our web portal, and if it does a getMulti with one of these ids it silently kills the connection. We can search for as many ids as we want after and we will not get any error, but every get will return a timeout until we
restart the connection.

In the couchbase logs I see this error when searching for one of the ids:

[ns_server:error,2016-04-20T19:54:53.749,ns_1@127.0.0.1:<0.19748.148>:menelaus_web:loop:170]Server error during processing: ["web request failed",
                                 {path,
                                  "/pools/default/buckets/<bucket name>/docs/ffb9f1de-d0e1-4711-abb9-e46e5be5ad5e"},
                                 {type,error},
                                 {what,
                                  {case_clause,
                                   {badrpc,
                                    {'EXIT',
                                     {{badmatch,{error,timeout}},
                                      {gen_server,call,
                                       [{'ns_memcached-todos',
                                         'ns_1@127.0.0.1'},
                                        {get,
                                         <<"ffb9f1de-d0e1-4711-abb9-e46e5be5ad5e">>,
                                         9},
                                        180000]}}}}}},
                                 {trace,
                                  [{menelaus_web_crud,handle_get,3,
                                    [{file,"src/menelaus_web_crud.erl"},
                                     {line,166}]},
                                   {request_throttler,do_request,3,
                                    [{file,"src/request_throttler.erl"},
                                     {line,59}]},
                                   {menelaus_web,loop,2,
                                    [{file,"src/menelaus_web.erl"},
                                     {line,149}]},
                                   {mochiweb_http,headers,5,
                                    [{file,
                                      "/home/buildbot/buildbot_slave/ubuntu-1204-x64-30x-builder/build/build/couchdb/src/mochiweb/mochiweb_http.erl"},
                                     {line,94}]},
                                   {proc_lib,init_p_do_apply,3,
                                    [{file,"proc_lib.erl"},{line,239}]}]}]

Is anyone familiar with this error or know what is wrong with our views and how we can repair them?

2

@alexegli

Could you share your view function.

also your using Node.js SDK correct?

3

This is one of the views returning problem ids:

function (doc, meta) {
  if (meta.id.split("_sync")[0] != "" && !doc._deleted && doc._sync && doc._sync.channels
     && doc.type!="docType1" && doc.type!="docType2" && doc.type!='docType3'
     && doc.type!="docType4" && doc.type != "docType5" && doc.type != 'docType6'
     && doc.type != 'docType7' && doc.type != 'docType8' && doc.type != "docType9"
     && doc.type != 'docType10' && doc.type != 'docType11' && doc.type != 'docType12') {
    for (var channel in doc._sync.channels) {
      if (doc._sync.channels.hasOwnProperty(channel)) {
        emit(channel,doc.type);
      }
    }
  }
}

There is no reduce function. I’m emitting a value so that I can filter by it in my nodejs application. To connect in nodejs we use the following function. It’s run once when the application is started, and we pass the connection object to our express router objects that hold onto it for as long as the application is running.

function () {
    var cluster = new couchbase.Cluster('couchbase://127.0.0.1');
    couchDb = cluster.openBucket(g.bucket, function(err) {
        if(err) {
            console.error("Failed to connect to bucket: " + err);
            process.exit(1);
        }

        console.log((new Date()).toISOString() + ' - Couchbase connected to bucket');
    });

    return couchDb;
};

When running a query we use the following syntax (where couchDb is our couchbase connection object):

var query = ViewQuery.from('design_doc', 'viewName')
        .stale(ViewQuery.Update.BEFORE)
        .keys(channels)
        .full_set(true);

    couchDb.query(query, function(err,results) {
        if (err) {
            return callback(err);
        } else if (!results || (results && results.length == 0)) {
            return callback(null,[]);
        }

        var numRows = results.length;
        var ids = [];
        for (var i=0; i < numRows;i++) {
            if (results[i] && (results[i].value == type)) {
                ids.push(results[i].id);
            }
        }

        if (ids.length > 0) {

        couchDb.getMulti(ids, function (numDocsFailed, results) {
            var failedIds = [];
            if (numDocsFailed) {
                logger.error('getMulti failed for ' + numDocsFailed + ' docs');
            }

            var retrievedDocs = [];
            for (var id in results) {
                if (results[id] && results[id].error) {
                    failedIds.push(id);
                } else {
                    var document = results[id].value;
                    if (document && document._sync) {
                        document._id = id;
                        document._rev = document._sync.rev;
                        delete document._sync;
                    }
                    retrievedDocs.push(document);
                }
            }

            callback(failedIds,retrievedDocs);
        });
    } else {
        callback([],[]);
    }
   });
4

@alexegli , How many query per second or minute are you doing?
How many updates per second or minutes are you doing?
How many document do you have in couchbase?

Can you change from BEFORE to NONE?

5

During peak times about 1 query per minute, otherwise about 1 query every 5 or 10 minutes. Our couchbase console reports on average about 80 ops/second, but it peaks to about 110 at some points during the day. Not sure how many of those are updates but I think the majority are reads.We currently have 3259921 documents in the bucket with that view.
I know changing it from BEFORE to NONE will make it more responsive, but for this view we need to have the latest data so I can’t.

6

I was mistaken in which view was returning the problem ids. It is actually this view, which also has the built-in _count reduce function applied to it:

function (doc, meta) {
  if (doc.type == "docType" && meta.id.split("_sync")[0] != "" && !doc._deleted && doc.userId && doc.startDateTime) {
    var start = new Date(doc.startDateTime);
    emit([doc.userId,start.getTime().toString()],null);
  }
}

We query it using this code:

var query = ViewQuery.from('design_doc', 'viewName')
    .stale(ViewQuery.Update.NONE)
    .range([members[i], startDate], [members[i], endDate], true)
    .reduce(false)
    .full_set(true);

couchDb.query(query, function (err, results) {

So we are actually using Update.NONE for this one.