Intermittent 5001 panic runtime error

cmchow · March 14, 2022, 5:05am

Hi I’m getting intermittent 5001 panic runtime error using N1QL queries. I’ve reproduced it in the web admin query by executing multiple times which the panic error pop up randomly no matter I am using the same N1QL query or across different queries.

I’m using Couchbase EE 7.0.0 build 5302 and I’ve extracted the following from query.log which seems to be the same error whenever the panic occurs (I’ve edited the bucket name here)

_time=2022-03-14T04:47:09.949+00:00 _level=SEVERE _msg=panic: runtime error: invalid memory address or nil pointer dereference
_time=2022-03-14T04:47:09.949+00:00 _level=SEVERE _msg=request text: <ud>SELECT data.*, comments, user_data FROM `bucket` AS data

LET comments = (
 SELECT count(data1.id) as total FROM `bucket` AS data1
 USE KEYS (select raw meta().id from `bucket` AS cr where cr.data_type='community_reply_data' AND cr.`status`='active' AND cr.`public_status`='active')
 WHERE data1.community_id=data.id
)[0].total,

follower_count = (
 SELECT count(data3.id) as total FROM `bucket` AS data3
 USE KEYS (select raw meta().id from `bucket` AS ud where ud.data_type='user_data' AND ud.`status`='active' AND ud.`following` != [])
 WHERE ARRAY_CONTAINS(data3.`following`, data.`user_id`)
)[0].total,

user_data = (
 SELECT data2.name, data2.tag, data2.user_type, data2.profile_image, follower_count FROM `bucket` AS data2
 USE KEYS 'user_'||TOSTRING(data.user_id)
)[0]

WHERE data.`data_type`='community_data' AND data.`status`='active' AND data.`public_status`='active' data.`create_date` < "2022-03-13T23:59:13Z" and data.`user_id` not in []
order by data.id desc  OFFSET 10 LIMIT 10;</ud>
_time=2022-03-14T04:47:09.949+00:00 _level=SEVERE _msg=query context: <ud></ud>
_time=2022-03-14T04:47:09.949+00:00 _level=SEVERE _msg=stack: goroutine 50202719 [running]:
github.com/couchbase/query/execution.(*Context).Recover(0xc02efcbb80, 0xc0064ed440)
        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/execution/context.go:934 +0xbb
panic(0x22d39e0, 0x378b7a0)
        /home/couchbase/.cbdepscache/exploded/x86_64/go-1.13.7/go/src/runtime/panic.go:679 +0x1b2
github.com/couchbase/query/execution.(*base).close(0xc0064ed440, 0xc02efcbb80)
        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/execution/base.go:269 +0x22a
panic(0x22d39e0, 0x378b7a0)
        /home/couchbase/.cbdepscache/exploded/x86_64/go-1.13.7/go/src/runtime/panic.go:679 +0x1b2
github.com/couchbase/query/execution.(*base).notify(0xc0064ed440)
        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/execution/base.go:980 +0x6e
panic(0x22d39e0, 0x378b7a0)
        /home/couchbase/.cbdepscache/exploded/x86_64/go-1.13.7/go/src/runtime/panic.go:679 +0x1b2
github.com/couchbase/query/value.(*annotatedValue).Field(0xc032719f00, 0xc030e0c5d0, 0x4, 0x28bcde0, 0xc03279b9d0, 0xc032775700)
        <autogenerated>:1 +0x2e
github.com/couchbase/query/value.(*ScopeValue).Field(0xc031aa2030, 0xc030e0c5d0, 0x4, 0x28bc2a0, 0xc0327bab00, 0xc009881b01)
        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/value/scope.go:143 +0x90
github.com/couchbase/query/expression.(*Identifier).Evaluate(0xc030cd0340, 0x28bc2a0, 0xc0327baf80, 0x289f440, 0xc02efcbb80, 0x28bd140, 0xc030d244a0, 0x28bccc0, 0xc032781a78)
        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/expression/identifier.go:63 +0x48
github.com/couchbase/query/expression.(*Field).Evaluate(0xc031926be0, 0x28bc2a0, 0xc0327baf80, 0x289f440, 0xc02efcbb80, 0x28bccc0, 0xc032781a78, 0x0, 0x0)
        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/expression/nav_field.go:55 +0x7c
github.com/couchbase/query/expression.(*Eq).Evaluate(0xc031926c30, 0x28bc2a0, 0xc0327baf80, 0x289f440, 0xc02efcbb80, 0xc009881c90, 0x1d6c799, 0xc03521b9d4, 0xffffffffffffffff)
        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/expression/comp_eq.go:51 +0xf4
github.com/couchbase/query/expression.(*And).Evaluate(0xc031926d20, 0x28bc2a0, 0xc0327baf80, 0x289f440, 0xc02efcbb80, 0x28fbd80, 0xc0327baf80, 0x1, 0x100000000000000)
        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/expression/logic_and.go:50 +0xaf
github.com/couchbase/query/execution.(*Filter).processItem(0xc0064ed440, 0x28fbd80, 0xc0327baf80, 0xc02efcbb80, 0xc0327baf80)
        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/execution/filter.go:72 +0xa4
github.com/couchbase/query/execution.(*base).runConsumer.func1()
        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/execution/base.go:907 +0x2d7
github.com/couchbase/query/util.(*Once).Do(0xc0064ed578, 0xc009881f00)
        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/util/sync.go:55 +0x4a
github.com/couchbase/query/execution.(*base).runConsumer(0xc0064ed440, 0x288b480, 0xc0064ed440, 0xc02efcbb80, 0x28bc2a0, 0xc032719f00)
        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/execution/base.go:852 +0xa2
github.com/couchbase/query/execution.(*Filter).RunOnce(0xc0064ed440, 0xc02efcbb80, 0x28bc2a0, 0xc032719f00)
        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/execution/filter.go:62 +0x5c
github.com/couchbase/query/execution.execOp(0x28d5460, 0xc0064ed440, 0xc02efcbb80, 0x28bc2a0, 0xc032719f00)
        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/execution/base.go:505 +0x54
created by github.com/couchbase/query/execution.(*base).fork
        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/execution/base.go:516 +0x103

goroutine 50202719 [running]:
github.com/couchbase/query/execution.(*Context).Recover(0xc02efcbb80, 0xc0064ed440)
        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/execution/context.go:934 +0xbb
panic(0x22d39e0, 0x378b7a0)
        /home/couchbase/.cbdepscache/exploded/x86_64/go-1.13.7/go/src/runtime/panic.go:679 +0x1b2
github.com/couchbase/query/execution.(*base).close(0xc0064ed440, 0xc02efcbb80)
        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/execution/base.go:269 +0x22a
panic(0x22d39e0, 0x378b7a0)
        /home/couchbase/.cbdepscache/exploded/x86_64/go-1.13.7/go/src/runtime/panic.go:679 +0x1b2
github.com/couchbase/query/execution.(*base).notify(0xc0064ed440)
        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/execution/base.go:980 +0x6e
panic(0x22d39e0, 0x378b7a0)
        /home/couchbase/.cbdepscache/exploded/x86_64/go-1.13.7/go/src/runtime/panic.go:679 +0x1b2
github.com/couchbase/query/value.(*annotatedValue).Field(0xc032719f00, 0xc030e0c5d0, 0x4, 0x28bcde0, 0xc03279b9d0, 0xc032775700)
        <autogenerated>:1 +0x2e
github.com/couchbase/query/value.(*ScopeValue).Field(0xc031aa2030, 0xc030e0c5d0, 0x4, 0x28bc2a0, 0xc0327bab00, 0xc009881b01)
        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/value/scope.go:143 +0x90
github.com/couchbase/query/expression.(*Identifier).Evaluate(0xc030cd0340, 0x28bc2a0, 0xc0327baf80, 0x289f440, 0xc02efcbb80, 0x28bd140, 0xc030d244a0, 0x28bccc0, 0xc032781a78)
        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/expression/identifier.go:63 +0x48
github.com/couchbase/query/expression.(*Field).Evaluate(0xc031926be0, 0x28bc2a0, 0xc0327baf80, 0x289f440, 0xc02efcbb80, 0x28bccc0, 0xc032781a78, 0x0, 0x0)
        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/expression/nav_field.go:55 +0x7c
github.com/couchbase/query/expression.(*Eq).Evaluate(0xc031926c30, 0x28bc2a0, 0xc0327baf80, 0x289f440, 0xc02efcbb80, 0xc009881c90, 0x1d6c799, 0xc03521b9d4, 0xffffffffffffffff)
        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/expression/comp_eq.go:51 +0xf4
github.com/couchbase/query/expression.(*And).Evaluate(0xc031926d20, 0x28bc2a0, 0xc0327baf80, 0x289f440, 0xc02efcbb80, 0x28fbd80, 0xc0327baf80, 0x1, 0x100000000000000)
        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/expression/logic_and.go:50 +0xaf
github.com/couchbase/query/execution.(*Filter).processItem(0xc0064ed440, 0x28fbd80, 0xc0327baf80, 0xc02efcbb80, 0xc0327baf80)
        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/execution/filter.go:72 +0xa4
github.com/couchbase/query/execution.(*base).runConsumer.func1()
        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/execution/base.go:907 +0x2d7
github.com/couchbase/query/util.(*Once).Do(0xc0064ed578, 0xc009881f00)
        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/util/sync.go:55 +0x4a
github.com/couchbase/query/execution.(*base).runConsumer(0xc0064ed440, 0x288b480, 0xc0064ed440, 0xc02efcbb80, 0x28bc2a0, 0xc032719f00)
        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/execution/base.go:852 +0xa2
github.com/couchbase/query/execution.(*Filter).RunOnce(0xc0064ed440, 0xc02efcbb80, 0x28bc2a0, 0xc032719f00)
        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/execution/filter.go:62 +0x5c
github.com/couchbase/query/execution.execOp(0x28d5460, 0xc0064ed440, 0xc02efcbb80, 0x28bc2a0, 0xc032719f00)
        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/execution/base.go:505 +0x54
created by github.com/couchbase/query/execution.(*base).fork
        /home/couchbase/jenkins/workspace/couchbase-server-unix/goproj/src/github.com/couchbase/query/execution/base.go:516 +0x103

Is this a bug or a device specific memory issue which I can fix on my end?

dh · March 14, 2022, 1:05pm

This looks like a bug to me. You note it is random and this is problematic for identifying the issue - I’ve tried to reproduce this faking up some data to qualify the statement’s criteria but it always runs without issue. (I run it a few hundred times with a few sessions in parallel.) The actual code area where the fault occurs is simple, well exercised and unchanged across several versions so it is something to do with the input/set-up.

Could you provide the index definitions for the bucket and an explain plan of the statement?

Further, would it be possible to provide (suitably redacted) sample data that may help reproduce the issue, in case I’ve missed something necessary?

vsr1 · March 14, 2022, 1:55pm

Could you please try with latest 7.0.3 . It looks to me some race condition. Try couple of times without OFFSET and LIMIT see if you can repro.

FYI: This is not specific to issue you are looking. But thought useful.
The query is pagination and involves lot of aggregate subqueries. This involves lot of unnecessary work.
One option will be find what the documents required and then only do aggregate subqueries (Main query USE KEYS using covered index identify 10 documents, Fetch those 10 documents. Aggregate queries doing in projection defers until it required).

CREATE INDEX ix1 ON `bucket`(status, public_status, create_date, user_id, id) WHERE data_type "community_data";
CREATE INDEX ix2 ON `bucket`(status, public_status, community_id) WHERE data_type "community_reply_data";
CREATE INDEX ix3 ON `bucket`(status, DISTINCT following, id) WHERE data_type "user_data";

SELECT  data.*,
     (SELECT RAW COUNT(1)
      FROM `bucket` AS c
      WHERE c.data_type = "community_reply_data"
            AND c.status = "active"
            AND c.public_status = "active"
            AND c.community_id = data.id)[0] AS comments,
     ( WITH follower_count AS (( SELECT RAW COUNT(ud.id)
                                FROM `bucket` AS ud
                                WHERE ud.data_type = "user_data"
                                      AND ud.`status` = "active"
                                      AND ANY v IN ud.`following` SATISFIES v = data.`user_id` END)[0])
      SELECT u.name, u.tag, u.user_type, u.profile_image, follower_count
      FROM `bucket` AS u USE KEYS "user_"||TOSTRING(data.user_id)) AS user_data
FROM `bucket` AS data USE KEYS ( SELECT RAW META(d).id
                                 FROM `bucket` AS d
                                 WHERE d.`data_type` = 'community_data'
                                       AND d.`status` = 'active'
                                       AND d.`public_status` = 'active'
                                       AND d.`create_date` < "2022-03-13T23:59:13Z"
                                       AND d.`user_id` NOT IN []
                                 ORDER BY d.id DESC
                                 OFFSET 10
                                 LIMIT 10);

Topic		Replies	Views
5001 Query Error Couchbase Server query , n1ql , dot-net	4	377	February 14, 2024
Panic: runtime error: invalid memory address or nil pointer dereference Couchbase Server query , n1ql	10	4435	February 26, 2023
Error: Panic: runtime error: invalid memory address or nil pointer dereference SQL++ query , n1ql	9	2311	May 12, 2021
Panic runtime error while running Hello Couchbase application Go SDK	3	1647	May 13, 2020
N1QL Not working it throws panic but sdk collection.get working whats wrong? Go SDK n1ql	8	1460	May 21, 2020

Intermittent 5001 panic runtime error

Related topics