Data Modeling and Quering

Couchbase Server
1

Hi,
I have a list of 50,000 structures I wish to save every 10 to 15 seconds.
How would you suggest to model it?

I tried creating a document every 10 secons that holds the 50,000 structures of data however, when I query the bucket through the CBQ web UI the server crushes.

Thanx!

2

could you give me the query that crashes the cbq and a sample document?
thanks

3

I’ve inserted 3 such lists to the bucket and the simple query “select * from bucket” crushed the server not the UI (and the UI also but that is besides the point).
If I run the same query on the same data from the cqb.exe the query succeeds

4

I understand. I’d like to reproduce that on our end. Could you send me the sample script?
thanks
-cihan

5

The generation code is as follows:

//////////////////The data structures://///////////////////
public class Stracture: IEnumerable
{
public List DataList = new List();
public DateTime UpdateTime;

    public IEnumerator GetEnumerator()
    {
        foreach (Data o in DataList)
        {
            if (o == null)
            {
                break;
            }

            yield return o;
        }
    }

    public class Data
    {
        public int ID;

        #region floats
        public float F1;
        public float F2;
        public float F3;
        public float F4;
        public float F5;
        public float F6;
        public float F7;
        public float F8;
        public float F9;
        public float F10;
        #endregion

        #region booleans
        public bool B1;
        public bool B2;
        public bool B3;
        public bool B4;
        public bool B5;
        public bool B6;
        public bool B7;
        public bool B8;
        public bool B9;
        public bool B10;
        #endregion

        #region integers
        public int I1;
        public int I2;
        public int I3;
        public int I4;
        public int I5;
        public int I6;
        public int I7;
        public int I8;
        public int I9;
        public int I10;
        #endregion
    }

/////////////////////////The data generation////////////////////////////////
class DummyWriter
{

    public const int NumOfStructures = 50000;
    static void Main(string[] args)
    {
        Writter = new CouchbaseWriter();
        
        var timer = new Timer(10000);
        timer.Elapsed += (sender, eventArgs) => WriteStructure();
        timer.Start();

        Console.WriteLine("Started Writter");
        Console.WriteLine("Press any key to terminate...");
        Console.ReadKey();
    }

    private static void WriteStructure()
    {
        Writter.WriteStructure(GenerateRandomStructure());
    }

    private static Structure GenerateRandomStructure()
    {
        var s = new Stracture();
        for (var i = 0; i < NumOfStructures ; ++i)
        {
            s.DataList.Add(new Stracture.Data
            {
                ID = i,
                F1 = i,
                I1 = i,
                B1 = 1%2 == 0,
            });
        }
        s.UpdateTime = DateTime.Now;
        return s;
    }

////////////////////// The CouchBase writter/////////////////////////////////
public class CouchbaseWriter
{

    private static List<Uri> SERVERS = new List<Uri>
                {
                new Uri("http://192.168.23.29:8091/pools"),
                new Uri("http://192.168.23.30:8091/pools")
                };

    public static ClientConfiguration config = new ClientConfiguration
    {
        Servers = SERVERS,
        UseSsl = false,
        DefaultOperationLifespan = 1000,
    };

    private static readonly Cluster cluster = new Cluster(config);



    public bool WriteStructure(Structure s)
    {
        bool retval = true;

        using (var bucket = cluster.OpenBucket("theRightBucket"))
        {
            try
            {
                
                var document = new Document<dynamic>
                {
                    Id = DateTime.Now.ToString(),
                    Content = s.DataList.ToDictionary(item => item.ID, item => item)

                };

                var upsert = bucket.Upsert(document);
                if (!upsert.Success)
                {
                    retval = false;
                    Console.WriteLine("couldn't write to db");
                }
            }

            catch (Exception e)
            {
                Console.WriteLine(e.ToString());
                retval = false;
            }
            finally
            {
                if (bucket != null)
                {
                    cluster.CloseBucket(bucket);
                }
            }
        }
        return retval;
    }

////////////////////////////The Query//////////////////////////////////////
run it on the web ui
SELECT * FROM theRightBucket