Why do these documents sync to mobile?

I’m trying to get the import filters and sync function to do what I need. So I’m sending test documents back and forth. However, I have a question as to why certain documents (type: Image) are replicated to mobile?

Obviously, I’m missing something in my import filter and/or sync. formula. I can see that as soon as I start the replication (push-pull as a logged in user) I recieve Image documents…

This is my import filter:

            function(doc) {
               // Some document types not allowed on mobile
                if (doc.type == 'EnvLake' || doc.type == 'EnvMeasurement' || doc.type == 'ActivityLog' || doc.type == 'Image') {
                    return false;
                }
                if ((doc.type == 'FishingTrip' || doc.type == 'Catch') && typeof doc.clubonlykey !== 'undefined' && doc.clubonlykey != '') {
                    return false;
                }
                return true;
            }

And this is my sync function (still work in progress):

function (doc, oldDoc) {
                function _log(t) {
                    console.log('SG: ' + t);
                }
                function _getUserKey(d) {
                    var key = null;
                    if (d) {
                        if (d.type == 'User') {
                            key = d.key;
                        } else if (d.type == 'FishingTrip' || d.type == 'Catch' || d.type == 'Photo' || d.type == 'Private' || d.type == 'Image' || d.type == 'Feedback') {
                            // TODO: Not sure about Feedback here....??
                            key = d.userkey;
                        }
                    }
                    return key;
                }
            
              //if (doc && doc._deleted && oldDoc) {
                if (doc && doc._deleted) {
                    // Doc. deleted -> if public then require
            /*
                    var userkey = _getUserKey(oldDoc);
                    if (userkey != null) {
                        requireUser(userkey);
                    } else {
                    channel('!');
                        requireAdmin();
                    }
            */
                    _log('doc deleted, id: ' + (doc._id || 'no id!') + ', ' + (oldDoc ? ('old key=' + oldDoc.key + ', userkey=' + oldDoc.userkey) : 'no oldDoc'));
                    return;
                }
                _log('doc id: ' + (doc._id || 'no id!') + ', ' + (oldDoc ? ('old key=' + oldDoc.key + ', userkey=' + oldDoc.userkey) : 'no oldDoc'));
                // Document type is mandatory
                if (typeof doc.type === 'undefined') {
                    throw ({ forbidden: "Document type is required. id=" + JSON.stringify(doc) });
                }
                //  --> Moved to import filter!
				/*
                // Some document types not allowed to sync to mobile
                if (doc.type == 'EnvLake' || doc.type == 'EnvMeasurement' || doc.type == 'ActivityLog' || doc.type == 'Image') {
                    throw ({ forbidden: "Document type " + doc.type + " not allowed to sync to mobile..." });
                }
                // Some document types not allowed on mobile
                if(oldDoc != null) {
                    if (doc.type == 'EnvLake' || doc.type == 'EnvMeasurement' || doc.type == 'ActivityLog' || doc.type == 'Image') {
                        throw ({ forbidden: "Document type " + doc.type + " not allowed to sync to mobile..." });
                    }
                }
                */
                // Document key is mandatory
                if (typeof doc.key === 'undefined') {
                	throw ({ forbidden: "Document key is required. id=" + doc._id });
                }
                // Allow anyone to create a Feedback on the server
                if (oldDoc == null && doc.type == 'Feedback') {
                    _log('Created feedback: ' + (doc._id || 'no id!') + ', key: ' + doc.key + ', user: ' + doc.userkey);
                    return;
                }
                // Allow anyone to create a new Image or delete an existing on the server
                if (doc.type == 'Image') {
                    _log((oldDoc == null ? 'Created' : 'Deleted') + ' image: ' + (doc._id || 'no id!') + ', key: ' + doc.key + ', user: ' + doc.userkey);
                    return;
                }
            
                // All public docs are available in the app
                if (doc.ispublic) {
                    _log('public, id: ' + (doc._id || 'no id!'));
                    channel('!');
                }
                // All non-club fishing trips and catches are available (for stats)
                if ((doc.type == 'FishingTrip' || doc.type == 'Catch') && doc.clubonlykey === 'undefined') {
                    _log('non-club trips, id: ' + (doc._id || 'no id!'));
                    channel('!');
                }
                // All non-specific user info is available (for stats)
                if (doc.type == 'User') {
                    _log('User doc, id: ' + (doc._id || 'no id!'));
                    channel('!');
                }
            
                // Only non-public docs "owned" by user can be replicated
                var userkey = _getUserKey(doc);
                if (userkey != null) {
                    if (oldDoc != null) {
                        // Update
                        if (oldDoc.type != doc.type) {
                    		_log('Update: doc.type=' + doc.type + ', oldDoc.type=' + oldDoc.type);
                            throw ({ forbidden: "Can't change doc type" });
                        }
                        if (oldDoc.key != doc.key) {
                    		_log('Update: doc.key=' + doc.key + ', oldDoc.key=' + oldDoc.key);
                            throw ({ forbidden: "Can't change doc key" });
                        }
                        if (oldDoc.userkey && oldDoc.userkey != doc.userkey) {
                    		_log('Update: doc.userkey=' + doc.userkey + ', oldDoc.userkey=' + oldDoc.userkey);
                            throw ({ forbidden: "Can't change user key" });
                        }
                    }
                    _log('User owned, id: ' + (doc._id || 'no id!') + ', type: ' + doc.type + ', user: ' + doc.userkey);
                    if(doc.ispublic){
                        requireAdmin();
                    }
                    requireUser(userkey);
                    channel('channel.' + userkey);
                    access(userkey, 'channel.' + userkey);
                }
}            

The thing is that I do want to send an Imagefrom mobile to the server as I use replication to upload the photos/images (and purge them locally afterwards).

Following another question about sync. I could “guess” that I may need to add these images to a “fictive” channel that no user is using… (e.g. named “server-only”). But if this is so then I’m a little confused about the role of the import filters?

I also want to be able to delete an image on the server (considering creating a new Image in the seldom event and delete it afterwards to send the deletion to the server - or if that won’t work then I need some “flag” to disable replication to the server after the first replication (so I can purge the base64 fields with the photo in them…)

If these image documents are created via the mobile app, they will be available for sync to other authorized clients. The import filtering is for documents created on Couchbase Server side. If you don’t want any user other than the one that created it to have access to it, then you should have such private documents in user specific private channels. You can learn more about shared bucket access and import processing here.

FYI: CBL pull (import) filters are inefficient if they reject lots of documents, since those documents have to be downloaded first. Using channels is the preferred way to filter out documents from being synced to mobile devices.

Import filters are useful for:

  • Filtering in P2P replication or local-to-local replication, where there are no channels
  • Schema validation of incoming documents (again, mostly useful in P2P scenarios)
  • Ad-hoc filtering in ways that aren’t possible with channels, or if the app developer isn’t able to get channels assigned for this purpose (i.e. “Sorry, the server devs won’t be able to get to that until March…”)

To avoid confusion, want to add that the reporter is referring to the Sync Gateway import filters (and not the CBL pull filters)

Ok, I’m a little confused…

I showed the filter function (as set up in the sync_gateway.json file). Is this a bad way of doing it? I did it following an advice from @priya.rajagopal in another thread - but I may have misunderstood it???

Then I see different aadvices about the sync function. Whether it only handles doccuments sent to the mobile client - or (as my experience is) that it matters in both directions… But this is not very clear from your documentation (it may be “implicitly” if you have followed the products from way back… - but I started from CBL 2 and CB Server 5). I have many years of experience with Notes/Domino and replication - which is very mature. So it may just be that CB is not providing quite the same full-featured replication yet - and I’m expecting this?

Ok, so I have a few different requirements:

  1. Some documents (e.g. ActivityLog), should never be sent to or created on mobile
  2. Some documents (e.g. FishingTrip) should all replicate to mobile (for stats purposes etc.) - but user should only be allowed to change/create/delete own fishing trips (there is a userkey field on each document)
  3. Then I have some documents (e.g. Image) that I want to create on mobile, send to server, remove from mobile once transferred to the server - and then load from a URL (which all other clients will do - if they want to see those images). Right now, I create an Image doc. and then purge it once sent to the server - but my challenge is if a user wants to delete an image… How do I send that info to the server? Can I create a new doc. with the same id (I know what it was) and delete it subsequently to replicate the deletion?

So I guess I would like to turn the question around and ask - what would be the best way to implement the above - using filters and sync. functions?

As I tried to clarify above, Jens was referring to Couchbase Lite side pull filters (which sometimes gets referred to as import filters). That is completely different than Sync Gateway import filters that you specify in the config file . I explained here what the filters do and what the recommended approach it.

  1. Use sgw import filters to stop sync to clients . To stop creation of docs on mobile but you can use sync function to reject
  2. Channels with sync gateway enforcing access control
  3. Channels . More in docs
    Couchbase Mobile supports of the fine grained filtering, access control and data routing that you are looking for, Again, I would strongly recommend that you review the documentation links that was shared in my earlier post(s) .
    Just wanted to add : we are restructuring the documentation so its bit easier to locate stuff ( menu names are bit more intuitive etc)

Thanks for your reply @priya.rajagopal

However, if you look at my original post, you’ll see that I have included “Image” in my import filter - but… I do receive a bunch of image documents on mobile? So back to my original question - why…?

Because you created the document on mobile CBL side… right ? In which case, the documents are coming in from the clients and will be synced via the sync gateway to all authorized clients. If you want to prevent access, then use channels.
If these documents were created on the server side , the import filter will prevent the sync gateway from importing it and processing it to make it “sync-ready”…so these documents will be hidden from all CBL clients. Remember import refers to the import of documents from a shared CBS bucket .

Hmmm… no, for the current issue it seems as it’s all the Images that are “public” (or have userkey equal to the user id). These documents were created back before I migrated to Couchbase… - so I guess these documents should have been considered as “server crreated” - and that obviously doesn’t happen… - or I have broken something else?? :slight_smile:

Hmmm…Its hard to say what’s going on- quite a few moving parts.
Some thoughts

  • It may have something to do with when you migrated the documents over and when you setup the filters (i.e. its possible that an earlier instance of the sync gateway had already imported these documents before you made the import filter change that was suggested ).

  • Presuming you have only one sync gateway setup as import node (i.e. for instance, you don’t have another sync gateway node that’s allowing these documents )

  • I’d also take a step back. The sync function has quite a bit going on. You specified three requirements. I’d suggest that you work through the first one and then extending the sync function to include other access control policies with channels. It would be much simpler to debug that way. So, I’d suggest that you start with default sync function with the required import filter and ensure that documents that shouldn’t get imported don’t/

To confirm that the your config works- please add a filter for a new document type, say “foo”. Create the document on couchbase server directly (SDK or UI) with type of “foo” and make sure that clients don’t see it.

Well, I’m glad that you think it is complicated :grinning: - that makes me feel a little better :innocent:

I did actually build the sync function the way you suggested - but as it was a big black box with sporadic documentation then it’s been a challenge to say the least - and I have asked several questions here about that. So you’re right I added the import filters after I had originally had them sync to the mobile. I thought I was going to make all photos (i.e. “Image” docs) available offline - but it just took up way too much space. So I changed that concept.

Following another question about creating a “fresh” demo environment from my production environment, I understand that the sync. info is part of the data (in the meta data) - so if I create a fresh demo environment then I suppose I should get rid of that “history”?

In the demo environment I only have one Sync. Gateway (planning on having two in production) - so right now it’s a simple environment.

I really feel there is a need for a complete description of the sync. function (and import filter) with some well covering examples. It may be out there somewhere - but I’ve not been able to find it. And I have read the documentation (more than once) and still I have some problems with understanding some of the concepts.

The major issue I have is understanding the “direction” in the sync. So when I design it I need to understand when docis coming from the server to mobile (and then oldDoc then is the doc on mobile that is about to be updated). I can understand that setting the channels is an example of the “direction” - so if a document is not added to the public ("!") channel or the user’s own channel - then it is not sent to mobile… But then the big questions is when (or perhaps more precisely what part of the code handles…) the direction is from mobile to server? This is where I would need requireUser() and requireAdmin() to control what changes a user can send (creations, updates, deletions).

But thanks for some hints to understanding why it may behave differently from what I expect. I’ll have to step back, create a clean environment and then test step by step again :face_with_raised_eyebrow: (time is never available for that is it?)

Following another question about creating a “fresh” demo environment from my production environment, I understand that the sync. info is part of the data (in the meta data) - so if I create a fresh demo environment then I suppose I should get rid of that “history”?

That’s right. Start with a clean bucket - get rid of the old sync data.

I really feel there is a need for a complete description of the sync. function (and import filter) with some well covering examples. It may be out there somewhere - but I’ve not been able to find it. And I have read the documentation (more than once) and still I have some problems with understanding some of the concepts.

I agree that it can be improved. We have started restructuring some of our documentation around the sync gateway - hopefully once thats in place, some of the above concepts are clearer. It will be an iterative process. In addition to making some improvements to our docs website , we’ve started with some best practices/patterns blogs and what you described may be a good candidate for a future blog on sync gateway.

The major issue I have is understanding the “direction” in the sync. So when I design it I need to understand when doc is coming from the server to mobile (and then oldDoc then is the doc on mobile that is about to be updated). I can understand that setting the channels is an example of the “direction”

Doc always refers to the new revision/mutation that is coming in through the Sync Gateway . oldDoc refers to the current version that’s on the server. So when a document is created , oldDoc is always nil. It’s discussed in the first section here but maybe another case of hard-to-find.

  • so if a document is not added to the public (“!”) channel or the user’s own channel - then it is not sent to mobile…

Channels are for data segregation/routing and access control - not direction. The concepts are described here
Well, in addition to what you described, all documents are by default added to the all-docs channel- That channel is created by system. So if a user is granted access to the all-docs/star channel, then they have access to the documents. Its typically not used in production but wanted to clarify.
Think of this flow
Documents ->[ assigned to] → Channels.
Users → [granted access to] —> Channels .
Roles —> [granted access to] —> channels.
Users —> [assigned] → Roles.
In other words, a user or a role goes through a channel to get access to a document. Channel dictates the access. No direction involved.

But then the big questions is when (or perhaps more precisely what part of the code handles… ) the direction is from mobile to server? This is where I would need requireUser() and requireAdmin() to control what changes a user can send (creations, updates, deletions).

The direction is specified on the client side when you set up the replication - there is a replicatorType that determines direction
Again, don’t get confused with access control and directionality of sync.

So doc is the mobile document in a push replication where oldDoc is then the server doc? And in a pull replication doc is the server documentt and oldDoc is the mobile doc?

Obviously, this is important in designing the logic in the sync. function - and that’s why I want to be able to identify the direction so I can specify different rules for e.g. a document created from mobile and another document created on the server.

… looking at your response you say oldDoc refers to the current version that’s on the server. - where do I see new documents created on the server - and apply logic to them? Is that through adding them to channels?

So doc is the mobile document in a push replication where oldDoc is then the server doc? And in a pull replication doc is the server documentt and oldDoc is the mobile doc?

No…no… Sync function is called on a write (push replication , REST API write or import from server). It won’t be called on a pull replication (which is a read).

There is no distinction w.r.t the source of a document when it arrives at the Sync Gateway. The write could be a server side write, a REST API write or something from CBL client. The doc refers to the new revision corresponding to the write (and source of write could be any of the ones I mention above). The oldDoc is the revision of the document that is being replaced (naturally this revision if it exists would be on the server)

Obviously, this is important in designing the logic in the sync. function - and that’s why I want to be able to identify the direction so I can specify different rules for e.g. a document created from mobile and another document created on the server.

As you can see from above, that’s not how it works. Sync Function enforces access control on every write coming in through Sync Gateway. Note that the same document could be mutated from both server and mobile so you can’t really define the ACLs by source of write . Once again, quoting from the docs .
The sync function is called every time a new revision/update is made to a document, and the changes to channels and access made by the sync function are tied to that revision . If the document is later updated, the sync function will be called again on the new revision, and the new channel assignments and user/channel access replace the ones from the first call.

( Quick primer on document revision tree - Every doc in Couchbase is a revision tree. When you create a doc, you create a revision with a unique revId. Every change to doc adds a new revision with a unique revId. So the doc and oldDoc really refer to revisions of the document)

I know you indicated that you’ve read the docs but this stuff is core to sync, so would recommend another read.

Thanks for trying to explain this!

So when the mobile app does a “pull” it basically ignores (or doesn’t run through) the sync. formula as you see that as a read - I guess my confusion is that I also see that as a write to the mobile database…

And secondly, a write on the server (e.g. through the Java SDK in my web app) is also a write where source and target destination is the same “place” (ie. the server) - that is not the way I have understood it.

I do know that trying to read the docs where some overall assumption is different from what I understand will make it difficult to understand the details. I would really have liked to see examples of these three “ways” into the sync. function - and what the result (state of the doc) of running the code would have on the docs afterwards…

I’ll go do some tests now - to see if I have understood it now :slight_smile:

So when the mobile app does a “pull” it basically ignores (or doesn’t run through) the sync.`

Yes . The access control policies are applied during document write . So reads/ pulls are extremely performant

I guess my confusion is that I also see that as a write to the mobile database…

You should view this from the perspective of Sync Gateway.

And secondly, a write on the server (e.g. through the Java SDK in my web app) is also a write where source and target destination is the same “place” (ie. the server) - that is not the way I have understood it.

Yes. That’s why don’t think of source and destination or direction or client type. Think from the perspective of Sync Gateway - it gets a document and processes it for sync. Because sync gateway has to process the write to apply ACLs and other sync metadata. That’s the power of the sync technology. You get unified access control policies applied regardless of source of write which is critical to ensure consistent access. I discussed import processing/shared bucket access in earlier response and in described in this document that I linked earlier.

Thanks for the replies, @priya.rajagopal - and yes, I think the most important lesson (to me anyway) is to always look at the sync. function from the perspective of the sync.gateway!

Now, I’ve been playing with a few different scenarios today. I’ld like to have your input to a c0uple of situations:

First my import filter:

           function(doc) {
                if (doc.type == 'A1') {
                    return false;
                }
                return true;
            }

That basically just says: Don’t send A1 type docs to mobile… This seems to work fine.

Then here is the sync function:

			function (doc, oldDoc) {
                function _log(t) {
                    // Does not work - yet...
                    console.log('SG: ' + t);
                }
                function _getUserKey(d) {
                    var key = null;
                    if (d) {
                        if (d.type == 'User') {
                            key = d.key;
                        } else {
                            key = d.userkey;
                        }
                    }
                    return key;
                }
            
                if (doc && doc._deleted) {
                    // Doc. deleted -> if public then require
					if(oldDoc){
	                    var userkey = _getUserKey(oldDoc);
	                    _log('delete doc id: ' + doc._id + ', userkey=' + userkey);
    	                if (userkey != null) {
                    		channel('channel.' + userkey);
                    		access(userkey, 'channel.' + userkey);
		                    requireUser(userkey);
            	        } else {
                	        channel('!');
                    	    requireAdmin();
						}
                    }
                    _log('doc deleted, id: ' + (doc._id || 'no id!') + ', ' + (oldDoc ? ('old key=' + oldDoc.key + ', userkey=' + userkey) : 'no oldDoc'));
                    return;
                }
                _log('doc id: ' + (doc._id || 'no id!') + ', ispublic: ' + doc.ispublic + ', userkey=' + userkey + ', ' + (oldDoc ? (oldDoc._deleted ? 'oldDoc is deleted' : ('old key=' + oldDoc.key + ', oldDoc.userkey=' + oldDoc.userkey + ' update')) : ' creation'));
                // Document type is mandatory
                if (typeof doc.type === 'undefined') {
                    throw ({ forbidden: "Document type is required. id=" + JSON.stringify(doc) });
                }
                // Document key is mandatory
                if (typeof doc.key === 'undefined') {
                throw ({ forbidden: "Document key is required. id=" + doc._id });
                }
                // Allow anyone to create a A2 on the server - this may not be necessary?
                if (oldDoc == null && doc.type == 'A2') {
                    _log('Created A2: ' + (doc._id || 'no id!') + ', key: ' + doc.key + ', userkey: ' + doc.userkey);
                    //return;
                }
                // Document type A1 - sync disabled...?
                if (doc.type === 'A1' && doc.issyncdisabled) {
                    throw ({ forbidden: "Sync. disabled for id=" + JSON.stringify(doc) });
                }
            
                // All public docs are available in the app
                if (doc.ispublic) {
                    _log('public, id: ' + (doc._id || 'no id!'));
                    channel('!');
                }
            
                // Only non-public docs "owned" by user can be replicated
                var userkey = _getUserKey(doc);
                if (userkey != null) {
                    if (oldDoc != null && ! oldDoc._deleted) {
                        // Update
                        if (oldDoc.type != doc.type) {
                            throw ({ forbidden: "Can't change doc type" });
                        }
                        if (oldDoc.key != doc.key) {
                            throw ({ forbidden: "Can't change doc key" });
                        }
                        if (oldDoc.userkey && oldDoc.userkey != doc.userkey) {
                            throw ({ forbidden: "Can't change user key" });
                        }
                    }
                    _log('User owned, id: ' + (doc._id || 'no id!') + ', type: ' + doc.type + ', user: ' + userkey);
                    channel('channel.' + userkey);
                    access(userkey, 'channel.' + userkey);
					requireUser(userkey);
                }
             }

This basically controls that you cannot create/save docs without a key and type - and you cannot change those nor the userkey. I have a couple of questions in relation to this function:

  • Is there a better way to disable replication for a specific doc than I have done? The use case is that I have Images as base64 encoded text. These take up quite some space so once sent to the server I would like them to be “disappear” locally (and just read the photos via a url where they are cached locally). I could purge them - but then I cannot find a way to later delete an image. So by throwing an exception I can clean out the bulk data and leave a “stub” doc that I can activate by removing the disabled flag. This will not work for existing photos -but I cannot fix that.

  • When should I use the Channel and Access functions? _I have added them to the code where I delete a document (and when I save/create a document with a userkey). I think I need them in the latter case - but not in the first? _

And finally, thank you for being patient with me. I now know why I found it so difficult to get the concept fit into my mind - but I think it is way closer now (and its a major improvement that we can have our own log statements written to the SG log!)

HI @priya.rajagopal

Ok, I thought I understood it now, but apparently not… :face_with_raised_eyebrow:

In my real filter I have:

            function(doc) {
               // Some document types not allowed on mobile
               if (doc.type == 'EnvLake' || doc.type == 'EnvMeasurement' || doc.type == 'ActivityLog' || doc.type == 'Feedback' || doc.type == 'Image') {
                return false;
                }
                if ((doc.type == 'FishingTrip' || doc.type == 'Catch') && typeof doc.clubonlykey !== 'undefined' && doc.clubonlykey != '') {
                    return false;
                }
                return true;
            }

So I would expect “EnvLake” etc. not to be sent to mobile… Having made changes to the sync config file I stop and start the sync. gateway, via PostMan I take the database offline, do a resync, take it back online. Then I removed the app from my local simulator prior to starting the app and thus get a “clean” sync…

Well, as you can see from this screen shot some of the datatypes that shouldn’t go to mobile actualle goes there…
image
This is the sync. formula - but is that in play already now (with the steps I mentioned above)?

			function (doc, oldDoc) {
                function _log(t) {
                    // Write to sg_info.log
                    console.log('SG: ' + t);
                }
                function _getUserKey(d) {
                    var key = null;
                    if (d) {
                        if (d.type == 'User') {
                            key = d.key;
                        } else {
                            key = d.userkey;
                        }
                    }
                    return key;
                }
            
                if (doc && doc._deleted) {
                    // Doc. deleted -> if public then require
					if(oldDoc){
	                    var userkey = _getUserKey(oldDoc);
	                    _log('delete doc id: ' + doc._id + ', userkey=' + userkey);
    	                if (userkey != null) {
		                    requireUser(userkey);
            	        } else {
                    	    requireAdmin();
						}
                    }
                    _log('doc deleted, id: ' + (doc._id || 'no id!') + ', ' + (oldDoc ? ('old key=' + oldDoc.key + ', userkey=' + userkey) : 'no oldDoc'));
                    return;
                }
                _log('doc id: ' + (doc._id || 'no id!') + ', ispublic: ' + doc.ispublic + ', userkey=' + userkey + ', ' + (oldDoc ? (oldDoc._deleted ? 'oldDoc is deleted' : ('old key=' + oldDoc.key + ', oldDoc.userkey=' + oldDoc.userkey + ' update')) : ' creation'));
                // Document type is mandatory
                if (typeof doc.type === 'undefined') {
                    _log('Document type missing: ' + JSON.stringify(doc));
                    throw ({ forbidden: "Document type is required. id=" + doc._id });
                }
                // Document key is mandatory
                if (typeof doc.key === 'undefined') {
                    _log('Document key missing: ' + JSON.stringify(doc));
                    throw ({ forbidden: "Document key is required. id=" + doc._id });
                }
                // Document type Image - sync disabled... (why have type?)
                if (doc.type === 'Image' && doc.issyncdisabled) {
                    throw ({ forbidden: "Sync. disabled for id=" + doc._id });
                }
            
                // All public docs are available in the app
                if (doc.ispublic) {
                    _log('public, id: ' + (doc._id || 'no id!'));
                    channel('!');
                }

                // Update: Cannot allow change of type or key
                if (oldDoc != null && !oldDoc._deleted) {
                    // Update
                    if (oldDoc.type != doc.type) {
                        throw ({ forbidden: "Can't change doc type" });
                    }
                    if (oldDoc.key != doc.key) {
                        throw ({ forbidden: "Can't change doc key" });
                    }
                }

                // Only non-public docs "owned" by user can be created/updated (and replicated)
                var userkey = _getUserKey(doc);
                if (userkey != null) {
                    if (oldDoc != null && ! oldDoc._deleted) {
                        // Update
                        if (oldDoc.userkey && oldDoc.userkey != doc.userkey) {
                            throw ({ forbidden: "Can't change user key" });
                        }
                    }
                    _log('User owned, id: ' + (doc._id || 'no id!') + ', type: ' + doc.type + ', user: ' + userkey);
                    channel('channel.' + userkey);
                    access(userkey, 'channel.' + userkey);
					requireUser(userkey);
                } else {
                    // Creation/update without user
                    if(doc.type !== 'Feedback' && doc.type !== 'Observation'){
-->                     _log('document type cannot be created without user key: ' + JSON.stringify(doc));
                        throw ({ forbidden: "This document type cannot be created without user key. id=" + doc._id });
                    }
                }
             }

Furthermore, there are a number of documents that have ispublic=true that are not sent to mobile now… E.g. type “Lake”. They basically all fall into the ifblock I have put an arrow next to at the bottom of the function:

Javascript: Sync SG: doc id: Lake:6064, ispublic: true, userkey=undefined,  creation
2019-10-11T17:33:46.308+02:00 [INF] Javascript: Sync SG: public, id: Lake:6064
2019-10-11T17:33:46.309+02:00 [INF] Javascript: Sync SG: document type cannot be created without user key: {"_id":"Lake:6064","_rev":"1-f11b851601a337c43c9998173ec6e75a","ispublic":true,"key":"6064","name":"Studentersø","points":[{"lat":55.3976232736,"lon":11.3819596121}],"revisioninfo":{"created":"2018-08-16T13:35:43+0200","createdby":"John Dalsgaard/BA171123846CEBF1C1257CB2002DA330/Fangst","modifiedcount":0},"type":"Lake"}
2019-10-11T17:33:46.309+02:00 [INF] CRUD: 	Doc "Lake:6064" / "1-f11b851601a337c43c9998173ec6e75a" in channels "{!}"
2019-10-11T17:33:46.309+02:00 [INF] Access: Saving updated channels and access grants of "Lake:6064"

Why is that a write now??? Why doesn’t it just use the info that this document has been added to the “!” channel that is “public”?

I’m sorry to have to bother you with this - but it just doesn’t make sense! Well, rather, it somehow probably does - but I just cannot see why??? :frowning:

Ok, so by “brute force” testing I have a solution that seems to work - but I still don’t understand why the problematic docs are writes nor why the types put in the filter still replicates to mobile.

So this is the current sync. function that kind of works…

			function (doc, oldDoc) {
                function _log(t) {
                    // Write to sg_info.log
                    console.log('SG: ' + t);
                }
                function _getUserKey(d) {
                    var key = null;
                    if (d) {
                        if (d.type == 'User') {
                            key = d.key;
                        } else {
                            key = d.userkey;
                        }
                    }
                    return key;
                }
            
                if (doc && doc._deleted) {
                    // Doc. deleted -> if public then require
					if(oldDoc){
	                    var userkey = _getUserKey(oldDoc);
	                    _log('delete doc id: ' + doc._id + ', userkey=' + userkey);
    	                if (userkey != null) {
		                    requireUser(userkey);
            	        } else {
                    	    requireAdmin();
						}
                    }
                    _log('doc deleted, id: ' + (doc._id || 'no id!') + ', ' + (oldDoc ? ('old key=' + oldDoc.key + ', userkey=' + userkey) : 'no oldDoc'));
                    return;
                }
                _log('doc id: ' + (doc._id || 'no id!') + ', ispublic: ' + doc.ispublic + ', userkey=' + userkey + ', ' + (oldDoc ? (oldDoc._deleted ? 'oldDoc is deleted' : ('old key=' + oldDoc.key + ', oldDoc.userkey=' + oldDoc.userkey + ' update')) : ' creation'));
                // Document type is mandatory
                if (typeof doc.type === 'undefined') {
                    _log('Document type missing: ' + JSON.stringify(doc));
                    throw ({ forbidden: "Document type is required. id=" + doc._id });
                }
                // Document key is mandatory
                if (typeof doc.key === 'undefined') {
                    _log('Document key missing: ' + JSON.stringify(doc));
                    throw ({ forbidden: "Document key is required. id=" + doc._id });
                }
                // Document type Image - sync disabled... (why have type?)
                if (doc.type === 'Image' && doc.issyncdisabled) {
                    throw ({ forbidden: "Sync. disabled for id=" + doc._id });
                }
                // Reject unwanted doc. types on mobile...
                if (doc.type == 'EnvLake' || doc.type == 'EnvMeasurement' || doc.type == 'ActivityLog') {
                    throw ({ forbidden: "Not allowed on mobile... id=" + doc._id });
                }
            
                // All public docs are available in the app
                if (doc.ispublic) {
                    _log('public, id: ' + (doc._id || 'no id!'));
                    channel('!');
                }

                // Update: Cannot allow change of type or key
                if (oldDoc != null && !oldDoc._deleted) {
                    // Update
                    if (oldDoc.type != doc.type) {
                        throw ({ forbidden: "Can't change doc type" });
                    }
                    if (oldDoc.key != doc.key) {
                        throw ({ forbidden: "Can't change doc key" });
                    }
                }

                // Allow anyone to create a Feedback or Observation on the server
                if (oldDoc == null && doc.userkey == null && (doc.type == 'Feedback' || doc.type == 'Observation')) {
                    _log('Created feedback: ' + (doc._id || 'no id!') + ', key: ' + doc.key + ', user: ' + doc.userkey);
                    return;
                }

                // Only non-public docs "owned" by user can be created/updated (and replicated)
                var userkey = _getUserKey(doc);
                if (userkey != null) {
                    if (oldDoc != null && ! oldDoc._deleted) {
                        // Update
                        if (oldDoc.userkey && oldDoc.userkey != doc.userkey) {
                            throw ({ forbidden: "Can't change user key" });
                        }
                    }
                    _log('User owned, id: ' + (doc._id || 'no id!') + ', type: ' + doc.type + ', user: ' + userkey);
                    channel('channel.' + userkey);
                    access(userkey, 'channel.' + userkey);
					requireUser(userkey);
                } else if(doc.ispublic){
                    requireAdmin();
                } else {
                    // Creation/update without user
                    if(doc.type !== 'Feedback' && doc.type !== 'Observation'){
                        _log('Document type cannot be created without user key: ' + (doc.type === 'Image' ? doc._id : JSON.stringify(doc)));
                        throw ({ forbidden: "This document type cannot be created without user key. id=" + doc._id });
                    }
                }
             }

And “yes” I did try to read the documents that you linked to once more - and still not sure why I experience the above pattern…

I guess it has to do with some of the functionality being “routing” and some of it “write access” - so ideally I should be able to divide the code better for each of these purposes. But still very confusing… :frowning:

As discussed in other post , did you start with a clean bucket on server. To avoid the possibility of some of these docs already imported prior to the filter change