DOC-8068 -- Replicator retry logic

modules/ROOT/pages/_partials/commons/common-p2psync-websocket-using-active.adoc

@@ -90,7 +90,7 @@ endif::[]
 
 == Configure Replicator
 In this section::
-<<lbl-cfg-tgt>>  |  <<lbl-cfg-sync>>  |  <<lbl-cfg-htbt>>  |  <<lbl-auth-lstnr>>  | <<lbl-authclnt>>
+<<lbl-cfg-tgt>>  |  <<lbl-cfg-sync>>  |  <<lbl-cfg-retry>>  |  <<lbl-auth-lstnr>>  | <<lbl-authclnt>>
 
 [#lbl-cfg-tgt]
 === Configure Target
@@ -102,10 +102,10 @@ include::{root-commons}sgw-replication-cfg-tgt.adoc[]
 
 include::{root-commons}sgw-replication-cfg-sync-mode.adoc[]
 
-[#lbl-cfg-htbt]
-=== Heartbeat
+[#lbl-cfg-retry]
+=== Retry Configuration
 
-include::{root-commons}sgw-replication-cfg-heartbeat.adoc[]
+include::{root-commons}sgw-replication-cfg-retry.adoc[]
 
 [#lbl-auth-lstnr]
 === Authenticating the Listener

modules/ROOT/pages/_partials/commons/common-sgw-replication-cfg-retry.adoc

@@ -0,0 +1,85 @@
+// BEGIN -- inclusion -- common-sgw-replication-cfg-retryadoc
+//  Begin -- inclusion definition
+//    Use - output text pertaining to replication retry logic and config
+//    Params:
+//      :is-p2p: - sets the server as a listener and not a Sync Gateway
+//    Location -- modules/ROOT/pages/_partials/commons/
+//
+//    Inclusions and Attributes:
+//      Uses attributes from the _attributes-module.adoc to links to
+//      REST API properties (eg {url-api-prop-<blah>})
+//      You can find _attributes-module.adoc in this location for each platform:
+//        modules/<platform>/pages/_partials/
+//
+//  End -- inclusion definition
+
+:this-svr: Sync Gateway
+ifdef::is-p2p[]
+:this-svr: listener
+endif::[]
+
+Couchbase Lite for {param-title}'s replication retry logic assures a resilient connection.
+
+The replicator minimizes the chance and impact of dropped connections by maintaining a heartbeat; essentially pinging the {this-svr} at a configurable interval to ensure the connection remains alive.
+
+In the event it detects a transient error, the replicator will attempt to reconnect, stopping only when the connection is re-established, or the number of retries exceeds the retry limit (9 times for a single-shot replication and unlimited for a continuous replication).
+
+On each retry the interval between attempts is increased exponentially (exponential backoff) up to the maximum wait time limit (5 minutes).
+
+The REST API provides configurable control over this replication retry logic using a set of configiurable properties -- see: <<tbl-repl-retry>>.
+
+.Replication Retry Configuration Properties
+[#tbl-repl-retry,cols="2,3,5"]
+|===
+
+h|Property
+h|Use cases
+h|Description
+
+|{url-api-prop-replicator-config-setHeartbeat}
+a|* Reduce to detect connection errors sooner
+* Align to load-balancer or proxy `keep-alive` interval -- see Sync Gateway's topic {xref-sgw-bmk-load-balancer-keepalive}
+a|The interval (in seconds) between the heartbeat pulses.
+
+Default: The replicator pings the {this-svr} every 300 seconds.
+
+|{url-api-prop-replicator-config-setMaxRetries}
+|Change this to limit or extend the number of retry attempts.
+a| The maximum number of retry attempts
+
+* Set this to zero (0) to prevent any retry attempt
+* The retry attempt count is reset when the replicator is able to connect and replicate
+* Default values are:
+** Single-shot replication = 9;
+** Continuous replication = maximum integer value
+* Negative values generate a Couchbase exception `InvalidArgumentException`
+
+|{url-api-prop-replicator-config-setMaxRetryWaitTime}
+|Change this to adjust the interval between retries.
+a|The maximum interval between retry attempts
+
+Whilst you can configure the *maximum permitted* wait time,  each individual interval is calculated by the replicator's exponential backoff algorithm and is not configurable.
+
+* Default value: 300 seconds (5 minutes)
+* Zero or negative values generate a Couchbase exception, `InvalidArgumentException`.
+
+|===
+
+When necessary you can adjust any or all of those configurable values -- see: <<ex-repl-retry>> for how to do this.
+
+.Configuring Replication Retries
+[#ex-repl-retry]
+====
+[source, {source-language}]
+----
+include::{snippet}[tags=replication-retry-config, indent=0]
+
+----
+
+<.> Here we use {url-api-prop-replicator-config-setHeartbeat} to set the required interval (in seconds) between the heartbeat pulses
+<.> Here we use {url-api-prop-replicator-config-setMaxRetries} to set the required number of retry attempts
+<.> Here we use {url-api-prop-replicator-config-setMaxRetryWaitTime} to set the required interval between retry attempts.
+
+====
+
+// END -- inclusion -- common-sgw-replication-cfg-retryadoc

modules/ROOT/pages/_partials/commons/common-sgw-replication.adoc

@@ -107,7 +107,7 @@ Here we say to provide _Basic Authentication_ credentials. Other options are ava
 == Configuration
 
 In this section::
-<<lbl-cfg-tgt>>  |  <<lbl-cfg-sync>>  |  <<lbl-cfg-htbt>>  |  <<lbl-svr-auth>>  | <<lbl-client-auth>> | <<lbl-repl-evnts>> |  <<lbl-repl-hdrs>> |  <<lbl-repl-ckpt>> |  <<lbl-repl-fltrs>> | <<lbl-repl-chan>> | <<lbl-repl-delta>>
+<<lbl-cfg-tgt>>  |  <<lbl-cfg-sync>>  |  <<lbl-cfg-retry>>  |  <<lbl-svr-auth>>  | <<lbl-client-auth>> | <<lbl-repl-evnts>> |  <<lbl-repl-hdrs>> |  <<lbl-repl-ckpt>> |  <<lbl-repl-fltrs>> | <<lbl-repl-chan>> | <<lbl-repl-delta>>
 
 
 [#lbl-cfg-tgt]
@@ -121,10 +121,10 @@ include::{root-commons}sgw-replication-cfg-tgt.adoc[]
 
 include::{root-commons}sgw-replication-cfg-sync-mode.adoc[]
 
-[#lbl-cfg-htbt]
-=== Heartbeat
+[#lbl-cfg-retry]
+=== Retry Configuration
 
-include::{root-commons}sgw-replication-cfg-heartbeat.adoc[]
+include::{root-commons}sgw-replication-cfg-retry.adoc[]
 
 [#lbl-svr-auth]
 === Server Authentication

modules/android/pages/_partials/_attributes-module.adoc

@@ -266,10 +266,14 @@
 
 :url-api-prop-replicator-config-setPinnedServerCertificate: {url-api-references-replicator-config-prop}setPinnedServerCertificate-byte:A-[setPinnedServerCertificate]
 
-// Begin Replicator Heartbeat
+// Begin Replicator Retry Config
 :url-api-prop-replicator-config-setHeartbeat: {url-api-references-replicator-config-prop}heartbeat[setHeartbeat()]
-:url-api-prop-replicator-config-getHeartbeat: {url-api-references-replicator-config-prop}heartbeat[setHeartbeat()]
-// End Replicator Heartbeat
+:url-api-prop-replicator-config-getHeartbeat: {url-api-references-replicator-config-prop}heartbeat[getHeartbeat()]
+:url-api-prop-replicator-config-setMaxRetries: {url-api-references-replicator-config-prop}maxretries[setMaxRetries()]
+:url-api-prop-replicator-config-getMaxRetries: {url-api-references-replicator-config-prop}maxretries[getMaxRetries()]
+:url-api-prop-replicator-config-setMaxRetryWaitTime: {url-api-references-replicator-config-prop}maxretrywaittime[setMaxRetryWaitTime()]
+:url-api-prop-replicator-config-getMaxRetryWaitTime: {url-api-references-replicator-config-prop}maxretrywaittime[getMaxRetryWaitTime()]
+// End Replicator Retry Config
 
 :url-api-prop-replicator-config-database: {url-api-references-replicator-config-prop}#getDatabase--[getDatabase]
 

modules/csharp/examples/code_snippets/Program.cs

@@ -1272,23 +1272,31 @@ private static void PullWithFilter(Database database)
             // end::replication-pull-filter[]
         }
 
-        public void TestCustomHeartbeat()
+        public void TestCustomRetryConfig()
         {
-        // tag::replication-set-heartbeat[]
+        // tag::replication-retry-config[]
             var url = new Uri("ws://localhost:4984/mydatabase");
             var target = new URLEndpoint(url);
 
             var config = new ReplicatorConfiguration(database, target);
 
         //  other config as required . . .
 
-            config.Heartbeat = TimeSpan.FromSeconds(60); //  <.>
+        // tag::replication-set-heartbeat[]
+            config.Heartbeat = TimeSpan.FromSeconds(120); //  <.>
+        // end::replication-set-heartbeat[]
+        // tag::replication-set-maxretries[]
+            config.MaxRetries = 20; //  <.>
+        // end::replication-set-maxretries[]
+        // tag::replication-set-maxretrywaittime[]
+            config.MaxRetryWaitTime = TimeSpan.FromSeconds(600); //  <.>
+        // end::replication-set-maxretrywaittime[]
 
         //  other config as required . . .
 
             var repl = new Replicator(config);
 
-        // end::replication-set-heartbeat[]
+        // end::replication-retry-config[]
         }
 
 

modules/csharp/pages/_partials/_attributes-module.adoc

@@ -265,13 +265,14 @@
 
 :url-api-prop-replicator-config-setPinnedServerCertificate: {url-api-references-replicator-config-prop}PinnedServerCertificate[PinnedServerCertificate]
 
-// Begin Replicator Heartbeat
+// Begin Replicator Retry Config
 :url-api-prop-replicator-config-setHeartbeat: {url-api-references-replicator-config-prop}Heartbeat[Heartbeat()]
 :url-api-prop-replicator-config-getHeartbeat: {url-api-references-replicator-config-prop}Heartbeat[Heartbeat()]
-// End Replicator Heartbeat
-
-
-
+:url-api-prop-replicator-config-setMaxRetries: {url-api-references-replicator-config-prop}MaxRetries[MaxRetries()]
+:url-api-prop-replicator-config-getMaxRetries: {url-api-references-replicator-config-prop}MaxRetries[MaxRetries()]
+:url-api-prop-replicator-config-setMaxRetryWaitTime: {url-api-references-replicator-config-prop}MaxRetryWaitTime[MaxRetryWaitTime()]
+:url-api-prop-replicator-config-getMaxRetryWaitTime: {url-api-references-replicator-config-prop}MaxRetryWaitTime[MaxRetryWaitTime()]
+// End Replicator Retry Config
 
 :url-api-prop-replicator-config-AcceptOnlySelfSignedServerCertificate: {url-api-references-replicator-config-prop}AcceptOnlySelfSignedServerCertificate[AcceptOnlySelfSignedServerCertificate]
 :url-api-prop-replicator-config-auth: {url-api-references-replicator-config-prop}Authenticator[Authenticator]

modules/java/examples/code_snippets/Examples.java

@@ -1601,23 +1601,30 @@ public void testReplicationPullFilter() throws URISyntaxException {
     }
 
     //
-    public void testCustomHeartbeat() throws URISyntaxException {
-    // tag::replication-set-heartbeat[]
+    public void testCustomRetryConfig() throws URISyntaxException {
+    // tag::replication-retry-config[]
         URLEndpoint target =
             new URLEndpoint(new URI("ws://localhost:4984/mydatabase"));
 
         ReplicatorConfiguration config =
             new ReplicatorConfiguration(database, target);
 
         //  other config as required . . .
-
-        config.setHeartbeat(60L); // <.>
+        // tag::replication-heartbeat-config[]
+        config.setHeartbeat(150L); // <.>
+        // end::replication-heartbeat-config[]
+        // tag::replication-maxretries-config[]
+        config.setMaxRetries(20L); // <.>
+        // end::replication-maxretries-config[]
+        // tag::replication-maxretrywaittime-config[]
+        config.setMaxRetryWaitTime(600L); // <.>
+        // end::replication-maxretrywaittime-config[]
 
         //  other config as required . . .
 
         Replicator repl = new Replicator(config);
 
-    // end::replication-set-heartbeat[]
+    // end::replication-retry-config[]
     }
 
 

modules/java/pages/_partials/_attributes-module.adoc

@@ -261,6 +261,17 @@
 :url-api-prop-replicator-config-setPinnedServerCertificate: {url-api-references-replicator-config-prop}setPinnedServerCertificate-byte:A-[setPinnedServerCertificate()]
 
 
+// Begin Replicator Retry Config
+:url-api-prop-replicator-config-setHeartbeat: {url-api-references-replicator-config-prop}setHeartbeat-long-[setHeartbeat()]
+:url-api-prop-replicator-config-getHeartbeat: {url-api-references-replicator-config-prop}getHeartbeat--[getHeartbeat()]
+:url-api-prop-replicator-config-setMaxRetries: {url-api-references-replicator-config-prop}setmaxretries-long-[setMaxRetries()]
+:url-api-prop-replicator-config-getMaxRetries: {url-api-references-replicator-config-prop}getmaxretries--[getMaxRetries()]
+:url-api-prop-replicator-config-setMaxRetryWaitTime: {url-api-references-replicator-config-prop}setmaxretrywaittime-long-[setMaxRetryWaitTime()]
+:url-api-prop-replicator-config-getMaxRetryWaitTime: {url-api-references-replicator-config-prop}getmaxretrywaittime--[getMaxRetryWaitTime()]
+// End Replicator Retry Config
+
+
+
 
 
 :url-api-prop-replicator-config-AcceptOnlySelfSignedServerCertificate: {url-api-references-replicator-config-prop}setAcceptOnlySelfSignedServerCertificate-boolean-[setAcceptOnlySelfSignedServerCertificate]

modules/objc/examples/code_snippets/SampleCodeTest.m

@@ -1003,23 +1003,37 @@ - (void) dontTestReplicationPullFilter {
 }
 
 //  Added 2/Feb/21 - Ian Bridge
-    - void dontTestCustomHeartbeat {
-    // tag::replication-set-heartbeat[]
+    - void dontTestCustomRetryConfig {
+        // tag::replication-retry-config[]
         id target =
           [[CBLURLEndpoint alloc] initWithURL: [NSURL URLWithString: @"ws://foo.cbl.com/db"]];
 
-        CBLReplicatorConfiguration* config = 
+        CBLReplicatorConfiguration* config =
             [[CBLReplicatorConfiguration alloc] initWithDatabase: db target: target];
         config.type = kCBLReplicatorTypePush;
         config.continuous: YES;
         //  other config as required . . .
-        config.heartbeat = 60; // <.>
+
+        // tag::replication-heartbeat[]
+        config.heartbeat = 150; // <.>
+
+        // end::replication-heartbeat[]
+        // tag::replication-maxretries[]
+        config.maxretries = 20; // <.>
+
+        // end::replication-maxretries[]
+        // tag::replication-maxretrywaittime[]
+        config.maxretrywaittime = 600; // <.>
+
+        // end::replication-maxretrywaittime[]
         //  other config as required . . .
         repl = [[CBLReplicator alloc] initWithConfig: config];
 
         // Cleanup:
         repl = nil;
-    // end::replication-set-heartbeat[]
+
+        // end::replication-retry-config[]
+
     }
 
 

modules/objc/pages/_partials/_attributes-module.adoc

@@ -216,11 +216,14 @@
 
 :url-api-prop-replicator-config-setPinnedServerCertificate: {url-api-references-replicator-config-prop}(py)pinnedServerCertificate[setPinnedServerCertificate()]
 
-// Begin - Replicator Heartbeat
-:url-api-prop-replicator-config-setHeartbeat: {url-api-references-replicator-config-prop}(py)heartbeat[heartbeat()]
-:url-api-prop-replicator-config-getHeartbeat: {url-api-references-replicator-config-prop}(py)heartbeat[heartbeat()]
-
-// End - Replicator Heartbeat
+// Begin Replicator Retry Config
+:url-api-prop-replicator-config-setheartbeat: {url-api-references-replicator-config-prop}(py)heartbeat[heartbeat()]
+:url-api-prop-replicator-config-getheartbeat: {url-api-references-replicator-config-prop}(py)heartbeat[heartbeat()]
+:url-api-prop-replicator-config-setmaxretries: {url-api-references-replicator-config-prop}(py)maxretries[maxretries()]
+:url-api-prop-replicator-config-getmaxretries: {url-api-references-replicator-config-prop}(py)maxretries[maxretries()]
+:url-api-prop-replicator-config-setmaxretrywaittime: {url-api-references-replicator-config-prop}(py)maxretrywaittime[maxretrywaittime()]
+:url-api-prop-replicator-config-getmaxretrywaittime: {url-api-references-replicator-config-prop}(py)maxretrywaittime[maxretrywaittime()]
+// End Replicator Retry Config
 
 :url-api-prop-replicator-config-acceptOnlySelfSignedServerCertificate: {url-api-references-replicator-config-prop}(py)acceptOnlySelfSignedServerCertificate[acceptOnlySelfSignedServerCertificate]
 

modules/swift/examples/code_snippets/SampleCodeTest.swift

@@ -830,18 +830,30 @@ class SampleCodeTest {
     }
 
 //  Added 2/Feb/21 - Ian Bridge
-    func testCustomHeartbeat() {
-        // tag::replication-set-heartbeat[]
+//  Changed for 3.0.0 - Ian Bridge 3/Mar/21
+    func testCustomRetryConfig() {
+        // tag::replication-retry-config[]
         let target =
           URLEndpoint(url: URL(string: "ws://foo.couchbase.com/db")!)
 
         let config =  ReplicatorConfiguration(database: database, target: targetDatabase)
         config.type = .pushAndPull
         config.continuous = true
-        config.heartbeat = 60 // <.>
-        repl = Replicator(config: config)
+        // tag::replication-set-heartbeat[]
+        config.heartbeat = 150 // <.>
 
         // end::replication-set-heartbeat[]
+        // tag::replication-set-maxretries[]
+        config.maxretries = 20 // <.>
+
+        // end::replication-set-maxretries[]
+        // tag::replication-set-maxretrywaittime[]
+        config.maxretrywaittime = 600 // <.>
+        repl = Replicator(config: config)
+
+        // end::replication-set-maxretrywaittime[]
+
+        // end::replication-retry-config[]
     }
 
     #if COUCHBASE_ENTERPRISE