feat: add tokio-retry crate to retry failed requests

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "slog-loggly"
-version = "0.5.0"
+version = "0.5.1"
 edition = "2018"
 authors = ["Angelcam, Inc."]
 license = "MIT/Apache-2.0"
@@ -28,6 +28,7 @@ hyper-tls   = "0.5"
 serde       = "1.0"
 serde_json  = "1.0"
 slog        = "2.5"
+tokio-retry = "0.3"
 
 [dependencies.hyper]
 version = "0.14"

src/client.rs

@@ -14,7 +14,7 @@ use hyper::{
 };
 use hyper_tls::HttpsConnector;
 
-use crate::{batch::BatchStream, channel::Message, error::Error};
+use crate::{batch::BatchStream, channel::Message, error::Error, retry::with_retry};
 
 /// Default request timeout in seconds.
 const DEFAULT_REQUEST_TIMEOUT: u64 = 5;
@@ -138,7 +138,11 @@ impl LogglyClient {
 
     /// Try to send a given log message.
     pub async fn try_send(&self, msg: Bytes) -> Result<(), Error> {
-        let send = tokio::time::timeout(self.timeout, self.try_send_inner(msg));
+        let action = || {
+          tokio::time::timeout(self.timeout, self.try_send_inner(msg.clone()))
+        };
+
+        let send = with_retry(None, None, action);
 
         let res = match send.await {
             Ok(Ok(())) => Ok(()),

src/lib.rs

@@ -81,6 +81,7 @@ mod channel;
 mod client;
 mod error;
 mod serializer;
+mod retry;
 
 use std::{str, sync::Mutex, time::Duration};
 

src/retry.rs

@@ -0,0 +1,25 @@
+use tokio_retry::strategy::{FibonacciBackoff, jitter};
+use std::future::Future;
+use std::{
+  iter::{Take, Map},
+  time::Duration
+};
+
+fn retry_strategy(ms: u64, attempts: usize) -> Take<Map<FibonacciBackoff, fn(Duration) -> Duration>>{
+  FibonacciBackoff::from_millis(ms)
+    .map(jitter as fn(Duration) -> Duration)
+    .take(attempts)
+}
+
+
+pub async fn with_retry<A, F, R, E>(ms: Option<u64>, attempts: Option<usize>, action: A) -> Result<R, E>
+  where
+    A: FnMut() -> F,
+    E: std::fmt::Display,
+    F: Future<Output=Result<R, E>>
+{
+  let attempts = if let Some(attempts) = attempts { attempts } else { 10 };
+  let ms = if let Some(ms) = ms { ms } else { 1000 /*1 sec */};
+
+  tokio_retry::Retry::spawn(retry_strategy(ms, attempts), action).await
+}