From a3169057559c71f750495c36bbc5a42ad2aeaf6b Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Thu, 20 Oct 2022 17:49:04 +0200
Subject: [PATCH 01/73] add ImmutableStore trait and fs based impl

---
 Cargo.toml                 |  6 +--
 src/storage/dynamodb.rs    |  4 --
 src/storage/file_system.rs | 47 +++++++++++++++++++++++
 src/storage/mod.rs         | 76 ++++++++++++++++++++++++++------------
 4 files changed, 102 insertions(+), 31 deletions(-)
 create mode 100644 src/storage/file_system.rs

diff --git a/Cargo.toml b/Cargo.toml
index f7066d4c..f5bc8bbf 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -23,12 +23,10 @@ ethers-core = "0.6"
 futures = { default-features = false, version = "0.3.9", features = ["alloc", "std"] }
 hex = "0.4"
 hyper = "0.14" # Prometheus server
-ipfs = { git = "https://github.com/spruceid/rust-ipfs", branch = "feat/update-libipld" }
 iri-string = "0.5"
 lazy_static = "1.4.0"
-libipld = "0.13"
-libp2p = { default-features = false, features = ["floodsub", "identify", "kad", "tcp-tokio", "mplex", "noise", "ping", "yamux", "dns-tokio", "relay"], version = "0.43.0" }
-nom = "6"
+libipld = "0.14"
+libp2p = { default-features = false, features = ["floodsub", "identify", "kad", "tcp-tokio", "mplex", "noise", "ping", "yamux", "dns-tokio", "relay"], version = "0.49.0" }
 opentelemetry = { version = "0.17.0", features = ["rt-tokio"] }
 opentelemetry-jaeger = { version = "0.16.0", features = ["rt-tokio", "reqwest_collector_client"] }
 prometheus = { version = "0.13.0", features = ["process"] }
diff --git a/src/storage/dynamodb.rs b/src/storage/dynamodb.rs
index 2548018c..0fc329b2 100644
--- a/src/storage/dynamodb.rs
+++ b/src/storage/dynamodb.rs
@@ -11,10 +11,6 @@ use futures::{
     lock::Mutex,
     stream::{self, StreamExt, TryStreamExt},
 };
-use ipfs::{
-    refs::IpldRefsError,
-    repo::{PinKind, PinMode, PinStore},
-};
 use kepler_lib::libipld::cid::{multibase::Base, Cid};
 use rocket::async_trait;
 use std::{collections::BTreeSet, str::FromStr};
diff --git a/src/storage/file_system.rs b/src/storage/file_system.rs
new file mode 100644
index 00000000..9a968350
--- /dev/null
+++ b/src/storage/file_system.rs
@@ -0,0 +1,47 @@
+use super::ImmutableStore;
+use std::{
+    io::{Error, ErrorKind},
+    path::PathBuf,
+};
+use tokio::{
+    fs::{remove_file, File},
+    io::copy,
+};
+
+pub struct FileSystemStore {
+    path: PathBuf,
+}
+
+impl FileSystemStore {
+    pub fn new(path: PathBuf) -> Self {
+        Self { path }
+    }
+}
+
+impl ImmutableStore for FileSystemStore {
+    type Error = std::io::Error;
+    type Readable = File;
+    async fn write(&self, data: impl futures::io::AsyncRead) -> Result<Cid, Self::Error> {
+        todo!();
+        // need to stream data through a hasher into the file and return hash
+        // match File::open(path.join(cid.to_string())),await {
+        //     Ok(f) => copy(data, file).await
+        //     Err(e) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),
+        //     Err(e) => Err(e),
+        // }
+    }
+    async fn remove(&self, id: &Cid) -> Result<Option<()>, Self::Error> {
+        match remove_file(self.path.join(cid.to_string())).await {
+            Ok(()) => Ok(Some(())),
+            Err(e) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),
+            Err(e) => Err(e),
+        }
+    }
+    async fn read(&self, id: &Cid) -> Result<Option<Self::Readable>, Self::Error> {
+        match File::open(path.join(cid.to_string())).await {
+            Ok(f) => Ok(Some(f)),
+            Err(e) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),
+            Err(e) => Err(e),
+        }
+    }
+}
diff --git a/src/storage/mod.rs b/src/storage/mod.rs
index c3369b77..5a540a61 100644
--- a/src/storage/mod.rs
+++ b/src/storage/mod.rs
@@ -4,23 +4,15 @@ use aws_sdk_s3::{
     types::{ByteStream, SdkError},
 };
 use aws_smithy_http::body::SdkBody;
-use ipfs::{
-    repo::{
-        fs::{FsBlockStore, FsDataStore, FsLock},
-        mem::MemLock,
-        BlockPut, BlockRm, BlockRmError, BlockStore, Column, DataStore, Lock, LockError, PinKind,
-        PinMode, PinStore,
-    },
-    Block, RepoTypes,
-};
 use kepler_lib::libipld::cid::{multibase::Base, Cid};
 use kepler_lib::resource::OrbitId;
 use libp2p::identity::ed25519::Keypair as Ed25519Keypair;
 use rocket::tokio::fs;
-use std::{path::PathBuf, str::FromStr};
+use std::{collections::HashMap, path::PathBuf, str::FromStr};
 use tracing::instrument;
 
 mod dynamodb;
+mod file_system;
 mod indexes;
 mod s3;
 mod utils;
@@ -40,19 +32,13 @@ pub struct Repo;
 #[derive(Debug)]
 pub enum BlockStores {
     S3(Box<s3::S3BlockStore>),
-    Local(Box<FsBlockStore>),
+    Local(Box<file_system::FileSystemStore>),
 }
 
 #[derive(Debug)]
 pub enum DataStores {
     S3(Box<s3::S3DataStore>),
-    Local(Box<FsDataStore>),
-}
-
-#[derive(Debug)]
-pub enum Locks {
-    S3(MemLock),
-    Local(FsLock),
+    Local(Box<file_system::FileSystemStore>),
 }
 
 impl StorageUtils {
@@ -244,7 +230,7 @@ impl StorageUtils {
             config::BlockStorage::Local(r) => {
                 let path = r.path.join(orbit.to_string_of_base(Base::Base58Btc)?);
                 if !path.exists() {
-                    tokio::fs::create_dir_all(&path).await?;
+                    fs::create_dir_all(&path).await?;
                 }
                 Ok(path)
             }
@@ -252,10 +238,54 @@ impl StorageUtils {
     }
 }
 
-impl RepoTypes for Repo {
-    type TBlockStore = BlockStores;
-    type TDataStore = DataStores;
-    type TLock = Locks;
+#[derive(Error)]
+pub enum VecReadError<E> {
+    #[error(transparent)]
+    Store(#[from] E),
+    #[error(transparent)]
+    Read(#[from] futures::io::Error),
+}
+
+#[async_trait]
+trait ImmutableStore {
+    type Error;
+    type Readable: futures::io::AsyncRead;
+    async fn write(&self, data: impl futures::io::AsyncRead) -> Result<Cid, Self::Error>;
+    async fn remove(&self, id: &Cid) -> Result<Option<()>, Self::Error>;
+    async fn read(&self, id: &Cid) -> Result<Option<Self::Readable>, Self::Error>;
+    async fn read_to_vec(&self, id: &Cid) -> Result<Option<Vec<u8>>, VecReadError<Self::Error>> {
+        Ok(match self.read(id).await? {
+            Some(r) => {
+                let mut v = Vec::new();
+                r.read_all(&mut v).await?;
+                Some(v)
+            }
+            _ => None,
+        })
+    }
+}
+
+#[async_trait]
+trait StoreSeek: ImmutableStore {
+    type Seekable: futures::io::AsyncSeek;
+    async fn seek(&self, id: &Cid) -> Result<Option<Self::Seekable>, Self::Error>;
+}
+
+#[async_trait]
+trait IdempotentHeightGroup {
+    // write a height value for a Cid
+    // should error if given value already exists
+    // if successful, marks a Cid as 'fresh'
+    async fn see(&self, id: impl IntoIterator<Item = (&Cid, &u64)>) -> Result<(), Error>;
+    // mark a Cid as no longer 'fresh'
+    async fn stale(&self, id: impl IntoIterator<Item = &Cid>) -> Result<(), Error>;
+    // return 'fresh' Cids and their heights
+    async fn fresh(&self) -> Result<HashMap<Cid, u64>, Error>;
+    // return the heights of any Cids
+    async fn height<'a>(
+        &self,
+        id: impl IntoIterator<Item = &'a Cid>,
+    ) -> Result<HashMap<&'a Cid, u64>, Error>;
 }
 
 #[async_trait]

From eeace02a5b85461f940d3d5303edf3d47fea4257 Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Fri, 21 Oct 2022 14:55:29 +0200
Subject: [PATCH 02/73] use multihashes for content IDs (ignore codec)

---
 src/cas.rs                 |   9 --
 src/ipfs.rs                |  40 +------
 src/storage/file_system.rs |  19 +++-
 src/storage/mod.rs         | 227 ++-----------------------------------
 4 files changed, 24 insertions(+), 271 deletions(-)

diff --git a/src/cas.rs b/src/cas.rs
index 6d86d627..b4c54866 100644
--- a/src/cas.rs
+++ b/src/cas.rs
@@ -30,12 +30,3 @@ impl<'r> FromFormField<'r> for CidWrap {
         ))
     }
 }
-
-#[rocket::async_trait]
-pub trait ContentAddressedStorage: Send + Sync {
-    type Error;
-    async fn put(&self, content: &[u8], codec: SupportedCodecs) -> Result<Cid, Self::Error>;
-    async fn get(&self, address: &Cid) -> Result<Option<Vec<u8>>, Self::Error>;
-    async fn delete(&self, address: &Cid) -> Result<(), Self::Error>;
-    async fn list(&self) -> Result<Vec<Cid>, Self::Error>;
-}
diff --git a/src/ipfs.rs b/src/ipfs.rs
index d643f012..b25e5bb0 100644
--- a/src/ipfs.rs
+++ b/src/ipfs.rs
@@ -1,9 +1,4 @@
 use anyhow::Result;
-use ipfs::{
-    multiaddr,
-    p2p::{transport::TransportBuilder, TSwarm},
-    Ipfs as OIpfs, IpfsOptions, Keypair, PeerId, PinMode, Types, UninitializedIpfs,
-};
 use kepler_lib::libipld::{
     block::Block as OBlock,
     cid::{multibase::Base, Cid},
@@ -11,7 +6,7 @@ use kepler_lib::libipld::{
     raw::RawCodec,
     store::DefaultParams,
 };
-use libp2p::{core::transport::MemoryTransport, futures::TryStreamExt};
+use libp2p::{core::transport::MemoryTransport, futures::TryStreamExt, swarm::Swarm as TSwarm};
 use std::{future::Future, sync::mpsc::Receiver};
 
 use super::{cas::ContentAddressedStorage, codec::SupportedCodecs};
@@ -31,7 +26,6 @@ pub type KeplerParams = DefaultParams;
 //     type Hashes = Code;
 // }
 
-pub type Ipfs = OIpfs<Repo>;
 pub type Block = OBlock<KeplerParams>;
 pub type Swarm = TSwarm<Types>;
 
@@ -79,35 +73,3 @@ where
 
     Ok((ipfs, ipfs_task, receiver))
 }
-
-#[rocket::async_trait]
-impl ContentAddressedStorage for Ipfs {
-    type Error = anyhow::Error;
-    async fn put(&self, content: &[u8], _codec: SupportedCodecs) -> Result<Cid, Self::Error> {
-        // TODO find a way to stream this better? (use .take with max block size?)
-        let block: Block = Block::encode(RawCodec, Code::Blake3_256, content)?;
-        let cid = self.put_block(block).await?;
-        self.insert_pin(&cid, true).await?;
-        Ok(cid)
-    }
-    async fn get(&self, address: &Cid) -> Result<Option<Vec<u8>>, Self::Error> {
-        // TODO this api returns Result<Block, anyhow::Error>, with an err thrown for no block found
-        // until this API changes (a breaking change), we will error here when no block found
-        Ok(Some(self.get_block(address).await?.data().to_vec()))
-    }
-
-    async fn delete(&self, address: &Cid) -> Result<(), Self::Error> {
-        // TODO: does not recursively remove blocks, some cleanup will need to happen.
-        self.remove_pin(address, true).await?;
-        self.remove_block(*address).await?;
-        Ok(())
-    }
-    async fn list(&self) -> Result<Vec<Cid>, Self::Error> {
-        // return a list of all CIDs which are aliased/pinned
-        self.list_pins(Some(PinMode::Recursive))
-            .await
-            .map_ok(|(cid, _pin_mode)| cid)
-            .try_collect()
-            .await
-    }
-}
diff --git a/src/storage/file_system.rs b/src/storage/file_system.rs
index 9a968350..483315a7 100644
--- a/src/storage/file_system.rs
+++ b/src/storage/file_system.rs
@@ -1,4 +1,8 @@
 use super::ImmutableStore;
+use kepler_lib::libipld::cid::{
+    multibase::{encode, Base},
+    multihash::Multihash,
+};
 use std::{
     io::{Error, ErrorKind},
     path::PathBuf,
@@ -16,13 +20,18 @@ impl FileSystemStore {
     pub fn new(path: PathBuf) -> Self {
         Self { path }
     }
+
+    fn get_path(mh: &Multihash) -> PathBuf {
+        self.path.join(encode(Base::Base64Url, mh))
+    }
 }
 
 impl ImmutableStore for FileSystemStore {
     type Error = std::io::Error;
     type Readable = File;
-    async fn write(&self, data: impl futures::io::AsyncRead) -> Result<Cid, Self::Error> {
+    async fn write(&self, data: impl futures::io::AsyncRead) -> Result<Multihash, Self::Error> {
         todo!();
+        // write into tmp then rename, to name after the hash
         // need to stream data through a hasher into the file and return hash
         // match File::open(path.join(cid.to_string())),await {
         //     Ok(f) => copy(data, file).await
@@ -30,15 +39,15 @@ impl ImmutableStore for FileSystemStore {
         //     Err(e) => Err(e),
         // }
     }
-    async fn remove(&self, id: &Cid) -> Result<Option<()>, Self::Error> {
-        match remove_file(self.path.join(cid.to_string())).await {
+    async fn remove(&self, id: &Multihash) -> Result<Option<()>, Self::Error> {
+        match remove_file(self.get_path(id)).await {
             Ok(()) => Ok(Some(())),
             Err(e) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),
             Err(e) => Err(e),
         }
     }
-    async fn read(&self, id: &Cid) -> Result<Option<Self::Readable>, Self::Error> {
-        match File::open(path.join(cid.to_string())).await {
+    async fn read(&self, id: &Multihash) -> Result<Option<Self::Readable>, Self::Error> {
+        match File::open(self.get_path(id)).await {
             Ok(f) => Ok(Some(f)),
             Err(e) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),
             Err(e) => Err(e),
diff --git a/src/storage/mod.rs b/src/storage/mod.rs
index 5a540a61..26af5ae8 100644
--- a/src/storage/mod.rs
+++ b/src/storage/mod.rs
@@ -4,7 +4,7 @@ use aws_sdk_s3::{
     types::{ByteStream, SdkError},
 };
 use aws_smithy_http::body::SdkBody;
-use kepler_lib::libipld::cid::{multibase::Base, Cid};
+use kepler_lib::libipld::cid::{multibase::Base, multihash::Multihash, Cid};
 use kepler_lib::resource::OrbitId;
 use libp2p::identity::ed25519::Keypair as Ed25519Keypair;
 use rocket::tokio::fs;
@@ -250,10 +250,14 @@ pub enum VecReadError<E> {
 trait ImmutableStore {
     type Error;
     type Readable: futures::io::AsyncRead;
-    async fn write(&self, data: impl futures::io::AsyncRead) -> Result<Cid, Self::Error>;
-    async fn remove(&self, id: &Cid) -> Result<Option<()>, Self::Error>;
-    async fn read(&self, id: &Cid) -> Result<Option<Self::Readable>, Self::Error>;
-    async fn read_to_vec(&self, id: &Cid) -> Result<Option<Vec<u8>>, VecReadError<Self::Error>> {
+    async fn contains(&self, id: &Multihash) -> Result<bool, Self::Error>;
+    async fn write(&self, data: impl futures::io::AsyncRead) -> Result<Multihash, Self::Error>;
+    async fn remove(&self, id: &Multihash) -> Result<Option<()>, Self::Error>;
+    async fn read(&self, id: &Multihash) -> Result<Option<Self::Readable>, Self::Error>;
+    async fn read_to_vec(
+        &self,
+        id: &Multihash,
+    ) -> Result<Option<Vec<u8>>, VecReadError<Self::Error>> {
         Ok(match self.read(id).await? {
             Some(r) => {
                 let mut v = Vec::new();
@@ -287,216 +291,3 @@ trait IdempotentHeightGroup {
         id: impl IntoIterator<Item = &'a Cid>,
     ) -> Result<HashMap<&'a Cid, u64>, Error>;
 }
-
-#[async_trait]
-impl BlockStore for BlockStores {
-    fn new(path: PathBuf) -> Self {
-        if path.to_str().unwrap().starts_with("/s3bucket/") {
-            Self::S3(Box::new(s3::S3BlockStore::new(path)))
-        } else {
-            Self::Local(Box::new(FsBlockStore::new(path)))
-        }
-    }
-
-    async fn init(&self) -> Result<(), Error> {
-        match self {
-            Self::S3(r) => r.init().await,
-            Self::Local(r) => r.init().await,
-        }
-    }
-
-    async fn open(&self) -> Result<(), Error> {
-        match self {
-            Self::S3(r) => r.open().await,
-            Self::Local(r) => r.open().await,
-        }
-    }
-
-    async fn contains(&self, cid: &Cid) -> Result<bool, Error> {
-        match self {
-            Self::S3(r) => r.contains(cid).await,
-            Self::Local(r) => r.contains(cid).await,
-        }
-    }
-
-    #[instrument(name = "blocks::get", skip_all)]
-    async fn get(&self, cid: &Cid) -> Result<Option<Block>, Error> {
-        match self {
-            Self::S3(r) => r.get(cid).await,
-            Self::Local(r) => r.get(cid).await,
-        }
-    }
-
-    #[instrument(name = "blocks::put", skip_all)]
-    async fn put(&self, block: Block) -> Result<(Cid, BlockPut), Error> {
-        match self {
-            Self::S3(r) => r.put(block).await,
-            Self::Local(r) => r.put(block).await,
-        }
-    }
-
-    async fn remove(&self, cid: &Cid) -> Result<Result<BlockRm, BlockRmError>, Error> {
-        match self {
-            Self::S3(r) => r.remove(cid).await,
-            Self::Local(r) => r.remove(cid).await,
-        }
-    }
-
-    async fn list(&self) -> Result<Vec<Cid>, Error> {
-        match self {
-            Self::S3(r) => r.list().await,
-            Self::Local(r) => r.list().await,
-        }
-    }
-
-    async fn wipe(&self) {
-        match self {
-            Self::S3(r) => r.wipe().await,
-            Self::Local(r) => r.wipe().await,
-        }
-    }
-}
-
-#[async_trait]
-impl DataStore for DataStores {
-    fn new(path: PathBuf) -> DataStores {
-        if path.to_str().unwrap().starts_with("/s3bucket/") {
-            Self::S3(Box::new(s3::S3DataStore::new(path)))
-        } else {
-            Self::Local(Box::new(FsDataStore::new(path)))
-        }
-    }
-
-    async fn init(&self) -> Result<(), Error> {
-        match self {
-            Self::S3(r) => r.init().await,
-            Self::Local(r) => r.init().await,
-        }
-    }
-
-    async fn open(&self) -> Result<(), Error> {
-        match self {
-            Self::S3(r) => r.open().await,
-            Self::Local(r) => r.open().await,
-        }
-    }
-
-    async fn contains(&self, col: Column, key: &[u8]) -> Result<bool, Error> {
-        match self {
-            Self::S3(r) => r.contains(col, key).await,
-            Self::Local(r) => r.contains(col, key).await,
-        }
-    }
-
-    async fn get(&self, col: Column, key: &[u8]) -> Result<Option<Vec<u8>>, Error> {
-        match self {
-            Self::S3(r) => r.get(col, key).await,
-            Self::Local(r) => r.get(col, key).await,
-        }
-    }
-
-    async fn put(&self, col: Column, key: &[u8], value: &[u8]) -> Result<(), Error> {
-        match self {
-            Self::S3(r) => r.put(col, key, value).await,
-            Self::Local(r) => r.put(col, key, value).await,
-        }
-    }
-
-    async fn remove(&self, col: Column, key: &[u8]) -> Result<(), Error> {
-        match self {
-            Self::S3(r) => r.remove(col, key).await,
-            Self::Local(r) => r.remove(col, key).await,
-        }
-    }
-
-    async fn wipe(&self) {
-        match self {
-            Self::S3(r) => r.wipe().await,
-            Self::Local(r) => r.wipe().await,
-        }
-    }
-}
-
-#[async_trait]
-impl PinStore for DataStores {
-    async fn is_pinned(&self, cid: &Cid) -> Result<bool, Error> {
-        match self {
-            Self::S3(r) => r.is_pinned(cid).await,
-            Self::Local(r) => r.is_pinned(cid).await,
-        }
-    }
-
-    async fn insert_direct_pin(&self, target: &Cid) -> Result<(), Error> {
-        match self {
-            Self::S3(r) => r.insert_direct_pin(target).await,
-            Self::Local(r) => r.insert_direct_pin(target).await,
-        }
-    }
-
-    async fn insert_recursive_pin(
-        &self,
-        target: &Cid,
-        referenced: References<'_>,
-    ) -> Result<(), Error> {
-        match self {
-            Self::S3(r) => r.insert_recursive_pin(target, referenced).await,
-            Self::Local(r) => r.insert_recursive_pin(target, referenced).await,
-        }
-    }
-
-    async fn remove_direct_pin(&self, target: &Cid) -> Result<(), Error> {
-        match self {
-            Self::S3(r) => r.remove_direct_pin(target).await,
-            Self::Local(r) => r.remove_direct_pin(target).await,
-        }
-    }
-
-    async fn remove_recursive_pin(
-        &self,
-        target: &Cid,
-        referenced: References<'_>,
-    ) -> Result<(), Error> {
-        match self {
-            Self::S3(r) => r.remove_recursive_pin(target, referenced).await,
-            Self::Local(r) => r.remove_recursive_pin(target, referenced).await,
-        }
-    }
-
-    async fn list(
-        &self,
-        requirement: Option<PinMode>,
-    ) -> futures::stream::BoxStream<'static, Result<(Cid, PinMode), Error>> {
-        match self {
-            Self::S3(r) => r.list(requirement).await,
-            Self::Local(r) => r.list(requirement).await,
-        }
-    }
-
-    async fn query(
-        &self,
-        ids: Vec<Cid>,
-        requirement: Option<PinMode>,
-    ) -> Result<Vec<(Cid, PinKind<Cid>)>, Error> {
-        match self {
-            Self::S3(r) => r.query(ids, requirement).await,
-            Self::Local(r) => r.query(ids, requirement).await,
-        }
-    }
-}
-
-impl Lock for Locks {
-    fn new(path: PathBuf) -> Self {
-        if path.to_str().unwrap().starts_with("/s3bucket/") {
-            Self::S3(MemLock::new(path))
-        } else {
-            Self::Local(FsLock::new(path))
-        }
-    }
-
-    fn try_exclusive(&mut self) -> Result<(), LockError> {
-        match self {
-            Self::S3(r) => r.try_exclusive(),
-            Self::Local(r) => r.try_exclusive(),
-        }
-    }
-}

From 5de83abb4603be5a7376e6597e681317e1dc4f7b Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Fri, 21 Oct 2022 15:04:29 +0200
Subject: [PATCH 03/73] remove ipfs trait impls

---
 src/storage/s3.rs | 110 ----------------------------------------------
 1 file changed, 110 deletions(-)

diff --git a/src/storage/s3.rs b/src/storage/s3.rs
index b1f7c665..23c633fd 100644
--- a/src/storage/s3.rs
+++ b/src/storage/s3.rs
@@ -21,16 +21,6 @@ use std::{path::PathBuf, str::FromStr};
 use super::dynamodb::{DynamoPinStore, References};
 use crate::config;
 
-#[derive(Debug)]
-pub struct S3DataStore {
-    // TODO Remove is unused (orbit::delete is never called).
-    // When that changes we will need to use a mutex, either local or in Dynamo
-    pub client: Client,
-    pub bucket: String,
-    pub dynamodb: DynamoPinStore,
-    pub orbit: Cid,
-}
-
 // TODO we could use the same struct for both the block store and the data
 // (pin) store, but we need to remember that for now it will be two different
 // objects in rust-ipfs
@@ -40,7 +30,6 @@ pub struct S3BlockStore {
     // When that changes we will need to use a mutex, either local or in Dynamo
     pub client: Client,
     pub bucket: String,
-    pub dynamodb: DynamoPinStore,
     pub orbit: Cid,
 }
 
@@ -60,7 +49,6 @@ impl S3DataStore {
         S3DataStore {
             client: new_client(config.clone()),
             bucket: config.bucket,
-            dynamodb: DynamoPinStore::new(config.dynamodb, orbit),
             orbit,
         }
     }
@@ -111,7 +99,6 @@ impl S3BlockStore {
         S3BlockStore {
             client: new_client(config.clone()),
             bucket: config.bucket,
-            dynamodb: DynamoPinStore::new(config.dynamodb, orbit),
             orbit,
         }
     }
@@ -256,100 +243,3 @@ impl BlockStore for S3BlockStore {
         unimplemented!("wipe")
     }
 }
-
-#[async_trait]
-impl DataStore for S3DataStore {
-    fn new(path: PathBuf) -> S3DataStore {
-        let (config, orbit) = path_to_config(path);
-        S3DataStore::new_(config, orbit)
-    }
-
-    async fn init(&self) -> Result<(), Error> {
-        self.dynamodb.healthcheck().await?;
-        self.client
-            .head_bucket()
-            .bucket(self.bucket.clone())
-            .send()
-            .await?;
-        Ok(())
-    }
-
-    async fn open(&self) -> Result<(), Error> {
-        Ok(())
-    }
-
-    // None of those methods are implemented for Sled of Fs data stores 🤡
-
-    async fn contains(&self, _col: Column, _key: &[u8]) -> Result<bool, Error> {
-        Err(anyhow::anyhow!("not implemented"))
-    }
-
-    async fn get(&self, _col: Column, _key: &[u8]) -> Result<Option<Vec<u8>>, Error> {
-        Err(anyhow::anyhow!("not implemented"))
-    }
-
-    async fn put(&self, _col: Column, _key: &[u8], _value: &[u8]) -> Result<(), Error> {
-        Err(anyhow::anyhow!("not implemented"))
-    }
-
-    async fn remove(&self, _col: Column, _key: &[u8]) -> Result<(), Error> {
-        Err(anyhow::anyhow!("not implemented"))
-    }
-
-    async fn wipe(&self) {
-        todo!()
-    }
-}
-
-#[async_trait]
-impl PinStore for S3DataStore {
-    async fn is_pinned(&self, _cid: &Cid) -> Result<bool, Error> {
-        // self.dynamodb.is_pinned(cid).await
-        unimplemented!("Unused (so untested)");
-    }
-
-    async fn insert_direct_pin(&self, _target: &Cid) -> Result<(), Error> {
-        // self.dynamodb.insert_direct_pin(target).await
-        unimplemented!("Unused (so untested)");
-    }
-
-    async fn insert_recursive_pin(
-        &self,
-        _target: &Cid,
-        _referenced: References<'_>,
-    ) -> Result<(), Error> {
-        // self.dynamodb.insert_recursive_pin(target, referenced).await
-        unimplemented!("Unused (so untested)");
-    }
-
-    async fn remove_direct_pin(&self, _target: &Cid) -> Result<(), Error> {
-        // self.dynamodb.remove_direct_pin(target).await
-        unimplemented!("Unused (so untested)");
-    }
-
-    async fn remove_recursive_pin(
-        &self,
-        _target: &Cid,
-        _referenced: References<'_>,
-    ) -> Result<(), Error> {
-        // self.dynamodb.remove_recursive_pin(target, referenced).await
-        unimplemented!("Unused (so untested)");
-    }
-
-    async fn list(
-        &self,
-        _requirement: Option<PinMode>,
-    ) -> futures::stream::BoxStream<'static, Result<(Cid, PinMode), Error>> {
-        // self.dynamodb.list(requirement).await
-        unimplemented!("Unused (so untested)");
-    }
-
-    async fn query(
-        &self,
-        _ids: Vec<Cid>,
-        _requirement: Option<PinMode>,
-    ) -> Result<Vec<(Cid, PinKind<Cid>)>, Error> {
-        // self.dynamodb.query(ids, requirement).await
-        unimplemented!("Unused (so untested)");
-    }
-}

From a3be0b8376605838e765f140f17d1cf94bdca6a5 Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Fri, 21 Oct 2022 15:11:07 +0200
Subject: [PATCH 04/73] remove Ipfs from kv, remove ObjectReader for
 single-block contents

---
 src/kv/entries.rs |  60 -------------------
 src/kv/mod.rs     |  52 ++++++-----------
 src/kv/store.rs   | 144 ++++++++++------------------------------------
 3 files changed, 48 insertions(+), 208 deletions(-)

diff --git a/src/kv/entries.rs b/src/kv/entries.rs
index 467f3285..fb9cae13 100644
--- a/src/kv/entries.rs
+++ b/src/kv/entries.rs
@@ -13,66 +13,6 @@ use rocket::tokio::io::AsyncRead;
 use tokio_stream::iter;
 use tokio_util::io::{ReaderStream, StreamReader};
 
-pub type ObjectReader =
-    StreamReader<BoxStream<'static, Result<Cursor<Block>, io::Error>>, Cursor<Block>>;
-
-pub fn read_from_store(ipfs: Ipfs, content: Vec<(Cid, u32)>) -> ObjectReader {
-    let chunk_stream = Box::pin(
-        iter(content)
-            .then(move |(cid, _)| get_block(ipfs.clone(), cid))
-            .map_ok(Cursor::new)
-            .map_err(|ipfs_err| io::Error::new(ErrorKind::Other, ipfs_err)),
-    );
-    StreamReader::new(chunk_stream)
-}
-
-async fn get_block(ipfs: Ipfs, cid: Cid) -> Result<Block, ipfs::Error> {
-    ipfs.get_block(&cid).await
-}
-
-pub async fn write_to_store<R>(store: &Ipfs, source: R) -> anyhow::Result<Cid>
-where
-    R: AsyncRead + Unpin,
-{
-    let mut reader = ReaderStream::new(source);
-    let mut buffer: Vec<u8> = Vec::new();
-    let mut content: Vec<(Cid, u32)> = Vec::new();
-    while let Some(chunk) = reader.next().await.transpose()? {
-        buffer.write_all(&chunk)?;
-        while buffer.len() >= KeplerParams::MAX_BLOCK_SIZE {
-            flush_buffer_to_block(store, &mut buffer, &mut content).await?;
-        }
-    }
-    while !buffer.is_empty() {
-        flush_buffer_to_block(store, &mut buffer, &mut content).await?;
-    }
-    let block = to_block(&content)?;
-    let cid = store.put_block(block).await?;
-    Ok(cid)
-}
-
-async fn flush_buffer_to_block(
-    store: &Ipfs,
-    buffer: &mut Vec<u8>,
-    content: &mut Vec<(Cid, u32)>,
-) -> Result<(), io::Error> {
-    let len = KeplerParams::MAX_BLOCK_SIZE.min(buffer.len());
-    if len > 0 {
-        let (block_data, overflow) = buffer.split_at(len);
-        let block =
-            to_block_raw(&block_data).map_err(|e| io::Error::new(ErrorKind::InvalidData, e))?;
-        *buffer = overflow.to_vec();
-        tracing::debug!("flushing {} bytes to block {}", len, block.cid());
-        let cid = store
-            .put_block(block)
-            .await
-            .map_err(|e| io::Error::new(ErrorKind::Other, e))?;
-        tracing::debug!("block {} flushed {} bytes with {}", content.len(), len, cid);
-        content.push((cid, len as u32));
-    }
-    Ok(())
-}
-
 #[derive(DagCbor, PartialEq, Eq, Debug, Clone)]
 pub struct Object {
     pub key: Vec<u8>,
diff --git a/src/kv/mod.rs b/src/kv/mod.rs
index b10899c8..5d479483 100644
--- a/src/kv/mod.rs
+++ b/src/kv/mod.rs
@@ -11,48 +11,30 @@ pub mod behaviour;
 mod entries;
 mod store;
 
-use super::{
-    ipfs::{Block, Ipfs},
-    orbit::AbortOnDrop,
-};
+use super::{ipfs::Block, orbit::AbortOnDrop};
 
-pub use entries::{Object, ObjectBuilder, ObjectReader};
+pub use entries::{Object, ObjectBuilder};
 pub use store::Store;
 
 type TaskHandle = AbortOnDrop<()>;
 
 #[derive(Clone)]
-pub struct Service {
-    pub store: Store,
-    _task: Arc<TaskHandle>,
+pub struct Service<B> {
+    pub store: Store<B>,
 }
 
-impl Service {
-    fn new(store: Store, task: TaskHandle) -> Self {
-        Self {
-            store,
-            _task: Arc::new(task),
-        }
+impl Service<B> {
+    fn new(store: Store<B>) -> Self {
+        Self { store }
     }
 
-    pub async fn start(store: Store) -> Result<Self> {
-        let events = store
-            .ipfs
-            .pubsub_subscribe(store.id.clone())
-            .await?
-            .map(|msg| match bincode::deserialize(&msg.data) {
-                Ok(kv_msg) => Ok((msg.source, kv_msg)),
-                Err(e) => Err(anyhow!(e)),
-            });
-        let peer_id = store.ipfs.identity().await?.0.to_peer_id();
-        let task = AbortOnDrop::new(tokio::spawn(kv_task(events, store.clone(), peer_id)));
-        store.request_heads().await?;
-        Ok(Service::new(store, task))
+    pub async fn start(store: Store<B>) -> Result<Self> {
+        Ok(Self::new(store))
     }
 }
 
-impl std::ops::Deref for Service {
-    type Target = Store;
+impl<B> std::ops::Deref for Service<B> {
+    type Target = Store<B>;
     fn deref(&self) -> &Self::Target {
         &self.store
     }
@@ -98,9 +80,9 @@ enum KVMessage {
     StateReq,
 }
 
-async fn kv_task(
+async fn kv_task<B>(
     events: impl Stream<Item = Result<(PeerId, KVMessage)>> + Send,
-    store: Store,
+    store: Store<B>,
     peer_id: PeerId,
 ) {
     debug!("starting KV task");
@@ -118,11 +100,11 @@ async fn kv_task(
                     };
                 }
                 Ok((p, KVMessage::StateReq)) => {
-                    debug!("{} requests state", p);
+                    // debug!("{} requests state", p);
                     // send heads
-                    if let Err(e) = store.broadcast_heads().await {
-                        error!("failed to broadcast heads {}", e);
-                    };
+                    // if let Err(e) = store.broadcast_heads().await {
+                    //     error!("failed to broadcast heads {}", e);
+                    // };
                 }
                 Err(e) => {
                     error!("{}", e);
diff --git a/src/kv/store.rs b/src/kv/store.rs
index 834eb58e..4c54a584 100644
--- a/src/kv/store.rs
+++ b/src/kv/store.rs
@@ -1,6 +1,7 @@
 use crate::indexes::{AddRemoveSetStore, HeadStore};
 use crate::kv::entries::{read_from_store, write_to_store};
 use crate::kv::{Object, ObjectBuilder, Service};
+use crate::storage::ImmutableStore;
 use anyhow::Result;
 use async_recursion::async_recursion;
 use futures::stream::{self, StreamExt, TryStreamExt};
@@ -9,7 +10,7 @@ use rocket::{futures::future::try_join_all, tokio::io::AsyncRead};
 use std::{collections::BTreeMap, convert::TryFrom};
 use tracing::{debug, instrument};
 
-use super::{to_block, Block, Ipfs, KVMessage, ObjectReader};
+use super::{to_block, Block, KVMessage, ObjectReader};
 use crate::config;
 
 #[derive(DagCbor)]
@@ -98,21 +99,21 @@ impl Element {
 }
 
 #[derive(Clone)]
-pub struct Store {
+pub struct Store<B> {
     pub id: String,
-    pub ipfs: Ipfs,
+    blocks: B,
     index: AddRemoveSetStore,
     heads: HeadStore,
 }
 
-impl Store {
-    pub async fn new(orbit_id: Cid, ipfs: Ipfs, config: config::IndexStorage) -> Result<Self> {
+impl Store<B> {
+    pub async fn new(orbit_id: Cid, blocks: B, config: config::IndexStorage) -> Result<Self> {
         let index = AddRemoveSetStore::new(orbit_id, "kv".to_string(), config.clone()).await?;
         // heads tracking store
         let heads = HeadStore::new(orbit_id, "kv".to_string(), "heads".to_string(), config).await?;
         Ok(Self {
             id: orbit_id.to_string_of_base(Base::Base58Btc)?,
-            ipfs,
+            blocks,
             index,
             heads,
         })
@@ -145,27 +146,34 @@ impl Store {
             .await
             .into_iter()
     }
+}
 
+impl Store<B>
+where
+    B: ImmutableStore,
+{
     #[instrument(name = "kv::get", skip_all)]
-    pub async fn get<N: AsRef<[u8]>>(&self, name: N) -> Result<Option<Object>> {
-        let key = name;
-        match self.index.element(&key).await? {
+    pub async fn get<N>(&self, name: N) -> Result<Option<Object>>
+    where
+        N: AsRef<[u8]>,
+    {
+        let cid = match self.index.element(&name).await? {
             Some(Element(_, cid)) => Ok(
                 match self
                     .index
-                    .is_tombstoned(&Version(key, cid).to_bytes())
+                    .is_tombstoned(&Version(name, cid).to_bytes())
                     .await?
                 {
-                    false => Some(self.ipfs.get_block(&cid).await?.decode()?),
-                    _ => None,
+                    false => self.blocks.read_to_vec(cid).await,
+                    _ => return Ok(None),
                 },
             ),
-            None => Ok(None),
-        }
+            None => return Ok(None),
+        };
     }
 
     #[instrument(name = "kv::read", skip_all)]
-    pub async fn read<N>(&self, key: N) -> Result<Option<(BTreeMap<String, String>, ObjectReader)>>
+    pub async fn read<N>(&self, key: N) -> Result<Option<(BTreeMap<String, String>, B::Readable)>>
     where
         N: AsRef<[u8]>,
     {
@@ -173,34 +181,18 @@ impl Store {
             Ok(Some(content)) => content,
             _ => return Ok(None),
         };
-        match self
-            .ipfs
-            .get_block(&kv_obj.value)
-            .await?
-            .decode::<DagCborCodec, Vec<(Cid, u32)>>()
-        {
-            Ok(content) => Ok(Some((
-                kv_obj.metadata,
-                read_from_store(self.ipfs.clone(), content),
-            ))),
-            Err(_) => Ok(None),
+        match self.blocks.read(&kv_obj.value).await? {
+            Some(r) => Ok(Some((kv_obj.metadata, r))),
+            None => Err(anyhow!("Indexed contents missing from block store")),
         }
     }
 
-    #[instrument(name = "kv::request_heads", skip_all)]
-    pub(crate) async fn request_heads(&self) -> Result<()> {
-        debug!("requesting heads");
-        self.ipfs
-            .pubsub_publish(self.id.clone(), bincode::serialize(&KVMessage::StateReq)?)
-            .await?;
-        Ok(())
-    }
-
     #[instrument(name = "kv::write", skip_all)]
     pub async fn write<N, R>(
         &self,
         add: impl IntoIterator<Item = (ObjectBuilder, R)>,
         remove: impl IntoIterator<Item = (N, Option<(u64, Cid)>, Cid)>,
+        // TODO return list of new heads to be broadcast?
     ) -> Result<()>
     where
         N: AsRef<[u8]>,
@@ -209,10 +201,11 @@ impl Store {
         tracing::debug!("writing tx");
         let indexes: Vec<(Vec<u8>, Cid)> = try_join_all(add.into_iter().map(|(o, r)| async {
             // tracing::debug!("adding {:#?}", &o.key);
-            let cid = write_to_store(&self.ipfs, r).await?;
+            // store aaalllllll the content bytes under 1 CID
+            let cid = Cid::new_v1(0x55, self.blocks.write(r).await?);
             let obj = o.add_content(cid);
             let block = obj.to_block()?;
-            let obj_cid = self.ipfs.put_block(block).await?;
+            let obj_cid = Cid::new_v1(obj.cid().codec(), self.blocks.write(block.data()).await?);
             Ok((obj.key, obj_cid)) as Result<(Vec<u8>, Cid)>
         }))
         .await?
@@ -265,28 +258,6 @@ impl Store {
         // apply/pin root/update heads
         self.apply(&(block, delta), adds.0, rmvs.0).await?;
 
-        // broadcast
-        self.broadcast_heads().await?;
-        Ok(())
-    }
-
-    #[instrument(name = "kv::broadcast_heads", skip_all)]
-    pub(crate) async fn broadcast_heads(&self) -> Result<()> {
-        let (heads, height) = self.heads.get_heads().await?;
-        if !heads.is_empty() {
-            debug!(
-                "broadcasting {} heads at maxheight {} on {}",
-                heads.len(),
-                height,
-                self.id,
-            );
-            self.ipfs
-                .pubsub_publish(
-                    self.id.clone(),
-                    bincode::serialize(&KVMessage::Heads(heads))?,
-                )
-                .await?;
-        }
         Ok(())
     }
 
@@ -345,63 +316,10 @@ impl Store {
         self.heads
             .new_heads([*block.cid()], delta.prev.clone())
             .await?;
-        self.ipfs.put_block(block.clone()).await?;
+        self.blocks.write(block.data()).await?;
 
         Ok(())
     }
-
-    #[async_recursion]
-    pub(crate) async fn try_merge_heads(
-        &self,
-        heads: impl Iterator<Item = Cid> + Send + 'async_recursion,
-    ) -> Result<()> {
-        try_join_all(heads.map(|head| async move {
-            // fetch head block check block is an event
-            let delta_block = self.ipfs.get_block(&head).await?;
-            let delta: LinkedDelta = delta_block.decode()?;
-
-            // recurse through unseen prevs first
-            self.try_merge_heads(
-                stream::iter(delta.prev.iter().map(Ok).collect::<Vec<Result<_>>>())
-                    .try_filter_map(|p| async move {
-                        self.heads.get_height(p).await.map(|o| match o {
-                            Some(_) => None,
-                            None => Some(p),
-                        })
-                    })
-                    .try_collect::<Vec<Cid>>()
-                    .await?
-                    .into_iter(),
-            )
-            .await?;
-
-            let adds: Vec<(Vec<u8>, Cid)> =
-                try_join_all(delta.delta.add.iter().map(|c| async move {
-                    let obj: Object = self.ipfs.get_block(c).await?.decode()?;
-                    Ok((obj.key, *c)) as Result<(Vec<u8>, Cid)>
-                }))
-                .await?;
-
-            let removes: Vec<(Vec<u8>, Cid)> =
-                try_join_all(delta.delta.rmv.iter().map(|c| async move {
-                    let obj: Object = self.ipfs.get_block(&c.0).await?.decode()?;
-                    Ok((obj.key, c.0)) as Result<(Vec<u8>, Cid)>
-                }))
-                .await?;
-
-            // TODO verify authz stuff
-
-            self.apply(&(delta_block, delta), adds, removes).await?;
-
-            // dispatch ipfs::sync
-            debug!("syncing head {}", head);
-
-            self.ipfs.insert_pin(&head, true).await?;
-            Ok(()) as Result<()>
-        }))
-        .await?;
-        Ok(())
-    }
     pub async fn start_service(self) -> Result<Service> {
         Service::start(self).await
     }

From 65e4a7ddfa1dc5839fc1bb44703e4d00473a3ecc Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Fri, 21 Oct 2022 15:59:12 +0200
Subject: [PATCH 05/73] remove ipfs from cap store and service

---
 src/capabilities/mod.rs   | 51 ++++++++++-------------------
 src/capabilities/store.rs | 67 +++++++--------------------------------
 2 files changed, 29 insertions(+), 89 deletions(-)

diff --git a/src/capabilities/mod.rs b/src/capabilities/mod.rs
index 5e471b27..88d01a72 100644
--- a/src/capabilities/mod.rs
+++ b/src/capabilities/mod.rs
@@ -16,46 +16,29 @@ pub trait Invoke<T> {
 }
 
 #[derive(Clone)]
-pub struct Service {
-    pub store: Store,
-    _task: Arc<AbortOnDrop<()>>,
+pub struct Service<B> {
+    pub store: Store<B>,
 }
 
-impl std::ops::Deref for Service {
-    type Target = Store;
+impl<B> std::ops::Deref for Service<B> {
+    type Target = Store<B>;
     fn deref(&self) -> &Self::Target {
         &self.store
     }
 }
 
-impl Service {
-    fn new(store: Store, task: AbortOnDrop<()>) -> Self {
-        Self {
-            store,
-            _task: Arc::new(task),
-        }
+impl Service<B> {
+    fn new(store: Store<B>) -> Self {
+        Self { store }
     }
-    pub async fn start(store: Store) -> Result<Self> {
-        let events = store
-            .ipfs
-            .pubsub_subscribe(store.id.get_cid().to_string_of_base(Base::Base58Btc)?)
-            .await?
-            .map(
-                |msg| match CapsMessage::decode(DagCborCodec, &mut Cursor::new(&msg.data)) {
-                    Ok(m) => Ok((msg.source, m)),
-                    Err(e) => Err(anyhow!(e)),
-                },
-            );
-        let peer_id = store.ipfs.identity().await?.0.to_peer_id();
-        let task = AbortOnDrop::new(tokio::spawn(caps_task(events, store.clone(), peer_id)));
-        store.request_heads().await?;
-        Ok(Service::new(store, task))
+    pub async fn start(store: Store<B>) -> Result<Self> {
+        Ok(Service::new(store))
     }
 }
 
-async fn caps_task(
+async fn caps_task<B>(
     events: impl Stream<Item = Result<(PeerId, CapsMessage)>> + Send,
-    store: Store,
+    store: Store<B>,
     peer_id: PeerId,
 ) {
     debug!("starting caps task");
@@ -72,12 +55,12 @@ async fn caps_task(
                     }
                 }
                 Ok((_, CapsMessage::StateReq)) => {
-                    if let Err(e) = store.broadcast_heads().await {
-                        debug!(
-                            "failed to broadcast updates in response to state request {}",
-                            e
-                        );
-                    }
+                    // if let Err(e) = store.broadcast_heads().await {
+                    //     debug!(
+                    //         "failed to broadcast updates in response to state request {}",
+                    //         e
+                    //     );
+                    // }
                 }
                 Ok((
                     _,
diff --git a/src/capabilities/store.rs b/src/capabilities/store.rs
index 1b525823..6efaf958 100644
--- a/src/capabilities/store.rs
+++ b/src/capabilities/store.rs
@@ -29,17 +29,17 @@ pub enum InvokeError {
 const SERVICE_NAME: &str = "capabilities";
 
 #[derive(Clone)]
-pub struct Store {
+pub struct Store<B> {
     pub id: ResourceId,
-    pub ipfs: Ipfs,
     pub(crate) root: String,
+    blocks: B,
     index: AddRemoveSetStore,
     delegation_heads: HeadStore,
     invocation_heads: HeadStore,
 }
 
-impl Store {
-    pub async fn new(oid: &OrbitId, ipfs: Ipfs, config: config::IndexStorage) -> Result<Self> {
+impl Store<B> {
+    pub async fn new(oid: &OrbitId, blocks: B, config: config::IndexStorage) -> Result<Self> {
         let id = oid
             .clone()
             .to_resource(Some(SERVICE_NAME.to_string()), None, None);
@@ -70,7 +70,7 @@ impl Store {
         .await?;
         Ok(Self {
             id,
-            ipfs,
+            blocks,
             index,
             delegation_heads,
             invocation_heads,
@@ -84,7 +84,6 @@ impl Store {
     pub async fn transact(&self, updates: Updates) -> Result<()> {
         let event = self.make_event(updates).await?;
         self.apply(event).await?;
-        self.broadcast_heads().await?;
         Ok(())
     }
 
@@ -99,7 +98,7 @@ impl Store {
             revoke: event.revoke.iter().map(|r| *r.0.block.cid()).collect(),
         };
 
-        let cid = self.ipfs.put_block(eb.to_block()?).await?;
+        let cid = self.blocks.write(eb.to_block()?.data()).await?;
 
         // write element indexes
         try_join_all(event.delegate.into_iter().map(|d| async {
@@ -109,9 +108,9 @@ impl Store {
                 .await?;
             tracing::debug!("applied delegation {:?}", d.1.block.cid());
             // put delegation block (encoded ucan or cacao)
-            self.ipfs.put_block(d.1.block).await?;
+            self.blocks.write(d.1.block.data()).await?;
             // put link block
-            self.ipfs.put_block(d.0.block).await?;
+            self.blocks.write(d.0.block.data()).await?;
             Result::<()>::Ok(())
         }))
         .await?;
@@ -126,9 +125,9 @@ impl Store {
                 .await?;
             tracing::debug!("applied revocation {:?}", r.1.block.cid());
             // put revocation block (encoded ucan revocation or cacao)
-            self.ipfs.put_block(r.1.block).await?;
+            self.blocks.write(r.1.block.data()).await?;
             // put link block
-            self.ipfs.put_block(r.0.block).await?;
+            self.blocks.write(r.0.block.data()).await?;
             Result::<()>::Ok(())
         }))
         .await?;
@@ -212,7 +211,7 @@ impl Store {
             prev: event.prev,
             invoke: event.invoke.iter().map(|i| *i.0.block.cid()).collect(),
         };
-        let cid = self.ipfs.put_block(eb.to_block()?).await?;
+        let cid = self.blocks.write(eb.to_block()?.data()).await?;
 
         for e in event.invoke.iter() {
             self.index
@@ -284,49 +283,7 @@ impl Store {
     where
         T: FromBlock,
     {
-        WithBlock::<T>::try_from(self.ipfs.get_block(c).await?)
-    }
-
-    pub(crate) async fn broadcast_heads(&self) -> Result<()> {
-        let updates = self.delegation_heads.get_heads().await?.0;
-        let invocations = self.invocation_heads.get_heads().await?.0;
-        if !updates.is_empty() || !invocations.is_empty() {
-            debug!(
-                "broadcasting {} update heads and {} invocation heads on {}",
-                updates.len(),
-                invocations.len(),
-                self.id,
-            );
-            self.ipfs
-                .pubsub_publish(
-                    self.id
-                        .clone()
-                        .get_cid()
-                        .to_string_of_base(Base::Base58Btc)?,
-                    CapsMessage::Heads {
-                        updates,
-                        invocations,
-                    }
-                    .to_block()?
-                    .into_inner()
-                    .1,
-                )
-                .await?;
-        }
-        Ok(())
-    }
-
-    pub(crate) async fn request_heads(&self) -> Result<()> {
-        self.ipfs
-            .pubsub_publish(
-                self.id
-                    .clone()
-                    .get_cid()
-                    .to_string_of_base(Base::Base58Btc)?,
-                CapsMessage::StateReq.to_block()?.into_inner().1,
-            )
-            .await?;
-        Ok(())
+        WithBlock::<T>::try_from(Block::new_v1(c, self.blocks.read_to_vec(c.hash()).await?))
     }
 
     pub(crate) async fn try_merge_heads(

From 4f4ad2f00eeb6a8129637cb6efd42d53f23b1ea2 Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Fri, 21 Oct 2022 16:00:16 +0200
Subject: [PATCH 06/73] remove ipfs from orbit, remove more ipfs fns

---
 src/ipfs.rs  | 54 ----------------------------------------------
 src/orbit.rs | 61 ++++++++++++----------------------------------------
 2 files changed, 14 insertions(+), 101 deletions(-)

diff --git a/src/ipfs.rs b/src/ipfs.rs
index b25e5bb0..e8d1d428 100644
--- a/src/ipfs.rs
+++ b/src/ipfs.rs
@@ -17,59 +17,5 @@ use crate::{
 };
 
 pub type KeplerParams = DefaultParams;
-// #[derive(Clone, Debug, Default)]
-// pub struct KeplerParams;
-
-// impl StoreParams for KeplerParams {
-//     const MAX_BLOCK_SIZE: usize = 10_485_760;
-//     type Codecs = IpldCodec;
-//     type Hashes = Code;
-// }
-
 pub type Block = OBlock<KeplerParams>;
 pub type Swarm = TSwarm<Types>;
-
-pub async fn create_ipfs<I>(
-    orbit: Cid,
-    config: &config::Config,
-    keypair: Keypair,
-    allowed_peers: I,
-) -> Result<(Ipfs, impl Future<Output = ()>, Receiver<BehaviourEvent>)>
-where
-    I: IntoIterator<Item = PeerId> + 'static,
-{
-    let storage_utils = StorageUtils::new(config.storage.blocks.clone());
-
-    let ipfs_opts = IpfsOptions {
-        ipfs_path: storage_utils.ipfs_path(orbit).await?,
-        keypair,
-        bootstrap: vec![],
-        mdns: false,
-        kad_protocol: Some(format!(
-            "/kepler/{}",
-            orbit.to_string_of_base(Base::Base58Btc)?
-        )),
-        listening_addrs: vec![multiaddr!(P2pCircuit)],
-        span: None,
-    };
-
-    let (sender, receiver) = std::sync::mpsc::sync_channel::<BehaviourEvent>(100);
-    let behaviour = Behaviour::new(sender);
-
-    let (transport_builder, relay_behaviour) = TransportBuilder::new(ipfs_opts.keypair.clone())?
-        .or(MemoryTransport::default())
-        .relay();
-
-    let transport = transport_builder
-        .map_auth()
-        .map(crate::transport::auth_mapper(allowed_peers))
-        .build();
-
-    let (ipfs, ipfs_task) =
-        UninitializedIpfs::<Repo>::new(ipfs_opts, transport, Some(relay_behaviour))
-            .with_extended_behaviour(behaviour)
-            .start()
-            .await?;
-
-    Ok((ipfs, ipfs_task, receiver))
-}
diff --git a/src/orbit.rs b/src/orbit.rs
index 10e061bd..99b23b37 100644
--- a/src/orbit.rs
+++ b/src/orbit.rs
@@ -3,7 +3,6 @@ use crate::{
     cas::ContentAddressedStorage,
     codec::SupportedCodecs,
     config,
-    ipfs::create_ipfs,
     kv::{behaviour::BehaviourProcess, Service as KVService, Store},
     manifest::Manifest,
 };
@@ -52,26 +51,23 @@ impl<T> Deref for AbortOnDrop<T> {
 
 #[derive(Clone, Debug)]
 struct OrbitTasks {
-    _ipfs: Arc<AbortOnDrop<()>>,
     _behaviour_process: BehaviourProcess,
 }
 
 impl OrbitTasks {
-    fn new(ipfs_future: JoinHandle<()>, behaviour_process: BehaviourProcess) -> Self {
-        let ipfs = Arc::new(AbortOnDrop::new(ipfs_future));
+    fn new(behaviour_process: BehaviourProcess) -> Self {
         Self {
-            _ipfs: ipfs,
             _behaviour_process: behaviour_process,
         }
     }
 }
 
 #[derive(Clone)]
-pub struct Orbit {
-    pub service: KVService,
+pub struct Orbit<B> {
+    pub service: KVService<B>,
     _tasks: OrbitTasks,
     pub manifest: Manifest,
-    pub capabilities: CapService,
+    pub capabilities: CapService<B>,
 }
 
 impl Orbit {
@@ -83,52 +79,23 @@ impl Orbit {
     ) -> anyhow::Result<Self> {
         let id = manifest.id().get_cid();
         let local_peer_id = PeerId::from_public_key(&ipfs::PublicKey::Ed25519(kp.public()));
-        let (ipfs, ipfs_future, receiver) = create_ipfs(
-            id,
-            config,
-            Keypair::Ed25519(kp),
-            manifest
-                .bootstrap_peers()
-                .peers
-                .iter()
-                .map(|p| p.id)
-                .collect::<Vec<PeerId>>(),
-        )
-        .await?;
-
-        let ipfs_task = spawn(ipfs_future);
-        if let Some(r) = relay {
-            ipfs.connect(MultiaddrWithoutPeerId::try_from(r.1)?.with(r.0))
-                .await?;
-        };
 
-        let service_store = Store::new(id, ipfs.clone(), config.storage.indexes.clone()).await?;
+        // TODO config into block store
+        let blocks = todo!();
+        let service_store = Store::new(id, blocks.clone(), config.storage.indexes.clone()).await?;
         let service = KVService::start(service_store).await?;
 
-        let cap_store =
-            CapStore::new(manifest.id(), ipfs.clone(), config.storage.indexes.clone()).await?;
+        let cap_store = CapStore::new(
+            manifest.id(),
+            blocks.clone(),
+            config.storage.indexes.clone(),
+        )
+        .await?;
         let capabilities = CapService::start(cap_store).await?;
 
         let behaviour_process = BehaviourProcess::new(service.store.clone(), receiver);
 
-        let tasks = OrbitTasks::new(ipfs_task, behaviour_process);
-
-        tokio_stream::iter(
-            manifest
-                .bootstrap_peers()
-                .peers
-                .iter()
-                .filter(|p| p.id != local_peer_id)
-                .flat_map(|peer| {
-                    peer.addrs
-                        .clone()
-                        .into_iter()
-                        .zip(std::iter::repeat(peer.id))
-                })
-                .map(|(addr, peer_id)| Ok(MultiaddrWithoutPeerId::try_from(addr)?.with(peer_id))),
-        )
-        .try_for_each(|multiaddr| ipfs.connect(multiaddr))
-        .await?;
+        let tasks = OrbitTasks::new(behaviour_process);
 
         Ok(Orbit {
             service,

From bf44fa46f0091f93dcaccf3eb1560f0d7270ba92 Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Fri, 21 Oct 2022 16:52:10 +0200
Subject: [PATCH 07/73] remove various ipfs crate imports

---
 src/auth_guards.rs        |  6 ++++--
 src/capabilities/mod.rs   |  2 +-
 src/capabilities/store.rs |  2 +-
 src/kv/behaviour.rs       |  3 +--
 src/kv/entries.rs         |  3 +--
 src/kv/mod.rs             |  6 +++---
 src/orbit.rs              | 36 ++----------------------------------
 src/routes/mod.rs         |  9 ++++++---
 src/transport.rs          |  8 +++++---
 9 files changed, 24 insertions(+), 51 deletions(-)

diff --git a/src/auth_guards.rs b/src/auth_guards.rs
index 275ae164..a8427b8a 100644
--- a/src/auth_guards.rs
+++ b/src/auth_guards.rs
@@ -5,12 +5,14 @@ use crate::orbit::{create_orbit, load_orbit, Orbit};
 use crate::relay::RelayNode;
 use crate::routes::Metadata;
 use anyhow::Result;
-use ipfs::{Multiaddr, PeerId};
 use kepler_lib::{
     libipld::Cid,
     resource::{OrbitId, ResourceId},
 };
-use libp2p::identity::ed25519::Keypair as Ed25519Keypair;
+use libp2p::{
+    core::{Multiaddr, PeerId},
+    identity::ed25519::Keypair as Ed25519Keypair,
+};
 use rocket::{
     futures::future::try_join_all,
     http::Status,
diff --git a/src/capabilities/mod.rs b/src/capabilities/mod.rs
index 88d01a72..15b52730 100644
--- a/src/capabilities/mod.rs
+++ b/src/capabilities/mod.rs
@@ -2,8 +2,8 @@ pub mod store;
 
 use crate::orbit::AbortOnDrop;
 use anyhow::Result;
-use ipfs::PeerId;
 use kepler_lib::libipld::{cbor::DagCborCodec, codec::Decode, multibase::Base, Cid};
+use libp2p::identity::PeerId;
 use rocket::futures::{Stream, StreamExt};
 use store::{CapsMessage, Store};
 
diff --git a/src/capabilities/store.rs b/src/capabilities/store.rs
index 6efaf958..12c2c782 100644
--- a/src/capabilities/store.rs
+++ b/src/capabilities/store.rs
@@ -1,7 +1,7 @@
 use crate::{
     authorization::{CapStore, Delegation, Invocation, Revocation, Verifiable},
     indexes::{AddRemoveSetStore, HeadStore},
-    ipfs::{Block, Ipfs},
+    ipfs::Block,
     kv::to_block_raw,
 };
 use anyhow::Result;
diff --git a/src/kv/behaviour.rs b/src/kv/behaviour.rs
index 676acad9..2f92bc47 100644
--- a/src/kv/behaviour.rs
+++ b/src/kv/behaviour.rs
@@ -6,9 +6,8 @@ use std::{
     task::{Context, Poll},
 };
 
-use ipfs::{Multiaddr, PeerId};
 use libp2p::{
-    core::{connection::ConnectionId, ConnectedPoint},
+    core::{connection::ConnectionId, ConnectedPoint, Multiaddr, PeerId},
     swarm::{
         handler::DummyConnectionHandler, IntoConnectionHandler, NetworkBehaviour,
         NetworkBehaviourAction, PollParameters,
diff --git a/src/kv/entries.rs b/src/kv/entries.rs
index fb9cae13..9cb7d1dc 100644
--- a/src/kv/entries.rs
+++ b/src/kv/entries.rs
@@ -1,5 +1,5 @@
 use super::{to_block, to_block_raw};
-use crate::ipfs::{Block, Ipfs, KeplerParams};
+use crate::ipfs::{Block, KeplerParams};
 use anyhow::Result;
 use kepler_lib::libipld::{cid::Cid, store::StoreParams, DagCbor};
 use libp2p::futures::stream::BoxStream;
@@ -67,7 +67,6 @@ impl ObjectBuilder {
 
 #[cfg(test)]
 mod test {
-    use ipfs::IpfsOptions;
 
     use super::*;
     use crate::{config, kv::DagCborCodec, tracing::tracing_try_init};
diff --git a/src/kv/mod.rs b/src/kv/mod.rs
index 5d479483..152095ff 100644
--- a/src/kv/mod.rs
+++ b/src/kv/mod.rs
@@ -1,8 +1,8 @@
 use anyhow::Result;
-use ipfs::PeerId;
 use kepler_lib::libipld::{
     cbor::DagCborCodec, cid::Cid, codec::Encode, multihash::Code, raw::RawCodec,
 };
+use libp2p::core::PeerId;
 use rocket::futures::{Stream, StreamExt};
 use serde::{Deserialize, Serialize};
 use std::sync::Arc;
@@ -116,10 +116,10 @@ async fn kv_task<B>(
 
 #[cfg(test)]
 mod test {
-    use ipfs::{Keypair, MultiaddrWithoutPeerId, Protocol};
+    use libp2p::{identity::Keypair, multiaddr::Protocol};
 
     use super::*;
-    use crate::{config, ipfs::create_ipfs, relay::test::test_relay};
+    use crate::{config, relay::test::test_relay};
     use std::{
         collections::BTreeMap, convert::TryFrom, path::PathBuf, str::FromStr, time::Duration,
     };
diff --git a/src/orbit.rs b/src/orbit.rs
index 99b23b37..81d5042c 100644
--- a/src/orbit.rs
+++ b/src/orbit.rs
@@ -7,7 +7,6 @@ use crate::{
     manifest::Manifest,
 };
 use anyhow::{anyhow, Result};
-use ipfs::{MultiaddrWithPeerId, MultiaddrWithoutPeerId};
 use kepler_lib::libipld::cid::{
     multihash::{Code, MultihashDigest},
     Cid,
@@ -15,7 +14,7 @@ use kepler_lib::libipld::cid::{
 use kepler_lib::resource::OrbitId;
 use libp2p::{
     core::Multiaddr,
-    identity::{ed25519::Keypair as Ed25519Keypair, Keypair},
+    identity::{ed25519::Keypair as Ed25519Keypair, Keypair, PublicKey},
     PeerId,
 };
 use rocket::{futures::TryStreamExt, tokio::task::JoinHandle};
@@ -78,7 +77,7 @@ impl Orbit {
         relay: Option<(PeerId, Multiaddr)>,
     ) -> anyhow::Result<Self> {
         let id = manifest.id().get_cid();
-        let local_peer_id = PeerId::from_public_key(&ipfs::PublicKey::Ed25519(kp.public()));
+        let local_peer_id = PeerId::from_public_key(&PublicKey::Ed25519(kp.public()));
 
         // TODO config into block store
         let blocks = todo!();
@@ -186,37 +185,6 @@ pub fn hash_same<B: AsRef<[u8]>>(c: &Cid, b: B) -> Result<Cid> {
     ))
 }
 
-#[rocket::async_trait]
-impl ContentAddressedStorage for Orbit {
-    type Error = anyhow::Error;
-    async fn put(
-        &self,
-        content: &[u8],
-        codec: SupportedCodecs,
-    ) -> Result<Cid, <Self as ContentAddressedStorage>::Error> {
-        self.service.ipfs.put(content, codec).await
-    }
-    async fn get(
-        &self,
-        address: &Cid,
-    ) -> Result<Option<Vec<u8>>, <Self as ContentAddressedStorage>::Error> {
-        ContentAddressedStorage::get(&self.service.ipfs, address).await
-    }
-    async fn delete(&self, address: &Cid) -> Result<(), <Self as ContentAddressedStorage>::Error> {
-        self.service.ipfs.delete(address).await
-    }
-    async fn list(&self) -> Result<Vec<Cid>, <Self as ContentAddressedStorage>::Error> {
-        self.service.ipfs.list().await
-    }
-}
-
-impl Deref for Orbit {
-    type Target = Manifest;
-    fn deref(&self) -> &Self::Target {
-        &self.manifest
-    }
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;
diff --git a/src/routes/mod.rs b/src/routes/mod.rs
index f0de98c7..cb7bb75d 100644
--- a/src/routes/mod.rs
+++ b/src/routes/mod.rs
@@ -1,7 +1,10 @@
 use anyhow::Result;
-use ipfs::{PeerId, Protocol};
 use kepler_lib::libipld::Cid;
-use libp2p::identity::ed25519::Keypair as Ed25519Keypair;
+use libp2p::{
+    core::PeerId,
+    identity::{ed25519::Keypair as Ed25519Keypair, PublicKey},
+    multiaddr::Protocol,
+};
 use rocket::{
     data::{Data, ToByteUnit},
     http::{Header, Status},
@@ -89,7 +92,7 @@ pub fn open_host_key(
     s: &State<RwLock<HashMap<PeerId, Ed25519Keypair>>>,
 ) -> Result<String, (Status, &'static str)> {
     let keypair = Ed25519Keypair::generate();
-    let id = ipfs::PublicKey::Ed25519(keypair.public()).to_peer_id();
+    let id = PublicKey::Ed25519(keypair.public()).to_peer_id();
     s.write()
         .map_err(|_| (Status::InternalServerError, "cant read keys"))?
         .insert(id, keypair);
diff --git a/src/transport.rs b/src/transport.rs
index b4dd0c29..005839b3 100644
--- a/src/transport.rs
+++ b/src/transport.rs
@@ -3,8 +3,10 @@ use std::{
     future::{ready, Ready},
 };
 
-use ipfs::PeerId;
-use libp2p::{core::Endpoint, noise::NoiseError};
+use libp2p::{
+    core::{Endpoint, PeerId},
+    noise::NoiseError,
+};
 
 pub fn auth_mapper<A, I>(
     i: I,
@@ -42,7 +44,7 @@ mod test {
     use std::convert::TryFrom;
 
     use crate::{config, ipfs::create_ipfs, relay::test::test_relay};
-    use ipfs::{MultiaddrWithoutPeerId, Protocol};
+    use ipfs::multiaddr::Protocol;
     use kepler_lib::libipld::Cid;
     use libp2p::identity::Keypair;
     use std::str::FromStr;

From f212870b3b9907d5773e2476a8934c459bfd613c Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Fri, 21 Oct 2022 17:02:09 +0200
Subject: [PATCH 08/73] fix dependancies

---
 Cargo.lock     | 2727 ++++++++++++++++++++++++------------------------
 Cargo.toml     |    3 +-
 lib/Cargo.toml |    8 +-
 3 files changed, 1350 insertions(+), 1388 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 34ada426..f0c3a40d 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -10,43 +10,44 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
 
 [[package]]
 name = "aead"
-version = "0.3.2"
+version = "0.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7fc95d1bdb8e6666b2b217308eeeb09f2d6728d104be3e31916cc74d15420331"
+checksum = "0b613b8e1e3cf911a086f53f03bf286f52fd7a7258e4fa606f0ef220d39d8877"
 dependencies = [
  "generic-array 0.14.6",
 ]
 
 [[package]]
 name = "aead"
-version = "0.4.3"
+version = "0.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b613b8e1e3cf911a086f53f03bf286f52fd7a7258e4fa606f0ef220d39d8877"
+checksum = "5c192eb8f11fc081b0fe4259ba5af04217d4e0faddd02417310a927911abd7c8"
 dependencies = [
+ "crypto-common",
  "generic-array 0.14.6",
 ]
 
 [[package]]
 name = "aes"
-version = "0.6.0"
+version = "0.7.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "884391ef1066acaa41e766ba8f596341b96e93ce34f9a43e7d24bf0a0eaf0561"
+checksum = "9e8b47f52ea9bae42228d07ec09eb676433d7c4ed1ebdf0f1d1c29ed446f1ab8"
 dependencies = [
- "aes-soft",
- "aesni",
- "cipher 0.2.5",
+ "cfg-if",
+ "cipher 0.3.0",
+ "cpufeatures",
+ "opaque-debug 0.3.0",
 ]
 
 [[package]]
 name = "aes"
-version = "0.7.5"
+version = "0.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9e8b47f52ea9bae42228d07ec09eb676433d7c4ed1ebdf0f1d1c29ed446f1ab8"
+checksum = "bfe0133578c0986e1fe3dfcd4af1cc5b2dd6c3dbf534d69916ce16a2701d40ba"
 dependencies = [
  "cfg-if",
- "cipher 0.3.0",
+ "cipher 0.4.3",
  "cpufeatures",
- "opaque-debug 0.3.0",
 ]
 
 [[package]]
@@ -59,28 +60,22 @@ dependencies = [
  "aes 0.7.5",
  "cipher 0.3.0",
  "ctr 0.8.0",
- "ghash",
+ "ghash 0.4.4",
  "subtle",
 ]
 
 [[package]]
-name = "aes-soft"
-version = "0.6.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "be14c7498ea50828a38d0e24a765ed2effe92a705885b57d029cd67d45744072"
-dependencies = [
- "cipher 0.2.5",
- "opaque-debug 0.3.0",
-]
-
-[[package]]
-name = "aesni"
-version = "0.10.0"
+name = "aes-gcm"
+version = "0.10.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ea2e11f5e94c2f7d386164cc2aa1f97823fed6f259e486940a71c174dd01b0ce"
+checksum = "82e1366e0c69c9f927b1fa5ce2c7bf9eafc8f9268c0b9800729e8b267612447c"
 dependencies = [
- "cipher 0.2.5",
- "opaque-debug 0.3.0",
+ "aead 0.5.1",
+ "aes 0.8.1",
+ "cipher 0.4.3",
+ "ctr 0.9.2",
+ "ghash 0.5.0",
+ "subtle",
 ]
 
 [[package]]
@@ -89,9 +84,9 @@ version = "0.7.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47"
 dependencies = [
- "getrandom 0.2.7",
+ "getrandom 0.2.8",
  "once_cell",
- "version_check",
+ "version_check 0.9.4",
 ]
 
 [[package]]
@@ -112,15 +107,6 @@ dependencies = [
  "libc",
 ]
 
-[[package]]
-name = "ansi_term"
-version = "0.12.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2"
-dependencies = [
- "winapi",
-]
-
 [[package]]
 name = "anyhow"
 version = "1.0.59"
@@ -145,15 +131,6 @@ version = "0.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6"
 
-[[package]]
-name = "ascii-canvas"
-version = "3.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8824ecca2e851cec16968d54a01dd372ef8f95b244fb84b84e70128be347c3c6"
-dependencies = [
- "term",
-]
-
 [[package]]
 name = "asn1_der"
 version = "0.7.5"
@@ -216,7 +193,7 @@ version = "1.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "83e21f3a490c72b3b0cf44962180e60045de2925d8dff97918f7ee43c8f637c7"
 dependencies = [
- "autocfg 1.1.0",
+ "autocfg",
  "concurrent-queue",
  "futures-lite",
  "libc",
@@ -306,9 +283,9 @@ checksum = "7a40729d2133846d9ed0ea60a8b9541bccddab49cd30f0715a1da672fe9a2524"
 
 [[package]]
 name = "async-trait"
-version = "0.1.57"
+version = "0.1.58"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "76464446b8bc32758d7e88ee1a804d9914cd9b1cb264c029899680b0be29826f"
+checksum = "1e805d94e6b5001b651426cf4cd446b1ab5f319d27bab5c644f61de0a804360c"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -340,7 +317,7 @@ version = "0.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b88d82667eca772c4aa12f0f1348b3ae643424c8876448f3f7bd5787032e234c"
 dependencies = [
- "autocfg 1.1.0",
+ "autocfg",
 ]
 
 [[package]]
@@ -360,15 +337,6 @@ dependencies = [
  "winapi",
 ]
 
-[[package]]
-name = "autocfg"
-version = "0.1.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0dde43e75fd43e8a1bf86103336bc699aa8d17ad1be60c76c0bdfd4828e19b78"
-dependencies = [
- "autocfg 1.1.0",
-]
-
 [[package]]
 name = "autocfg"
 version = "1.1.0"
@@ -553,7 +521,7 @@ dependencies = [
  "percent-encoding",
  "regex",
  "ring",
- "time 0.3.14",
+ "time 0.3.15",
  "tracing",
 ]
 
@@ -586,7 +554,7 @@ dependencies = [
  "hyper",
  "hyper-rustls",
  "lazy_static",
- "pin-project 1.0.12",
+ "pin-project",
  "pin-project-lite",
  "tokio",
  "tower",
@@ -620,7 +588,7 @@ dependencies = [
  "hyper",
  "once_cell",
  "percent-encoding",
- "pin-project 1.0.12",
+ "pin-project",
  "tokio",
  "tokio-util 0.6.10",
  "tracing",
@@ -636,7 +604,7 @@ dependencies = [
  "bytes",
  "http",
  "http-body",
- "pin-project 1.0.12",
+ "pin-project",
  "tower",
  "tracing",
 ]
@@ -669,7 +637,7 @@ dependencies = [
  "itoa",
  "num-integer",
  "ryu",
- "time 0.3.14",
+ "time 0.3.15",
 ]
 
 [[package]]
@@ -716,9 +684,15 @@ checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff"
 
 [[package]]
 name = "base64"
-version = "0.13.0"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
+
+[[package]]
+name = "base64ct"
+version = "1.5.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd"
+checksum = "b645a089122eccb6111b4f81cbc1a49f5900ac4666bb93ac027feaecf15607bf"
 
 [[package]]
 name = "bech32"
@@ -742,19 +716,10 @@ dependencies = [
 ]
 
 [[package]]
-name = "bit-set"
-version = "0.5.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1"
-dependencies = [
- "bit-vec",
-]
-
-[[package]]
-name = "bit-vec"
-version = "0.6.3"
+name = "bitfield"
+version = "0.13.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb"
+checksum = "46afbd2983a5d5a7bd740ccb198caf5b82f45c40c09c0eed36052d91cb92e719"
 
 [[package]]
 name = "bitflags"
@@ -762,18 +727,6 @@ version = "1.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 
-[[package]]
-name = "bitvec"
-version = "0.19.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "55f93d0ef3363c364d5976646a38f04cf67cfe1d4c8d160cdea02cab2c116b33"
-dependencies = [
- "funty",
- "radium 0.5.3",
- "tap",
- "wyz",
-]
-
 [[package]]
 name = "bitvec"
 version = "0.20.4"
@@ -781,7 +734,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7774144344a4faa177370406a7ff5f1da24303817368584c6206c8303eb07848"
 dependencies = [
  "funty",
- "radium 0.6.2",
+ "radium",
  "tap",
  "wyz",
 ]
@@ -817,17 +770,6 @@ dependencies = [
  "constant_time_eq",
 ]
 
-[[package]]
-name = "blake2s_simd"
-version = "0.5.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9e461a7034e85b211a4acb57ee2e6730b32912b06c08cc242243c39fc21ae6a2"
-dependencies = [
- "arrayref",
- "arrayvec 0.5.2",
- "constant_time_eq",
-]
-
 [[package]]
 name = "blake2s_simd"
 version = "1.0.0"
@@ -885,12 +827,12 @@ dependencies = [
 
 [[package]]
 name = "block-modes"
-version = "0.7.0"
+version = "0.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "57a0e8073e8baa88212fb5823574c02ebccb395136ba9a164ab89379ec6072f0"
+checksum = "2cb03d1bed155d89dce0f845b7899b18a9a163e148fd004e1c28421a783e2d8e"
 dependencies = [
  "block-padding 0.2.1",
- "cipher 0.2.5",
+ "cipher 0.3.0",
 ]
 
 [[package]]
@@ -924,12 +866,12 @@ dependencies = [
 
 [[package]]
 name = "blowfish"
-version = "0.7.0"
+version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "32fa6a061124e37baba002e496d203e23ba3d7b73750be82dbfbc92913048a5b"
+checksum = "fe3ff3fc1de48c1ac2e3341c4df38b0d1bfb8fdf04632a187c8b75aaa319a7ab"
 dependencies = [
  "byteorder",
- "cipher 0.2.5",
+ "cipher 0.3.0",
  "opaque-debug 0.3.0",
 ]
 
@@ -943,26 +885,20 @@ dependencies = [
 ]
 
 [[package]]
-name = "buffered-reader"
-version = "1.1.3"
+name = "buf_redux"
+version = "0.8.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e9f82920285502602088677aeb65df0909b39c347b38565e553ba0363c242f65"
+checksum = "b953a6887648bb07a535631f2bc00fbdb2a2216f135552cb3f534ed136b9c07f"
 dependencies = [
- "flate2",
- "libc",
+ "memchr",
+ "safemem",
 ]
 
 [[package]]
 name = "bumpalo"
-version = "3.11.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c1ad822118d20d2c234f427000d5acc36eabe1e29a348c89b63dd60b13f28e5d"
-
-[[package]]
-name = "byte-slice-cast"
-version = "1.2.1"
+version = "3.11.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87c5fdd0166095e1d463fc6cc01aa8ce547ad77a4e84d42eb6762b084e28067e"
+checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba"
 
 [[package]]
 name = "byte-tools"
@@ -981,9 +917,6 @@ name = "bytes"
 version = "1.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ec8a7b6a70fde80372154c65702f00a0f56f3e1c36abbc6c440484be248856db"
-dependencies = [
- "serde",
-]
 
 [[package]]
 name = "bytes-utils"
@@ -997,20 +930,20 @@ dependencies = [
 
 [[package]]
 name = "cacaos"
-version = "0.4.0"
-source = "git+https://github.com/spruceid/cacao-rs#066d57282757487a37c8b4d94700b9ed49eb01ed"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "92680fcbb8e24eaa4640c6deca0042db02da17c8b92fde62a0c517bcb6769d7d"
 dependencies = [
  "async-trait",
- "ethers-core",
  "hex",
  "http",
- "iri-string 0.4.1",
- "libipld",
+ "iri-string 0.6.0",
+ "libipld 0.14.0",
  "serde",
- "serde_with",
- "siwe",
+ "serde_with 2.0.1",
+ "siwe 0.5.0",
  "thiserror",
- "time 0.3.14",
+ "time 0.3.15",
  "url",
 ]
 
@@ -1068,26 +1001,26 @@ checksum = "3a4f925191b4367301851c6d99b09890311d74b0d43f274c0b34c86d308a3663"
 
 [[package]]
 name = "capgrok"
-version = "0.2.2"
+version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ab4b6ebc13190cd16ffe4e289f1b8667cc1f6caac781586054d4beff411e7a2a"
+checksum = "c5fec5445658b25821dee5ab2788b0788620194c49cddf17c7aca0431b5ffae8"
 dependencies = [
  "base64 0.12.3",
  "iri-string 0.4.1",
  "serde",
  "serde_json",
- "siwe",
+ "siwe 0.4.2",
  "thiserror",
 ]
 
 [[package]]
 name = "cast5"
-version = "0.9.0"
+version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1285caf81ea1f1ece6b24414c521e625ad0ec94d880625c20f2e65d8d3f78823"
+checksum = "f69790da27038b52ffcf09e7874e1aae353c674d65242549a733ad9372e7281f"
 dependencies = [
  "byteorder",
- "cipher 0.2.5",
+ "cipher 0.3.0",
  "opaque-debug 0.3.0",
 ]
 
@@ -1097,6 +1030,15 @@ version = "1.0.73"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11"
 
+[[package]]
+name = "cfb-mode"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "750dfbb1b1f84475c1a92fed10fa5e76cb11adc0cda5225f137c5cac84e80860"
+dependencies = [
+ "cipher 0.3.0",
+]
+
 [[package]]
 name = "cfg-if"
 version = "1.0.0"
@@ -1152,30 +1094,37 @@ checksum = "f6ed9c8b2d17acb8110c46f1da5bf4a696d745e1474a16db0cd2b49cd0249bf2"
 dependencies = [
  "core2",
  "multibase 0.9.1",
- "multihash 0.16.3",
+ "multihash",
  "serde",
  "serde_bytes",
- "unsigned-varint 0.7.1",
+ "unsigned-varint",
 ]
 
 [[package]]
 name = "cipher"
-version = "0.2.5"
+version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "12f8e7987cbd042a63249497f41aed09f8e65add917ea6566effbc56578d6801"
+checksum = "7ee52072ec15386f770805afd189a01c8841be8696bed250fa2f13c4c0d6dfb7"
 dependencies = [
  "generic-array 0.14.6",
 ]
 
 [[package]]
 name = "cipher"
-version = "0.3.0"
+version = "0.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7ee52072ec15386f770805afd189a01c8841be8696bed250fa2f13c4c0d6dfb7"
+checksum = "d1873270f8f7942c191139cb8a40fd228da6c3fd2fc376d7e92d47aa14aeb59e"
 dependencies = [
- "generic-array 0.14.6",
+ "crypto-common",
+ "inout",
 ]
 
+[[package]]
+name = "circular"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b0fc239e0f6cb375d2402d48afb92f76f5404fd1df208a41930ec81eda078bea"
+
 [[package]]
 name = "clear_on_drop"
 version = "0.2.5"
@@ -1186,13 +1135,22 @@ dependencies = [
 ]
 
 [[package]]
-name = "cmac"
-version = "0.5.1"
+name = "cmake"
+version = "0.1.48"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "73d4de4f7724e5fe70addfb2bd37c2abd2f95084a429d7773b0b9645499b4272"
+checksum = "e8ad8cef104ac57b68b89df3208164d228503abbdce70f6880ffa3d970e7443a"
 dependencies = [
- "crypto-mac 0.10.1",
- "dbl",
+ "cc",
+]
+
+[[package]]
+name = "codespan-reporting"
+version = "0.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e"
+dependencies = [
+ "termcolor",
+ "unicode-width",
 ]
 
 [[package]]
@@ -1212,21 +1170,15 @@ dependencies = [
 
 [[package]]
 name = "const-oid"
-version = "0.5.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "279bc8fc53f788a75c7804af68237d1fce02cde1e275a886a4b320604dc2aeda"
-
-[[package]]
-name = "const-oid"
-version = "0.6.2"
+version = "0.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9d6f2aa4d0537bcc1c74df8755072bd31c1ef1a3a1b85a68e8404a8c353b7b8b"
+checksum = "e4c78c047431fee22c1a7bb92e00ad095a02a983affe4d8a72e2a2c62c1b94f3"
 
 [[package]]
 name = "const-oid"
-version = "0.7.1"
+version = "0.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e4c78c047431fee22c1a7bb92e00ad095a02a983affe4d8a72e2a2c62c1b94f3"
+checksum = "722e23542a15cea1f65d4a1419c4cfd7a26706c70871a13a04238ca3f40f1661"
 
 [[package]]
 name = "constant_time_eq"
@@ -1236,20 +1188,20 @@ checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc"
 
 [[package]]
 name = "cookie"
-version = "0.16.0"
+version = "0.16.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "94d4706de1b0fa5b132270cddffa8585166037822e260a944fe161acd137ca05"
+checksum = "344adc371239ef32293cb1c4fe519592fcf21206c79c02854320afcdf3ab4917"
 dependencies = [
- "aes-gcm",
- "base64 0.13.0",
+ "aes-gcm 0.10.1",
+ "base64 0.13.1",
  "hkdf",
- "hmac 0.12.1",
+ "hmac",
  "percent-encoding",
  "rand 0.8.5",
  "sha2 0.10.6",
  "subtle",
- "time 0.3.14",
- "version_check",
+ "time 0.3.15",
+ "version_check 0.9.4",
 ]
 
 [[package]]
@@ -1286,6 +1238,12 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "crc24"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fd121741cf3eb82c08dd3023eb55bf2665e5f60ec20f89760cf836ae4562e6a0"
+
 [[package]]
 name = "crc32fast"
 version = "1.3.2"
@@ -1307,26 +1265,24 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-epoch"
-version = "0.9.10"
+version = "0.9.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "045ebe27666471bb549370b4b0b3e51b07f56325befa4284db65fc89c02511b1"
+checksum = "f916dfc5d356b0ed9dae65f1db9fc9770aa2851d2662b988ccf4fe3516e86348"
 dependencies = [
- "autocfg 1.1.0",
+ "autocfg",
  "cfg-if",
  "crossbeam-utils",
  "memoffset",
- "once_cell",
  "scopeguard",
 ]
 
 [[package]]
 name = "crossbeam-utils"
-version = "0.8.11"
+version = "0.8.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "51887d4adc7b564537b15adcfb307936f8075dfcd5f00dde9a9f1d29383682bc"
+checksum = "edbafec5fa1f196ca66527c1b12c2ec4745ca14b50f1ad8f9f6f720b55d11fac"
 dependencies = [
  "cfg-if",
- "once_cell",
 ]
 
 [[package]]
@@ -1337,21 +1293,19 @@ checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7"
 
 [[package]]
 name = "crypto-bigint"
-version = "0.2.11"
+version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f83bd3bb4314701c568e340cd8cf78c975aa0ca79e03d3f6d1677d5b0c9c0c03"
+checksum = "03c6a1d5fa1de37e071642dfa44ec552ca5b299adb128fab16138e24b548fd21"
 dependencies = [
  "generic-array 0.14.6",
- "rand_core 0.6.4",
  "subtle",
- "zeroize",
 ]
 
 [[package]]
 name = "crypto-bigint"
-version = "0.3.2"
+version = "0.4.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "03c6a1d5fa1de37e071642dfa44ec552ca5b299adb128fab16138e24b548fd21"
+checksum = "ef2b4b23cddf68b89b8f8069890e8c270d54e2d5fe1b143820234805e4cb17ef"
 dependencies = [
  "generic-array 0.14.6",
  "rand_core 0.6.4",
@@ -1366,30 +1320,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
 dependencies = [
  "generic-array 0.14.6",
+ "rand_core 0.6.4",
  "typenum",
 ]
 
-[[package]]
-name = "crypto-mac"
-version = "0.10.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bff07008ec701e8028e2ceb8f83f0e4274ee62bd2dbdc4fefff2e9a91824081a"
-dependencies = [
- "cipher 0.2.5",
- "generic-array 0.14.6",
- "subtle",
-]
-
-[[package]]
-name = "crypto-mac"
-version = "0.11.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b1d1a86f49236c215f271d40892d5fc950490551400b02ef360692c29815c714"
-dependencies = [
- "generic-array 0.14.6",
- "subtle",
-]
-
 [[package]]
 name = "ct-logs"
 version = "0.8.0"
@@ -1401,9 +1335,9 @@ dependencies = [
 
 [[package]]
 name = "ctor"
-version = "0.1.23"
+version = "0.1.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cdffe87e1d521a10f9696f833fe502293ea446d7f256c06128293a4119bdf4cb"
+checksum = "6d2301688392eb071b0bf1a37be05c469d3cc4dbbd95df672fe28ab021e6a096"
 dependencies = [
  "quote",
  "syn",
@@ -1411,20 +1345,20 @@ dependencies = [
 
 [[package]]
 name = "ctr"
-version = "0.6.0"
+version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fb4a30d54f7443bf3d6191dcd486aca19e67cb3c49fa7a06a319966346707e7f"
+checksum = "049bb91fb4aaf0e3c7efa6cd5ef877dbbbd15b39dad06d9948de4ec8a75761ea"
 dependencies = [
- "cipher 0.2.5",
+ "cipher 0.3.0",
 ]
 
 [[package]]
 name = "ctr"
-version = "0.8.0"
+version = "0.9.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "049bb91fb4aaf0e3c7efa6cd5ef877dbbbd15b39dad06d9948de4ec8a75761ea"
+checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835"
 dependencies = [
- "cipher 0.3.0",
+ "cipher 0.4.3",
 ]
 
 [[package]]
@@ -1464,6 +1398,50 @@ dependencies = [
  "zeroize",
 ]
 
+[[package]]
+name = "cxx"
+version = "1.0.79"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f83d0ebf42c6eafb8d7c52f7e5f2d3003b89c7aa4fd2b79229209459a849af8"
+dependencies = [
+ "cc",
+ "cxxbridge-flags",
+ "cxxbridge-macro",
+ "link-cplusplus",
+]
+
+[[package]]
+name = "cxx-build"
+version = "1.0.79"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "07d050484b55975889284352b0ffc2ecbda25c0c55978017c132b29ba0818a86"
+dependencies = [
+ "cc",
+ "codespan-reporting",
+ "once_cell",
+ "proc-macro2",
+ "quote",
+ "scratch",
+ "syn",
+]
+
+[[package]]
+name = "cxxbridge-flags"
+version = "1.0.79"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "99d2199b00553eda8012dfec8d3b1c75fce747cf27c169a270b3b99e3448ab78"
+
+[[package]]
+name = "cxxbridge-macro"
+version = "1.0.79"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dcb67a6de1f602736dd7eaead0080cf3435df806c61b24b13328db128c58868f"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "darling"
 version = "0.10.2"
@@ -1484,6 +1462,16 @@ dependencies = [
  "darling_macro 0.13.4",
 ]
 
+[[package]]
+name = "darling"
+version = "0.14.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4529658bdda7fd6769b8614be250cdcfc3aeb0ee72fe66f9e41e5e5eb73eac02"
+dependencies = [
+ "darling_core 0.14.1",
+ "darling_macro 0.14.1",
+]
+
 [[package]]
 name = "darling_core"
 version = "0.10.2"
@@ -1512,6 +1500,20 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "darling_core"
+version = "0.14.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "649c91bc01e8b1eac09fb91e8dbc7d517684ca6be8ebc75bb9cafc894f9fdb6f"
+dependencies = [
+ "fnv",
+ "ident_case",
+ "proc-macro2",
+ "quote",
+ "strsim 0.10.0",
+ "syn",
+]
+
 [[package]]
 name = "darling_macro"
 version = "0.10.2"
@@ -1534,6 +1536,17 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "darling_macro"
+version = "0.14.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ddfc69c5bfcbd2fc09a0f38451d2daf0e372e367986a83906d1b0dbc88134fb5"
+dependencies = [
+ "darling_core 0.14.1",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "data-encoding"
 version = "2.3.2"
@@ -1561,47 +1574,31 @@ dependencies = [
 ]
 
 [[package]]
-name = "dbl"
-version = "0.3.2"
+name = "der"
+version = "0.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bd2735a791158376708f9347fe8faba9667589d82427ef3aed6794a8981de3d9"
+checksum = "6919815d73839e7ad218de758883aae3a257ba6759ce7a9992501efbb53d705c"
 dependencies = [
- "generic-array 0.14.6",
+ "const-oid 0.7.1",
+ "crypto-bigint 0.3.2",
+ "pem-rfc7468",
 ]
 
 [[package]]
 name = "der"
-version = "0.3.5"
+version = "0.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2eeb9d92785d1facb50567852ce75d0858630630e7eabea59cf7eb7474051087"
+checksum = "13dd2ae565c0a381dde7fade45fce95984c568bdcb4700a4fdbe3175e0380b2f"
 dependencies = [
- "const-oid 0.5.2",
- "typenum",
+ "const-oid 0.9.0",
+ "zeroize",
 ]
 
 [[package]]
-name = "der"
-version = "0.4.5"
+name = "derive_builder"
+version = "0.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "79b71cca7d95d7681a4b3b9cdf63c8dbc3730d0584c2c74e31416d64a90493f4"
-dependencies = [
- "const-oid 0.6.2",
-]
-
-[[package]]
-name = "der"
-version = "0.5.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6919815d73839e7ad218de758883aae3a257ba6759ce7a9992501efbb53d705c"
-dependencies = [
- "const-oid 0.7.1",
-]
-
-[[package]]
-name = "derive_builder"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a2658621297f2cf68762a6f7dc0bb7e1ff2cfd6583daef8ee0fed6f7ec468ec0"
+checksum = "a2658621297f2cf68762a6f7dc0bb7e1ff2cfd6583daef8ee0fed6f7ec468ec0"
 dependencies = [
  "darling 0.10.2",
  "derive_builder_core",
@@ -1624,12 +1621,12 @@ dependencies = [
 
 [[package]]
 name = "des"
-version = "0.6.0"
+version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b24e7c748888aa2fa8bce21d8c64a52efc810663285315ac7476f7197a982fae"
+checksum = "ac41dd49fb554432020d52c875fc290e110113f864c6b1b525cd62c7e7747a5d"
 dependencies = [
  "byteorder",
- "cipher 0.2.5",
+ "cipher 0.3.0",
  "opaque-debug 0.3.0",
 ]
 
@@ -1669,7 +1666,7 @@ dependencies = [
 [[package]]
 name = "did-ethr"
 version = "0.1.1"
-source = "git+https://github.com/spruceid/ssi?branch=main#d19575d2d4251ddc200d54845a59174971964147"
+source = "git+https://github.com/spruceid/ssi?branch=main#c4a2051c9ed8fdaa503bec39175b13ca24523812"
 dependencies = [
  "async-trait",
  "chrono",
@@ -1685,7 +1682,7 @@ dependencies = [
 [[package]]
 name = "did-ion"
 version = "0.1.0"
-source = "git+https://github.com/spruceid/ssi?branch=main#d19575d2d4251ddc200d54845a59174971964147"
+source = "git+https://github.com/spruceid/ssi?branch=main#c4a2051c9ed8fdaa503bec39175b13ca24523812"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -1707,12 +1704,12 @@ dependencies = [
 [[package]]
 name = "did-method-key"
 version = "0.1.3"
-source = "git+https://github.com/spruceid/ssi?branch=main#d19575d2d4251ddc200d54845a59174971964147"
+source = "git+https://github.com/spruceid/ssi?branch=main#c4a2051c9ed8fdaa503bec39175b13ca24523812"
 dependencies = [
  "async-trait",
  "multibase 0.8.0",
  "serde_json",
- "simple_asn1 0.5.4",
+ "simple_asn1",
  "ssi-crypto",
  "ssi-dids",
  "ssi-jwk",
@@ -1722,7 +1719,7 @@ dependencies = [
 [[package]]
 name = "did-onion"
 version = "0.1.2"
-source = "git+https://github.com/spruceid/ssi?branch=main#d19575d2d4251ddc200d54845a59174971964147"
+source = "git+https://github.com/spruceid/ssi?branch=main#c4a2051c9ed8fdaa503bec39175b13ca24523812"
 dependencies = [
  "async-trait",
  "http",
@@ -1735,7 +1732,7 @@ dependencies = [
 [[package]]
 name = "did-pkh"
 version = "0.1.1"
-source = "git+https://github.com/spruceid/ssi?branch=main#d19575d2d4251ddc200d54845a59174971964147"
+source = "git+https://github.com/spruceid/ssi?branch=main#c4a2051c9ed8fdaa503bec39175b13ca24523812"
 dependencies = [
  "async-trait",
  "bech32",
@@ -1752,7 +1749,7 @@ dependencies = [
 [[package]]
 name = "did-tz"
 version = "0.1.2"
-source = "git+https://github.com/spruceid/ssi?branch=main#d19575d2d4251ddc200d54845a59174971964147"
+source = "git+https://github.com/spruceid/ssi?branch=main#c4a2051c9ed8fdaa503bec39175b13ca24523812"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -1772,7 +1769,7 @@ dependencies = [
 [[package]]
 name = "did-web"
 version = "0.1.2"
-source = "git+https://github.com/spruceid/ssi?branch=main#d19575d2d4251ddc200d54845a59174971964147"
+source = "git+https://github.com/spruceid/ssi?branch=main#c4a2051c9ed8fdaa503bec39175b13ca24523812"
 dependencies = [
  "async-trait",
  "http",
@@ -1784,27 +1781,22 @@ dependencies = [
 [[package]]
 name = "did-webkey"
 version = "0.1.1"
-source = "git+https://github.com/spruceid/ssi?branch=main#d19575d2d4251ddc200d54845a59174971964147"
+source = "git+https://github.com/spruceid/ssi?branch=main#c4a2051c9ed8fdaa503bec39175b13ca24523812"
 dependencies = [
  "anyhow",
  "async-trait",
  "hex",
  "http",
+ "pgp",
  "reqwest",
- "sequoia-openpgp",
  "serde",
  "serde_json",
  "sshkeys",
  "ssi-dids",
+ "ssi-jwk",
  "ssi-ssh",
 ]
 
-[[package]]
-name = "diff"
-version = "0.1.13"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8"
-
 [[package]]
 name = "digest"
 version = "0.8.1"
@@ -1834,73 +1826,21 @@ dependencies = [
  "subtle",
 ]
 
-[[package]]
-name = "dirs-next"
-version = "2.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1"
-dependencies = [
- "cfg-if",
- "dirs-sys-next",
-]
-
-[[package]]
-name = "dirs-sys-next"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d"
-dependencies = [
- "libc",
- "redox_users",
- "winapi",
-]
-
 [[package]]
 name = "dtoa"
-version = "1.0.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c6053ff46b5639ceb91756a85a4c8914668393a03170efd79c8884a529d80656"
-
-[[package]]
-name = "dyn-clone"
-version = "1.0.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4f94fa09c2aeea5b8839e414b7b841bf429fd25b9c522116ac97ee87856d88b2"
-
-[[package]]
-name = "eax"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e1f76e7a5e594b299a0fa9a99de627530725e341df41376aa342aecb2c5eb76e"
-dependencies = [
- "aead 0.3.2",
- "cipher 0.2.5",
- "cmac",
- "ctr 0.6.0",
- "subtle",
-]
-
-[[package]]
-name = "ecdsa"
-version = "0.11.1"
+version = "1.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "34d33b390ab82f2e1481e331dbd0530895640179d2128ef9a79cc690b78d1eba"
-dependencies = [
- "der 0.3.5",
- "elliptic-curve 0.9.12",
- "hmac 0.11.0",
- "signature",
-]
+checksum = "f8a6eee2d5d0d113f015688310da018bd1d864d86bd567c8fca9c266889e1bfa"
 
 [[package]]
 name = "ecdsa"
-version = "0.12.4"
+version = "0.14.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "43ee23aa5b4f68c7a092b5c3beb25f50c406adc75e2363634f242f28ab255372"
+checksum = "413301934810f597c1d19ca71c8710e99a3f1ba28a0d2ebc01551a2daeea3c5c"
 dependencies = [
- "der 0.4.5",
- "elliptic-curve 0.10.6",
- "hmac 0.11.0",
+ "der 0.6.0",
+ "elliptic-curve",
+ "rfc6979",
  "signature",
 ]
 
@@ -1935,60 +1875,24 @@ checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797"
 
 [[package]]
 name = "elliptic-curve"
-version = "0.9.12"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c13e9b0c3c4170dcc2a12783746c4205d98e18957f57854251eea3f9750fe005"
-dependencies = [
- "bitvec 0.20.4",
- "ff 0.9.0",
- "generic-array 0.14.6",
- "group 0.9.0",
- "pkcs8 0.6.1",
- "rand_core 0.6.4",
- "subtle",
- "zeroize",
-]
-
-[[package]]
-name = "elliptic-curve"
-version = "0.10.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "beca177dcb8eb540133e7680baff45e7cc4d93bf22002676cec549f82343721b"
-dependencies = [
- "crypto-bigint 0.2.11",
- "ff 0.10.1",
- "generic-array 0.14.6",
- "group 0.10.0",
- "pkcs8 0.7.6",
- "rand_core 0.6.4",
- "subtle",
- "zeroize",
-]
-
-[[package]]
-name = "elliptic-curve"
-version = "0.11.12"
+version = "0.12.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "25b477563c2bfed38a3b7a60964c49e058b2510ad3f12ba3483fd8f62c2306d6"
+checksum = "e7bb888ab5300a19b8e5bceef25ac745ad065f3c9f7efc6de1b91958110891d3"
 dependencies = [
  "base16ct",
- "crypto-bigint 0.3.2",
- "der 0.5.1",
+ "crypto-bigint 0.4.9",
+ "der 0.6.0",
+ "digest 0.10.5",
+ "ff",
  "generic-array 0.14.6",
+ "group",
+ "pkcs8 0.9.0",
  "rand_core 0.6.4",
+ "sec1",
  "subtle",
  "zeroize",
 ]
 
-[[package]]
-name = "ena"
-version = "0.14.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d7402b94a93c24e742487327a7cd839dc9d36fec9de9fb25b09f2dae459f36c3"
-dependencies = [
- "log",
-]
-
 [[package]]
 name = "encoding_rs"
 version = "0.8.31"
@@ -2000,9 +1904,9 @@ dependencies = [
 
 [[package]]
 name = "enum-as-inner"
-version = "0.4.0"
+version = "0.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "21cdad81446a7f7dc43f6a77409efeb9733d2fa65553efef6018ef257c959b73"
+checksum = "c9720bba047d567ffc8a3cba48bf19126600e249ab7f128e9233e6376976a116"
 dependencies = [
  "heck 0.4.0",
  "proc-macro2",
@@ -2011,70 +1915,24 @@ dependencies = [
 ]
 
 [[package]]
-name = "ethabi"
-version = "15.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f76ef192b63e8a44b3d08832acebbb984c3fba154b5c26f70037c860202a0d4b"
-dependencies = [
- "anyhow",
- "ethereum-types",
- "hex",
- "serde",
- "serde_json",
- "sha3 0.9.1",
- "thiserror",
- "uint",
-]
-
-[[package]]
-name = "ethbloom"
-version = "0.11.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bfb684ac8fa8f6c5759f788862bb22ec6fe3cb392f6bfd08e3c64b603661e3f8"
-dependencies = [
- "crunchy",
- "fixed-hash",
- "impl-rlp",
- "impl-serde",
- "tiny-keccak",
-]
-
-[[package]]
-name = "ethereum-types"
-version = "0.12.1"
+name = "errno"
+version = "0.2.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "05136f7057fe789f06e6d41d07b34e6f70d8c86e5693b60f97aaa6553553bdaf"
+checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1"
 dependencies = [
- "ethbloom",
- "fixed-hash",
- "impl-rlp",
- "impl-serde",
- "primitive-types 0.10.1",
- "uint",
+ "errno-dragonfly",
+ "libc",
+ "winapi",
 ]
 
 [[package]]
-name = "ethers-core"
-version = "0.6.3"
+name = "errno-dragonfly"
+version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f15e1a2a54bc6bc3f8ea94afafbb374264f8322fcacdae06fefda80a206739ac"
+checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf"
 dependencies = [
- "arrayvec 0.7.2",
- "bytes",
- "ecdsa 0.12.4",
- "elliptic-curve 0.11.12",
- "ethabi",
- "generic-array 0.14.6",
- "hex",
- "k256",
- "once_cell",
- "rand 0.8.5",
- "rlp",
- "rlp-derive",
- "serde",
- "serde_json",
- "thiserror",
- "tiny-keccak",
+ "cc",
+ "libc",
 ]
 
 [[package]]
@@ -2100,20 +1958,9 @@ dependencies = [
 
 [[package]]
 name = "ff"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "72a4d941a5b7c2a75222e2d44fcdf634a67133d9db31e177ae5ff6ecda852bfe"
-dependencies = [
- "bitvec 0.20.4",
- "rand_core 0.6.4",
- "subtle",
-]
-
-[[package]]
-name = "ff"
-version = "0.10.1"
+version = "0.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d0f40b2dcd8bc322217a5f6559ae5f9e9d1de202a2ecee2e9eafcbece7562a4f"
+checksum = "df689201f395c6b90dfe87127685f8dbfc083a5e779e613575d8bd7314300c3e"
 dependencies = [
  "rand_core 0.6.4",
  "subtle",
@@ -2121,28 +1968,16 @@ dependencies = [
 
 [[package]]
 name = "figment"
-version = "0.10.7"
+version = "0.10.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6e3bd154d9ae2f1bb0ada5b7eebd56529513efa5de7d2fc8c6adf33bc43260cf"
+checksum = "4e56602b469b2201400dec66a66aec5a9b8761ee97cd1b8c96ab2483fcc16cc9"
 dependencies = [
  "atomic",
  "pear",
  "serde",
  "toml",
  "uncased",
- "version_check",
-]
-
-[[package]]
-name = "filetime"
-version = "0.2.17"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e94a7bbaa59354bc20dd75b67f23e2797b4490e9d6928203fb105c79e448c86c"
-dependencies = [
- "cfg-if",
- "libc",
- "redox_syscall",
- "windows-sys",
+ "version_check 0.9.4",
 ]
 
 [[package]]
@@ -2151,18 +1986,9 @@ version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cfcf0ed7fe52a17a03854ec54a9f76d6d84508d1c0e66bc1793301c73fc8493c"
 dependencies = [
- "byteorder",
- "rand 0.8.5",
- "rustc-hex",
  "static_assertions",
 ]
 
-[[package]]
-name = "fixedbitset"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "37ab347416e802de484e4d03c7316c48f1ecb56574dfd4a46a80f173ce1de04d"
-
 [[package]]
 name = "fixedbitset"
 version = "0.4.2"
@@ -2233,9 +2059,9 @@ checksum = "fed34cd105917e91daa4da6b3728c47b068749d6a62c59811f06ed2ac71d9da7"
 
 [[package]]
 name = "futures"
-version = "0.3.24"
+version = "0.3.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7f21eda599937fba36daeb58a22e8f5cee2d14c4a17b5b7739c7c8e5e3b8230c"
+checksum = "38390104763dc37a5145a53c29c63c1290b5d316d6086ec32c293f6736051bb0"
 dependencies = [
  "futures-channel",
  "futures-core",
@@ -2248,9 +2074,9 @@ dependencies = [
 
 [[package]]
 name = "futures-channel"
-version = "0.3.24"
+version = "0.3.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "30bdd20c28fadd505d0fd6712cdfcb0d4b5648baf45faef7f852afb2399bb050"
+checksum = "52ba265a92256105f45b719605a571ffe2d1f0fea3807304b522c1d778f79eed"
 dependencies = [
  "futures-core",
  "futures-sink",
@@ -2258,15 +2084,15 @@ dependencies = [
 
 [[package]]
 name = "futures-core"
-version = "0.3.24"
+version = "0.3.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4e5aa3de05362c3fb88de6531e6296e85cde7739cccad4b9dfeeb7f6ebce56bf"
+checksum = "04909a7a7e4633ae6c4a9ab280aeb86da1236243a77b694a49eacd659a4bd3ac"
 
 [[package]]
 name = "futures-executor"
-version = "0.3.24"
+version = "0.3.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9ff63c23854bee61b6e9cd331d523909f238fc7636290b96826e9cfa5faa00ab"
+checksum = "7acc85df6714c176ab5edf386123fafe217be88c0840ec11f199441134a074e2"
 dependencies = [
  "futures-core",
  "futures-task",
@@ -2276,9 +2102,9 @@ dependencies = [
 
 [[package]]
 name = "futures-io"
-version = "0.3.24"
+version = "0.3.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bbf4d2a7a308fd4578637c0b17c7e1c7ba127b8f6ba00b29f717e9655d85eb68"
+checksum = "00f5fb52a06bdcadeb54e8d3671f8888a39697dcb0b81b23b55174030427f4eb"
 
 [[package]]
 name = "futures-lite"
@@ -2297,9 +2123,9 @@ dependencies = [
 
 [[package]]
 name = "futures-macro"
-version = "0.3.24"
+version = "0.3.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "42cd15d1c7456c04dbdf7e88bcd69760d74f3a798d6444e16974b505b0e62f17"
+checksum = "bdfb8ce053d86b91919aad980c220b1fb8401a9394410e1c289ed7e66b61835d"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -2308,15 +2134,15 @@ dependencies = [
 
 [[package]]
 name = "futures-sink"
-version = "0.3.24"
+version = "0.3.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "21b20ba5a92e727ba30e72834706623d94ac93a725410b6a6b6fbc1b07f7ba56"
+checksum = "39c15cf1a4aa79df40f1bb462fb39676d0ad9e366c2a33b590d7c66f4f81fcf9"
 
 [[package]]
 name = "futures-task"
-version = "0.3.24"
+version = "0.3.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a6508c467c73851293f390476d4491cf4d227dbabcd4170f3bb6044959b294f1"
+checksum = "2ffb393ac5d9a6eaa9d3fdf37ae2776656b706e200c8e16b1bdb227f5198e6ea"
 
 [[package]]
 name = "futures-timer"
@@ -2326,9 +2152,9 @@ checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c"
 
 [[package]]
 name = "futures-util"
-version = "0.3.24"
+version = "0.3.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "44fb6cb1be61cc1d2e43b262516aafcf63b241cffdb1d3fa115f91d9c7b09c90"
+checksum = "197676987abd2f9cadff84926f410af1c183608d36641465df73ae8211dc65d6"
 dependencies = [
  "futures-channel",
  "futures-core",
@@ -2361,7 +2187,7 @@ dependencies = [
  "libc",
  "log",
  "rustversion",
- "windows",
+ "windows 0.32.0",
 ]
 
 [[package]]
@@ -2380,7 +2206,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9"
 dependencies = [
  "typenum",
- "version_check",
+ "version_check 0.9.4",
 ]
 
 [[package]]
@@ -2390,17 +2216,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce"
 dependencies = [
  "cfg-if",
- "js-sys",
  "libc",
  "wasi 0.9.0+wasi-snapshot-preview1",
- "wasm-bindgen",
 ]
 
 [[package]]
 name = "getrandom"
-version = "0.2.7"
+version = "0.2.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4eb1a864a501629691edf6c15a593b7a51eebaa1e8468e9ddc623de7c9b58ec6"
+checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31"
 dependencies = [
  "cfg-if",
  "js-sys",
@@ -2416,7 +2240,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1583cc1656d7839fd3732b80cf4f38850336cdb9b8ded1cd399ca62958de3c99"
 dependencies = [
  "opaque-debug 0.3.0",
- "polyval",
+ "polyval 0.5.3",
+]
+
+[[package]]
+name = "ghash"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d930750de5717d2dd0b8c0d42c076c0e884c81a73e6cab859bbd2339c71e3e40"
+dependencies = [
+ "opaque-debug 0.3.0",
+ "polyval 0.6.0",
 ]
 
 [[package]]
@@ -2439,22 +2273,11 @@ dependencies = [
 
 [[package]]
 name = "group"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "61b3c1e8b4f1ca07e6605ea1be903a5f6956aec5c8a67fd44d56076631675ed8"
-dependencies = [
- "ff 0.9.0",
- "rand_core 0.6.4",
- "subtle",
-]
-
-[[package]]
-name = "group"
-version = "0.10.0"
+version = "0.12.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1c363a5301b8f153d80747126a04b3c82073b9fe3130571a9d170cacdeaf7912"
+checksum = "5dfbfb3a6cfbd390d5c9564ab283a0349b9b9fcd46a706c1eb10e0db70bfbac7"
 dependencies = [
- "ff 0.10.1",
+ "ff",
  "rand_core 0.6.4",
  "subtle",
 ]
@@ -2478,12 +2301,6 @@ dependencies = [
  "tracing",
 ]
 
-[[package]]
-name = "hash_hasher"
-version = "2.0.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "74721d007512d0cb3338cd20f0654ac913920061a4c4d0d8708edb3f2a698c0c"
-
 [[package]]
 name = "hashbrown"
 version = "0.11.2"
@@ -2505,7 +2322,7 @@ version = "0.3.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f3e372db8e5c0d213e0cd0b9be18be2aca3d44cf2fe30a9d46a65581cd454584"
 dependencies = [
- "base64 0.13.0",
+ "base64 0.13.1",
  "bitflags",
  "bytes",
  "headers-core",
@@ -2560,17 +2377,7 @@ version = "0.12.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "791a029f6b9fc27657f6f188ec6e5e43f6911f6f878e0dc5501396e09809d437"
 dependencies = [
- "hmac 0.12.1",
-]
-
-[[package]]
-name = "hmac"
-version = "0.11.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2a2a2320eb7ec0ebe8da8f744d7812d9fc4cb4d09344ac01898dbcb6a20ae69b"
-dependencies = [
- "crypto-mac 0.11.1",
- "digest 0.9.0",
+ "hmac",
 ]
 
 [[package]]
@@ -2683,26 +2490,26 @@ dependencies = [
 
 [[package]]
 name = "iana-time-zone"
-version = "0.1.48"
+version = "0.1.51"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "237a0714f28b1ee39ccec0770ccb544eb02c9ef2c82bb096230eefcffa6468b0"
+checksum = "f5a6ef98976b22b3b7f2f3a806f858cb862044cfa66805aa3ad84cb3d3b785ed"
 dependencies = [
  "android_system_properties",
  "core-foundation-sys",
+ "iana-time-zone-haiku",
  "js-sys",
- "once_cell",
  "wasm-bindgen",
  "winapi",
 ]
 
 [[package]]
-name = "idea"
-version = "0.3.0"
+name = "iana-time-zone-haiku"
+version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fcdd4b114cf2265123bbdc5d32a39f96a343fbdf141267d2b5232b7e14caacb3"
+checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca"
 dependencies = [
- "cipher 0.2.5",
- "opaque-debug 0.3.0",
+ "cxx",
+ "cxx-build",
 ]
 
 [[package]]
@@ -2743,41 +2550,21 @@ dependencies = [
 ]
 
 [[package]]
-name = "impl-codec"
-version = "0.5.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "161ebdfec3c8e3b52bf61c4f3550a1eea4f9579d10dc1b936f3171ebdcd6c443"
-dependencies = [
- "parity-scale-codec",
-]
-
-[[package]]
-name = "impl-rlp"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f28220f89297a075ddc7245cd538076ee98b01f2a9c23a53a4f1105d5a322808"
-dependencies = [
- "rlp",
-]
-
-[[package]]
-name = "impl-serde"
-version = "0.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4551f042f3438e64dbd6226b20527fc84a6e1fe65688b58746a2f53623f25f5c"
-dependencies = [
- "serde",
-]
-
-[[package]]
-name = "impl-trait-for-tuples"
-version = "0.2.2"
+name = "if-watch"
+version = "2.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "11d7a9f6330b71fea57921c9b61c47ee6e84f72d394754eff6163ae67e7395eb"
+checksum = "065c008e570a43c00de6aed9714035e5ea6a498c255323db9091722af6ee67dd"
 dependencies = [
- "proc-macro2",
- "quote",
- "syn",
+ "async-io",
+ "core-foundation",
+ "fnv",
+ "futures",
+ "if-addrs",
+ "ipnet",
+ "log",
+ "rtnetlink",
+ "system-configuration",
+ "windows 0.34.0",
 ]
 
 [[package]]
@@ -2786,7 +2573,7 @@ version = "1.9.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "10a35a97730320ffe8e2d410b5d3b69279b98d2c14bdb8b70ea89ecf7888d41e"
 dependencies = [
- "autocfg 1.1.0",
+ "autocfg",
  "hashbrown 0.12.3",
  "serde",
 ]
@@ -2797,6 +2584,15 @@ version = "0.1.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c8fae54786f62fb2918dcfae3d568594e50eb9b5c25bf04371af6fe7516452fb"
 
+[[package]]
+name = "inout"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5"
+dependencies = [
+ "generic-array 0.14.6",
+]
+
 [[package]]
 name = "instant"
 version = "0.1.12"
@@ -2812,6 +2608,12 @@ version = "3.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02"
 
+[[package]]
+name = "io-lifetimes"
+version = "0.7.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1ea37f355c05dde75b84bba2d767906ad522e97cd9e2eef2be7a4ab7fb442c06"
+
 [[package]]
 name = "ipconfig"
 version = "0.3.0"
@@ -2824,71 +2626,6 @@ dependencies = [
  "winreg 0.7.0",
 ]
 
-[[package]]
-name = "ipfs"
-version = "0.2.1"
-source = "git+https://github.com/spruceid/rust-ipfs?branch=feat/update-libipld#f4883c2ca38ce0f8f9ab4ee7f7d6443433013548"
-dependencies = [
- "anyhow",
- "async-stream",
- "async-trait",
- "base64 0.13.0",
- "byteorder",
- "bytes",
- "either",
- "fs2",
- "futures",
- "hash_hasher",
- "ipfs-bitswap",
- "ipfs-unixfs",
- "libipld",
- "libp2p",
- "once_cell",
- "serde",
- "serde_json",
- "sled",
- "thiserror",
- "tokio",
- "tokio-stream",
- "tokio-util 0.6.10",
- "tracing",
- "tracing-futures",
- "trust-dns-resolver",
- "void",
-]
-
-[[package]]
-name = "ipfs-bitswap"
-version = "0.1.0"
-source = "git+https://github.com/spruceid/rust-ipfs?branch=feat/update-libipld#f4883c2ca38ce0f8f9ab4ee7f7d6443433013548"
-dependencies = [
- "fnv",
- "futures",
- "hash_hasher",
- "libipld",
- "libp2p-core",
- "libp2p-swarm",
- "multihash 0.11.4",
- "prost 0.9.0",
- "prost-build 0.8.0",
- "thiserror",
- "tokio",
- "tracing",
- "unsigned-varint 0.7.1",
-]
-
-[[package]]
-name = "ipfs-unixfs"
-version = "0.2.0"
-source = "git+https://github.com/spruceid/rust-ipfs?branch=feat/update-libipld#f4883c2ca38ce0f8f9ab4ee7f7d6443433013548"
-dependencies = [
- "either",
- "filetime",
- "libipld",
- "quick-protobuf",
- "sha2 0.9.9",
-]
-
 [[package]]
 name = "ipnet"
 version = "2.5.0"
@@ -2912,7 +2649,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8f0f7638c1e223529f1bfdc48c8b133b9e0b434094d1d28473161ee48b235f78"
 dependencies = [
  "nom 7.1.1",
- "serde",
 ]
 
 [[package]]
@@ -2921,20 +2657,30 @@ version = "0.5.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bf071934ee7ee97e52fa1868a9540a7885eab75926bd70794030304a9797cea1"
 
+[[package]]
+name = "iri-string"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6d0586ad318a04c73acdbad33f67969519b5452c80770c4c72059a686da48a7e"
+dependencies = [
+ "memchr",
+ "serde",
+]
+
 [[package]]
 name = "itertools"
-version = "0.10.4"
+version = "0.10.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d8bf247779e67a9082a4790b45e71ac7cfd1321331a5c856a74a9faebdab78d0"
+checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473"
 dependencies = [
  "either",
 ]
 
 [[package]]
 name = "itoa"
-version = "1.0.3"
+version = "1.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6c8af84674fe1f223a982c933a0ee1086ac4d4052aa0fb8060c12c6ad838e754"
+checksum = "4217ad341ebadf8d8e724e264f13e593e0648f5b3e94b3896a5df283be015ecc"
 
 [[package]]
 name = "js-sys"
@@ -2979,15 +2725,15 @@ dependencies = [
 
 [[package]]
 name = "k256"
-version = "0.9.6"
+version = "0.11.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "903ae2481bcdfdb7b68e0a9baa4b7c9aff600b9ae2e8e5bb5833b8c91ab851ea"
+checksum = "72c1e0b51e7ec0a97369623508396067a486bd0cbed95a2659a4b863d28cfc8b"
 dependencies = [
  "cfg-if",
- "ecdsa 0.12.4",
- "elliptic-curve 0.10.6",
- "sha2 0.9.9",
- "sha3 0.9.1",
+ "ecdsa",
+ "elliptic-curve",
+ "sha2 0.10.6",
+ "sha3 0.10.6",
 ]
 
 [[package]]
@@ -3002,7 +2748,7 @@ version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ae0386ec98c26dd721aaa3412bf3a817156ff3ee7cb6959503f8d1095f4ccc51"
 dependencies = [
- "primitive-types 0.9.1",
+ "primitive-types",
  "tiny-keccak",
 ]
 
@@ -3017,21 +2763,18 @@ dependencies = [
  "aws-sdk-s3",
  "aws-smithy-http",
  "aws-types",
- "base64 0.13.0",
+ "base64 0.13.1",
  "bincode",
  "bs58",
  "cached 0.34.1",
- "ethers-core",
  "futures",
  "hex",
  "hyper",
- "ipfs",
  "iri-string 0.5.6",
  "kepler-lib",
  "lazy_static",
- "libipld",
+ "libipld 0.14.0",
  "libp2p",
- "nom 6.1.2",
  "opentelemetry",
  "opentelemetry-jaeger",
  "percent-encoding",
@@ -3041,11 +2784,11 @@ dependencies = [
  "rocket",
  "serde",
  "serde_json",
- "serde_with",
+ "serde_with 1.14.0",
  "sled",
  "tempdir",
  "thiserror",
- "time 0.3.14",
+ "time 0.3.15",
  "tokio",
  "tokio-stream",
  "tokio-util 0.6.10",
@@ -3063,7 +2806,7 @@ name = "kepler-lib"
 version = "0.1.0"
 dependencies = [
  "async-trait",
- "base64 0.13.0",
+ "base64 0.13.1",
  "cacaos",
  "capgrok",
  "did-ethr",
@@ -3074,12 +2817,12 @@ dependencies = [
  "did-tz",
  "did-web",
  "did-webkey",
- "iri-string 0.5.6",
+ "iri-string 0.6.0",
  "lazy_static",
- "libipld",
+ "libipld 0.14.0",
  "serde",
  "serde_json",
- "serde_with",
+ "serde_with 1.14.0",
  "ssi",
  "thiserror",
  "tokio",
@@ -3090,7 +2833,7 @@ dependencies = [
 name = "kepler-sdk"
 version = "0.1.0"
 dependencies = [
- "base64 0.13.0",
+ "base64 0.13.1",
  "chrono",
  "hex",
  "http",
@@ -3098,7 +2841,7 @@ dependencies = [
  "kepler-lib",
  "serde",
  "serde_json",
- "serde_with",
+ "serde_with 1.14.0",
  "thiserror",
  "tokio",
  "tracing",
@@ -3112,7 +2855,7 @@ dependencies = [
  "kepler-sdk",
  "serde",
  "serde_json",
- "serde_with",
+ "serde_with 1.14.0",
  "wasm-bindgen",
  "wasm-bindgen-futures",
 ]
@@ -3126,34 +2869,6 @@ dependencies = [
  "log",
 ]
 
-[[package]]
-name = "lalrpop"
-version = "0.19.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b30455341b0e18f276fa64540aff54deafb54c589de6aca68659c63dd2d5d823"
-dependencies = [
- "ascii-canvas",
- "atty",
- "bit-set",
- "diff",
- "ena",
- "itertools",
- "lalrpop-util",
- "petgraph 0.6.2",
- "regex",
- "regex-syntax",
- "string_cache",
- "term",
- "tiny-keccak",
- "unicode-xid",
-]
-
-[[package]]
-name = "lalrpop-util"
-version = "0.19.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bcf796c978e9b4d983414f4caedc9273aa33ee214c5b887bd55fde84c85d2dc4"
-
 [[package]]
 name = "langtag"
 version = "0.2.0"
@@ -3169,24 +2884,11 @@ dependencies = [
  "spin 0.5.2",
 ]
 
-[[package]]
-name = "lexical-core"
-version = "0.7.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6607c62aa161d23d17a9072cc5da0be67cdfc89d3afb1e8d9c842bebc2525ffe"
-dependencies = [
- "arrayvec 0.5.2",
- "bitflags",
- "cfg-if",
- "ryu",
- "static_assertions",
-]
-
 [[package]]
 name = "libc"
-version = "0.2.132"
+version = "0.2.135"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8371e4e5341c3a96db127eb2465ac681ced4c433e01dd0e938adbef26ba93ba5"
+checksum = "68783febc7782c6c5cb401fbda4de5a9898be1762314da0bb2c10ced61f18b0c"
 
 [[package]]
 name = "libipld"
@@ -3197,14 +2899,35 @@ dependencies = [
  "async-trait",
  "cached 0.30.0",
  "fnv",
- "libipld-cbor",
- "libipld-cbor-derive",
- "libipld-core",
- "libipld-json",
- "libipld-macro",
- "libipld-pb",
+ "libipld-cbor 0.13.1",
+ "libipld-cbor-derive 0.13.1",
+ "libipld-core 0.13.1",
+ "libipld-json 0.13.1",
+ "libipld-macro 0.13.1",
+ "libipld-pb 0.13.1",
+ "log",
+ "multihash",
+ "parking_lot 0.12.1",
+ "thiserror",
+]
+
+[[package]]
+name = "libipld"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac9c3aa309c260aa2f174bac968901eddc546e9d85950c28eae6a7bec402f926"
+dependencies = [
+ "async-trait",
+ "cached 0.30.0",
+ "fnv",
+ "libipld-cbor 0.14.0",
+ "libipld-cbor-derive 0.14.0",
+ "libipld-core 0.14.0",
+ "libipld-json 0.14.0",
+ "libipld-macro 0.14.0",
+ "libipld-pb 0.14.0",
  "log",
- "multihash 0.16.3",
+ "multihash",
  "parking_lot 0.12.1",
  "thiserror",
 ]
@@ -3216,7 +2939,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "46b3bbc4f35b8f25eb140d183f33d4610a1eb6531b312f01161fa06693f8c28a"
 dependencies = [
  "byteorder",
- "libipld-core",
+ "libipld-core 0.13.1",
+ "thiserror",
+]
+
+[[package]]
+name = "libipld-cbor"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8dd1ab68c9d26f20c7d0dfea6eecbae8c00359875210001b33ca27d4a02f3d09"
+dependencies = [
+ "byteorder",
+ "libipld-core 0.14.0",
  "thiserror",
 ]
 
@@ -3233,6 +2967,19 @@ dependencies = [
  "synstructure",
 ]
 
+[[package]]
+name = "libipld-cbor-derive"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "69ec2f49393a1347a2d95ebcb248ff75d0d47235919b678036c010a8cd927375"
+dependencies = [
+ "proc-macro-crate",
+ "proc-macro2",
+ "quote",
+ "syn",
+ "synstructure",
+]
+
 [[package]]
 name = "libipld-core"
 version = "0.13.1"
@@ -3243,19 +2990,45 @@ dependencies = [
  "cid",
  "core2",
  "multibase 0.9.1",
- "multihash 0.16.3",
+ "multihash",
  "serde",
  "thiserror",
 ]
 
+[[package]]
+name = "libipld-core"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d44790246ec6b7314cba745992c23d479d018073e66d49ae40ae1b64e5dd8eb5"
+dependencies = [
+ "anyhow",
+ "cid",
+ "core2",
+ "multibase 0.9.1",
+ "multihash",
+ "thiserror",
+]
+
 [[package]]
 name = "libipld-json"
 version = "0.13.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "415427568962961e21009eb92fd6052de95174423cff9d3c583491858e060f4e"
 dependencies = [
- "libipld-core",
- "multihash 0.16.3",
+ "libipld-core 0.13.1",
+ "multihash",
+ "serde",
+ "serde_json",
+]
+
+[[package]]
+name = "libipld-json"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "18aa481a87f084d98473dd9ece253a9569c762b75f6bbba8217d54e48c9d63b3"
+dependencies = [
+ "libipld-core 0.14.0",
+ "multihash",
  "serde",
  "serde_json",
 ]
@@ -3266,7 +3039,16 @@ version = "0.13.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c6c4cb1056262ef4056ad9e5fb41f252c45f55009888e21b7837ac051f38814a"
 dependencies = [
- "libipld-core",
+ "libipld-core 0.13.1",
+]
+
+[[package]]
+name = "libipld-macro"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "852c011562ae5059b67c3a917f9f5945af5a68df8e39ede4444fff33274d25e2"
+dependencies = [
+ "libipld-core 0.14.0",
 ]
 
 [[package]]
@@ -3275,12 +3057,24 @@ version = "0.13.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6b851a08aeff23e1e8df42081ef22052a838e8e39b769fcb7867d2fadd834d2d"
 dependencies = [
- "libipld-core",
+ "libipld-core 0.13.1",
  "prost 0.9.0",
  "prost-build 0.9.0",
  "thiserror",
 ]
 
+[[package]]
+name = "libipld-pb"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c003be513496578115256a1b4ac7b80d4ece2462c9869dfb736fd30d8bb1d1c0"
+dependencies = [
+ "libipld-core 0.14.0",
+ "prost 0.10.4",
+ "prost-build 0.10.4",
+ "thiserror",
+]
+
 [[package]]
 name = "libm"
 version = "0.2.5"
@@ -3289,15 +3083,14 @@ checksum = "292a948cd991e376cf75541fe5b97a1081d713c618b4f1b9500f8844e49eb565"
 
 [[package]]
 name = "libp2p"
-version = "0.43.0"
+version = "0.49.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4e8570e25fa03d4385405dbeaf540ba00e3ee50942f03d84e1a8928a029f35f9"
+checksum = "ec878fda12ebec479186b3914ebc48ff180fa4c51847e11a1a68bf65249e02c1"
 dependencies = [
- "atomic",
  "bytes",
  "futures",
  "futures-timer",
- "getrandom 0.2.7",
+ "getrandom 0.2.8",
  "instant",
  "lazy_static",
  "libp2p-core",
@@ -3316,16 +3109,15 @@ dependencies = [
  "libp2p-yamux",
  "multiaddr",
  "parking_lot 0.12.1",
- "pin-project 1.0.12",
- "rand 0.7.3",
+ "pin-project",
  "smallvec",
 ]
 
 [[package]]
 name = "libp2p-core"
-version = "0.32.1"
+version = "0.37.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "db5b02602099fb75cb2d16f9ea860a320d6eb82ce41e95ab680912c454805cd5"
+checksum = "799676bb0807c788065e57551c6527d461ad572162b0519d1958946ff9e0539d"
 dependencies = [
  "asn1_der",
  "bs58",
@@ -3338,41 +3130,41 @@ dependencies = [
  "lazy_static",
  "log",
  "multiaddr",
- "multihash 0.16.3",
+ "multihash",
  "multistream-select",
  "parking_lot 0.12.1",
- "pin-project 1.0.12",
- "prost 0.9.0",
- "prost-build 0.9.0",
+ "pin-project",
+ "prost 0.11.0",
+ "prost-build 0.11.1",
  "rand 0.8.5",
- "ring",
  "rw-stream-sink",
  "sha2 0.10.6",
  "smallvec",
  "thiserror",
- "unsigned-varint 0.7.1",
+ "unsigned-varint",
  "void",
  "zeroize",
 ]
 
 [[package]]
 name = "libp2p-dns"
-version = "0.32.1"
+version = "0.37.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "066e33e854e10b5c93fc650458bf2179c7e0d143db260b0963e44a94859817f1"
+checksum = "2322c9fb40d99101def6a01612ee30500c89abbbecb6297b3cd252903a4c1720"
 dependencies = [
  "futures",
  "libp2p-core",
  "log",
+ "parking_lot 0.12.1",
  "smallvec",
  "trust-dns-resolver",
 ]
 
 [[package]]
 name = "libp2p-floodsub"
-version = "0.34.0"
+version = "0.40.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "df0b7d6c3fa2ead77a5bbeff580bd7507efcc9d7fa9d0caf873795b097d385c0"
+checksum = "e88a413aac86cd30078192e5c110fd4c859ba36a6d694e040e75f1ef0dc41583"
 dependencies = [
  "cuckoofilter",
  "fnv",
@@ -3380,36 +3172,40 @@ dependencies = [
  "libp2p-core",
  "libp2p-swarm",
  "log",
- "prost 0.9.0",
- "prost-build 0.9.0",
- "rand 0.7.3",
+ "prost 0.11.0",
+ "prost-build 0.11.1",
+ "rand 0.8.5",
  "smallvec",
 ]
 
 [[package]]
 name = "libp2p-identify"
-version = "0.34.0"
+version = "0.40.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1f219b4d4660fe3a04bf5fe6b5970902b7c1918e25b2536be8c70efc480f88f8"
+checksum = "dcf9a121f699e8719bda2e6e9e9b6ddafc6cff4602471d6481c1067930ccb29b"
 dependencies = [
+ "asynchronous-codec",
  "futures",
  "futures-timer",
  "libp2p-core",
  "libp2p-swarm",
  "log",
  "lru",
- "prost 0.9.0",
- "prost-build 0.9.0",
+ "prost 0.11.0",
+ "prost-build 0.11.1",
+ "prost-codec",
  "smallvec",
+ "thiserror",
+ "void",
 ]
 
 [[package]]
 name = "libp2p-kad"
-version = "0.35.0"
+version = "0.41.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aead5ee2322a7b825c7633065370909c8383046f955cda5b56797e6904db7a72"
+checksum = "6721c200e2021f6c3fab8b6cf0272ead8912d871610ee194ebd628cecf428f22"
 dependencies = [
- "arrayvec 0.5.2",
+ "arrayvec 0.7.2",
  "asynchronous-codec",
  "bytes",
  "either",
@@ -3420,22 +3216,22 @@ dependencies = [
  "libp2p-core",
  "libp2p-swarm",
  "log",
- "prost 0.9.0",
- "prost-build 0.9.0",
- "rand 0.7.3",
+ "prost 0.11.0",
+ "prost-build 0.11.1",
+ "rand 0.8.5",
  "sha2 0.10.6",
  "smallvec",
  "thiserror",
  "uint",
- "unsigned-varint 0.7.1",
+ "unsigned-varint",
  "void",
 ]
 
 [[package]]
 name = "libp2p-metrics"
-version = "0.4.0"
+version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d29e4e5e4c5aa567fe1ee3133afe088dc2d2fd104e20c5c2c5c2649f75129677"
+checksum = "9ee31b08e78b7b8bfd1c4204a9dd8a87b4fcdf6dafc57eb51701c1c264a81cb9"
 dependencies = [
  "libp2p-core",
  "libp2p-identify",
@@ -3448,9 +3244,9 @@ dependencies = [
 
 [[package]]
 name = "libp2p-mplex"
-version = "0.32.0"
+version = "0.37.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "442eb0c9fff0bf22a34f015724b4143ce01877e079ed0963c722d94c07c72160"
+checksum = "692664acfd98652de739a8acbb0a0d670f1d67190a49be6b4395e22c37337d89"
 dependencies = [
  "asynchronous-codec",
  "bytes",
@@ -3459,16 +3255,16 @@ dependencies = [
  "log",
  "nohash-hasher",
  "parking_lot 0.12.1",
- "rand 0.7.3",
+ "rand 0.8.5",
  "smallvec",
- "unsigned-varint 0.7.1",
+ "unsigned-varint",
 ]
 
 [[package]]
 name = "libp2p-noise"
-version = "0.35.0"
+version = "0.40.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9dd7e0c94051cda67123be68cf6b65211ba3dde7277be9068412de3e7ffd63ef"
+checksum = "048155686bd81fe6cb5efdef0c6290f25ad32a0a42e8f4f72625cf6a505a206f"
 dependencies = [
  "bytes",
  "curve25519-dalek 3.2.0",
@@ -3476,8 +3272,8 @@ dependencies = [
  "lazy_static",
  "libp2p-core",
  "log",
- "prost 0.9.0",
- "prost-build 0.9.0",
+ "prost 0.11.0",
+ "prost-build 0.11.1",
  "rand 0.8.5",
  "sha2 0.10.6",
  "snow",
@@ -3488,9 +3284,9 @@ dependencies = [
 
 [[package]]
 name = "libp2p-ping"
-version = "0.34.0"
+version = "0.40.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9ab44a12d372d6abdd326c468c1d5b002be06fbd923c5a799d6a9d3b36646ca3"
+checksum = "7228b9318d34689521349a86eb39a3c3a802c9efc99a0568062ffb80913e3f91"
 dependencies = [
  "futures",
  "futures-timer",
@@ -3498,15 +3294,15 @@ dependencies = [
  "libp2p-core",
  "libp2p-swarm",
  "log",
- "rand 0.7.3",
+ "rand 0.8.5",
  "void",
 ]
 
 [[package]]
 name = "libp2p-relay"
-version = "0.7.0"
+version = "0.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "517be90a2ce60b6c3bdfe88f34cc789c61dafe6f694a7b45e644af7353880fa3"
+checksum = "3266c322fa094e229eb63c2644b0209a7e1db1d78ab6f03956eb3534bb514155"
 dependencies = [
  "asynchronous-codec",
  "bytes",
@@ -3517,22 +3313,22 @@ dependencies = [
  "libp2p-core",
  "libp2p-swarm",
  "log",
- "pin-project 1.0.12",
- "prost 0.9.0",
- "prost-build 0.9.0",
+ "pin-project",
+ "prost 0.11.0",
+ "prost-build 0.11.1",
+ "prost-codec",
  "rand 0.8.5",
  "smallvec",
  "static_assertions",
  "thiserror",
- "unsigned-varint 0.7.1",
  "void",
 ]
 
 [[package]]
 name = "libp2p-swarm"
-version = "0.34.0"
+version = "0.40.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "53ab2d4eb8ef2966b10fdf859245cdd231026df76d3c6ed2cf9e418a8f688ec9"
+checksum = "46d13df7c37807965d82930c0e4b04a659efcb6cca237373b206043db5398ecf"
 dependencies = [
  "either",
  "fnv",
@@ -3541,8 +3337,8 @@ dependencies = [
  "instant",
  "libp2p-core",
  "log",
- "pin-project 1.0.12",
- "rand 0.7.3",
+ "pin-project",
+ "rand 0.8.5",
  "smallvec",
  "thiserror",
  "void",
@@ -3550,24 +3346,24 @@ dependencies = [
 
 [[package]]
 name = "libp2p-swarm-derive"
-version = "0.27.2"
+version = "0.30.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4f693c8c68213034d472cbb93a379c63f4f307d97c06f1c41e4985de481687a5"
+checksum = "a0eddc4497a8b5a506013c40e8189864f9c3a00db2b25671f428ae9007f3ba32"
 dependencies = [
+ "heck 0.4.0",
  "quote",
  "syn",
 ]
 
 [[package]]
 name = "libp2p-tcp"
-version = "0.32.0"
+version = "0.37.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "193447aa729c85aac2376828df76d171c1a589c9e6b58fcc7f9d9a020734122c"
+checksum = "9839d96761491c6d3e238e70554b856956fca0ab60feb9de2cd08eed4473fa92"
 dependencies = [
  "futures",
  "futures-timer",
- "if-addrs",
- "ipnet",
+ "if-watch",
  "libc",
  "libp2p-core",
  "log",
@@ -3577,30 +3373,46 @@ dependencies = [
 
 [[package]]
 name = "libp2p-yamux"
-version = "0.36.0"
+version = "0.41.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "be902ebd89193cd020e89e89107726a38cfc0d16d18f613f4a37d046e92c7517"
+checksum = "30f079097a21ad017fc8139460630286f02488c8c13b26affb46623aa20d8845"
 dependencies = [
  "futures",
  "libp2p-core",
+ "log",
  "parking_lot 0.12.1",
  "thiserror",
  "yamux",
 ]
 
+[[package]]
+name = "link-cplusplus"
+version = "1.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9272ab7b96c9046fbc5bc56c06c117cb639fe2d509df0c421cad82d2915cf369"
+dependencies = [
+ "cc",
+]
+
 [[package]]
 name = "linked-hash-map"
 version = "0.5.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f"
 
+[[package]]
+name = "linux-raw-sys"
+version = "0.0.46"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d4d2456c373231a208ad294c33dc5bff30051eafd954cd4caae83a712b12854d"
+
 [[package]]
 name = "lock_api"
-version = "0.4.8"
+version = "0.4.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9f80bf5aacaf25cbfc8210d1cfb718f2bf3b11c4c54e5afe36c236853a8ec390"
+checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df"
 dependencies = [
- "autocfg 1.1.0",
+ "autocfg",
  "scopeguard",
 ]
 
@@ -3631,9 +3443,9 @@ dependencies = [
 
 [[package]]
 name = "lru"
-version = "0.7.8"
+version = "0.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e999beba7b6e8345721bd280141ed958096a2e4abdf74f67ff4ce49b4b54e47a"
+checksum = "b6e8aaa3f231bb4bd57b84b2d5dc3ae7f350265df8aa96492e0bc394a1571909"
 dependencies = [
  "hashbrown 0.12.3",
 ]
@@ -3697,15 +3509,9 @@ version = "0.6.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce"
 dependencies = [
- "autocfg 1.1.0",
+ "autocfg",
 ]
 
-[[package]]
-name = "memsec"
-version = "0.6.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9ac78937f19a0c7807e45a931eac41f766f210173ec664ec046d58e6d388a5cb"
-
 [[package]]
 name = "mime"
 version = "0.3.16"
@@ -3736,7 +3542,7 @@ dependencies = [
  "libc",
  "log",
  "wasi 0.11.0+wasi-snapshot-preview1",
- "windows-sys",
+ "windows-sys 0.36.1",
 ]
 
 [[package]]
@@ -3747,9 +3553,9 @@ checksum = "8c1dd912d3c7c6c6fd557d1eadfc20ef57bb0a4de572cb0e875b171571c355e4"
 
 [[package]]
 name = "multer"
-version = "2.0.3"
+version = "2.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a30ba6d97eb198c5e8a35d67d5779d6680cca35652a60ee90fc23dc431d4fde8"
+checksum = "6ed4198ce7a4cbd2a57af78d28c6fbb57d81ac5f1d6ad79ac6c5587419cbdf22"
 dependencies = [
  "bytes",
  "encoding_rs",
@@ -3762,7 +3568,7 @@ dependencies = [
  "spin 0.9.4",
  "tokio",
  "tokio-util 0.7.4",
- "version_check",
+ "version_check 0.9.4",
 ]
 
 [[package]]
@@ -3775,11 +3581,11 @@ dependencies = [
  "bs58",
  "byteorder",
  "data-encoding",
- "multihash 0.16.3",
+ "multihash",
  "percent-encoding",
  "serde",
  "static_assertions",
- "unsigned-varint 0.7.1",
+ "unsigned-varint",
  "url",
 ]
 
@@ -3805,21 +3611,6 @@ dependencies = [
  "data-encoding-macro",
 ]
 
-[[package]]
-name = "multihash"
-version = "0.11.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "567122ab6492f49b59def14ecc36e13e64dca4188196dd0cd41f9f3f979f3df6"
-dependencies = [
- "blake2b_simd 0.5.11",
- "blake2s_simd 0.5.11",
- "digest 0.9.0",
- "sha-1",
- "sha2 0.9.9",
- "sha3 0.9.1",
- "unsigned-varint 0.5.1",
-]
-
 [[package]]
 name = "multihash"
 version = "0.16.3"
@@ -3827,7 +3618,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1c346cf9999c631f002d8f977c4eaeaa0e6386f16007202308d0b3757522c2cc"
 dependencies = [
  "blake2b_simd 1.0.0",
- "blake2s_simd 1.0.0",
+ "blake2s_simd",
  "blake3",
  "core2",
  "digest 0.10.5",
@@ -3835,8 +3626,8 @@ dependencies = [
  "serde",
  "serde-big-array",
  "sha2 0.10.6",
- "sha3 0.10.5",
- "unsigned-varint 0.7.1",
+ "sha3 0.10.6",
+ "unsigned-varint",
 ]
 
 [[package]]
@@ -3861,16 +3652,16 @@ checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a"
 
 [[package]]
 name = "multistream-select"
-version = "0.11.0"
+version = "0.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "363a84be6453a70e63513660f4894ef815daf88e3356bffcda9ca27d810ce83b"
+checksum = "9bc41247ec209813e2fd414d6e16b9d94297dacf3cd613fa6ef09cd4d9755c10"
 dependencies = [
  "bytes",
  "futures",
  "log",
- "pin-project 1.0.12",
+ "pin-project",
  "smallvec",
- "unsigned-varint 0.7.1",
+ "unsigned-varint",
 ]
 
 [[package]]
@@ -3892,49 +3683,116 @@ dependencies = [
 ]
 
 [[package]]
-name = "new_debug_unreachable"
-version = "1.0.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e4a24736216ec316047a1fc4252e27dabb04218aa4a3f37c6e7ddbf1f9782b54"
-
-[[package]]
-name = "nohash-hasher"
-version = "0.2.0"
+name = "netlink-packet-core"
+version = "0.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451"
+checksum = "345b8ab5bd4e71a2986663e88c56856699d060e78e152e6e9d7966fcd5491297"
+dependencies = [
+ "anyhow",
+ "byteorder",
+ "libc",
+ "netlink-packet-utils",
+]
 
 [[package]]
-name = "nom"
-version = "6.1.2"
+name = "netlink-packet-route"
+version = "0.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e7413f999671bd4745a7b624bd370a569fb6bc574b23c83a3c5ed2e453f3d5e2"
+checksum = "d9ea4302b9759a7a88242299225ea3688e63c85ea136371bb6cf94fd674efaab"
 dependencies = [
- "bitvec 0.19.6",
- "funty",
- "lexical-core",
- "memchr",
- "version_check",
+ "anyhow",
+ "bitflags",
+ "byteorder",
+ "libc",
+ "netlink-packet-core",
+ "netlink-packet-utils",
 ]
 
 [[package]]
-name = "nom"
-version = "7.1.1"
+name = "netlink-packet-utils"
+version = "0.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a8903e5a29a317527874d0402f867152a3d21c908bb0b933e416c65e301d4c36"
+checksum = "25af9cf0dc55498b7bd94a1508af7a78706aa0ab715a73c5169273e03c84845e"
 dependencies = [
- "memchr",
- "minimal-lexical",
+ "anyhow",
+ "byteorder",
+ "paste",
+ "thiserror",
 ]
 
 [[package]]
-name = "num-bigint"
-version = "0.2.6"
+name = "netlink-proto"
+version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304"
+checksum = "65b4b14489ab424703c092062176d52ba55485a89c076b4f9db05092b7223aa6"
 dependencies = [
- "autocfg 1.1.0",
- "num-integer",
- "num-traits",
+ "bytes",
+ "futures",
+ "log",
+ "netlink-packet-core",
+ "netlink-sys",
+ "thiserror",
+ "tokio",
+]
+
+[[package]]
+name = "netlink-sys"
+version = "0.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "92b654097027250401127914afb37cb1f311df6610a9891ff07a757e94199027"
+dependencies = [
+ "async-io",
+ "bytes",
+ "futures",
+ "libc",
+ "log",
+]
+
+[[package]]
+name = "nix"
+version = "0.24.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "195cdbc1741b8134346d515b3a56a1c94b0912758009cfd53f99ea0f57b065fc"
+dependencies = [
+ "bitflags",
+ "cfg-if",
+ "libc",
+]
+
+[[package]]
+name = "nohash-hasher"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451"
+
+[[package]]
+name = "nom"
+version = "4.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2ad2a91a8e869eeb30b9cb3119ae87773a8f4ae617f41b1eb9c154b2905f7bd6"
+dependencies = [
+ "memchr",
+ "version_check 0.1.5",
+]
+
+[[package]]
+name = "nom"
+version = "7.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a8903e5a29a317527874d0402f867152a3d21c908bb0b933e416c65e301d4c36"
+dependencies = [
+ "memchr",
+ "minimal-lexical",
+]
+
+[[package]]
+name = "nu-ansi-term"
+version = "0.46.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84"
+dependencies = [
+ "overload",
+ "winapi",
 ]
 
 [[package]]
@@ -3943,37 +3801,47 @@ version = "0.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f93ab6289c7b344a8a9f60f88d80aa20032336fe78da341afc91c8a2341fc75f"
 dependencies = [
- "autocfg 1.1.0",
+ "autocfg",
  "num-integer",
  "num-traits",
 ]
 
 [[package]]
 name = "num-bigint-dig"
-version = "0.6.1"
+version = "0.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5d51546d704f52ef14b3c962b5776e53d5b862e5790e40a350d366c209bd7f7a"
+checksum = "566d173b2f9406afbc5510a90925d5a2cd80cae4605631f1212303df265de011"
 dependencies = [
- "autocfg 0.1.8",
  "byteorder",
  "lazy_static",
  "libm",
  "num-integer",
  "num-iter",
  "num-traits",
- "rand 0.7.3",
+ "rand 0.8.5",
  "serde",
  "smallvec",
  "zeroize",
 ]
 
+[[package]]
+name = "num-derive"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "876a53fff98e03a936a674b29568b0e605f06b29372c2489ff4de23f1949743d"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "num-integer"
 version = "0.1.45"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9"
 dependencies = [
- "autocfg 1.1.0",
+ "autocfg",
  "num-traits",
 ]
 
@@ -3983,7 +3851,7 @@ version = "0.1.43"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252"
 dependencies = [
- "autocfg 1.1.0",
+ "autocfg",
  "num-integer",
  "num-traits",
 ]
@@ -3994,7 +3862,8 @@ version = "0.2.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd"
 dependencies = [
- "autocfg 1.1.0",
+ "autocfg",
+ "libm",
 ]
 
 [[package]]
@@ -4018,9 +3887,9 @@ dependencies = [
 
 [[package]]
 name = "once_cell"
-version = "1.14.0"
+version = "1.15.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2f7254b99e31cad77da24b08ebf628882739a608578bb1bcdfc1f9c21260d7c0"
+checksum = "e82dad04139b71a90c080c8463fe0dc7902db5192d939bd0950f074d014339e1"
 
 [[package]]
 name = "opaque-debug"
@@ -4036,9 +3905,9 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
 
 [[package]]
 name = "openssl"
-version = "0.10.41"
+version = "0.10.42"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "618febf65336490dfcf20b73f885f5651a0c89c64c2d4a8c3662585a70bf5bd0"
+checksum = "12fc0523e3bd51a692c8850d075d74dc062ccf251c0110668cbd921917118a13"
 dependencies = [
  "bitflags",
  "cfg-if",
@@ -4077,11 +3946,11 @@ dependencies = [
 
 [[package]]
 name = "openssl-sys"
-version = "0.9.75"
+version = "0.9.76"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e5f9bd0c2710541a3cda73d6f9ac4f1b240de4ae261065d309dbe73d9dceb42f"
+checksum = "5230151e44c0f05157effb743e8d517472843121cf9243e8b81393edb5acd9ce"
 dependencies = [
- "autocfg 1.1.0",
+ "autocfg",
  "cc",
  "libc",
  "openssl-src",
@@ -4103,7 +3972,7 @@ dependencies = [
  "js-sys",
  "lazy_static",
  "percent-encoding",
- "pin-project 1.0.12",
+ "pin-project",
  "rand 0.8.5",
  "thiserror",
  "tokio",
@@ -4161,60 +4030,20 @@ dependencies = [
 ]
 
 [[package]]
-name = "owning_ref"
-version = "0.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6ff55baddef9e4ad00f88b6c743a2a8062d4c6ade126c2a528644b8e444d52ce"
-dependencies = [
- "stable_deref_trait",
-]
-
-[[package]]
-name = "p256"
-version = "0.8.1"
+name = "overload"
+version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2f05f5287453297c4c16af5e2b04df8fd2a3008d70f252729650bc6d7ace5844"
-dependencies = [
- "ecdsa 0.11.1",
- "elliptic-curve 0.9.12",
- "sha2 0.9.9",
-]
+checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39"
 
 [[package]]
 name = "p256"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d053368e1bae4c8a672953397bd1bd7183dde1c72b0b7612a15719173148d186"
-dependencies = [
- "ecdsa 0.12.4",
- "elliptic-curve 0.10.6",
- "sha2 0.9.9",
-]
-
-[[package]]
-name = "parity-scale-codec"
-version = "2.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "373b1a4c1338d9cd3d1fa53b3a11bdab5ab6bd80a20f7f7becd76953ae2be909"
-dependencies = [
- "arrayvec 0.7.2",
- "bitvec 0.20.4",
- "byte-slice-cast",
- "impl-trait-for-tuples",
- "parity-scale-codec-derive",
- "serde",
-]
-
-[[package]]
-name = "parity-scale-codec-derive"
-version = "2.3.1"
+version = "0.11.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1557010476e0595c9b568d16dcfb81b93cdeb157612726f5170d31aa707bed27"
+checksum = "51f44edd08f51e2ade572f141051021c5af22677e42b7dd28a88155151c33594"
 dependencies = [
- "proc-macro-crate",
- "proc-macro2",
- "quote",
- "syn",
+ "ecdsa",
+ "elliptic-curve",
+ "sha2 0.10.6",
 ]
 
 [[package]]
@@ -4241,7 +4070,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f"
 dependencies = [
  "lock_api",
- "parking_lot_core 0.9.3",
+ "parking_lot_core 0.9.4",
 ]
 
 [[package]]
@@ -4260,17 +4089,23 @@ dependencies = [
 
 [[package]]
 name = "parking_lot_core"
-version = "0.9.3"
+version = "0.9.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929"
+checksum = "4dc9e0dc2adc1c69d09143aff38d3d30c5c3f0df0dad82e6d25547af174ebec0"
 dependencies = [
  "cfg-if",
  "libc",
  "redox_syscall",
  "smallvec",
- "windows-sys",
+ "windows-sys 0.42.0",
 ]
 
+[[package]]
+name = "paste"
+version = "1.0.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b1de2e551fb905ac83f73f7aedf2f0cb4a0da7e35efa24a202a936269f1f18e1"
+
 [[package]]
 name = "pct-str"
 version = "1.1.0"
@@ -4301,14 +4136,12 @@ dependencies = [
 ]
 
 [[package]]
-name = "pem"
-version = "0.8.3"
+name = "pem-rfc7468"
+version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fd56cbd21fea48d0c440b41cd69c589faacade08c992d9a54e471b79d0fd13eb"
+checksum = "01de5d978f34aa4b2296576379fcc416034702fd94117c56ffd8a1a767cefb30"
 dependencies = [
- "base64 0.13.0",
- "once_cell",
- "regex",
+ "base64ct",
 ]
 
 [[package]]
@@ -4317,86 +4150,63 @@ version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e"
 
-[[package]]
-name = "pest"
-version = "2.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cb779fcf4bb850fbbb0edc96ff6cf34fd90c4b1a112ce042653280d9a7364048"
-dependencies = [
- "thiserror",
- "ucd-trie",
-]
-
-[[package]]
-name = "pest_derive"
-version = "2.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "502b62a6d0245378b04ffe0a7fb4f4419a4815fce813bd8a0ec89a56e07d67b1"
-dependencies = [
- "pest",
- "pest_generator",
-]
-
-[[package]]
-name = "pest_generator"
-version = "2.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "451e629bf49b750254da26132f1a5a9d11fd8a95a3df51d15c4abd1ba154cb6c"
-dependencies = [
- "pest",
- "pest_meta",
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "pest_meta"
-version = "2.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bcec162c71c45e269dfc3fc2916eaeb97feab22993a21bcce4721d08cd7801a6"
-dependencies = [
- "once_cell",
- "pest",
- "sha1",
-]
-
-[[package]]
-name = "petgraph"
-version = "0.5.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "467d164a6de56270bd7c4d070df81d07beace25012d5103ced4e9ff08d6afdb7"
-dependencies = [
- "fixedbitset 0.2.0",
- "indexmap",
-]
-
 [[package]]
 name = "petgraph"
 version = "0.6.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e6d5014253a1331579ce62aa67443b4a658c5e7dd03d4bc6d302b94474888143"
 dependencies = [
- "fixedbitset 0.4.2",
+ "fixedbitset",
  "indexmap",
 ]
 
 [[package]]
-name = "phf_shared"
-version = "0.10.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096"
-dependencies = [
- "siphasher",
-]
-
-[[package]]
-name = "pin-project"
-version = "0.4.30"
+name = "pgp"
+version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3ef0f924a5ee7ea9cbcea77529dba45f8a9ba9f622419fe3386ca581a3ae9d5a"
+checksum = "c0c63db779c3f090b540dfa0484f8adc2d380e3aa60cdb0f3a7a97454e22edc5"
 dependencies = [
- "pin-project-internal 0.4.30",
+ "aes 0.7.5",
+ "base64 0.13.1",
+ "bitfield",
+ "block-modes",
+ "block-padding 0.2.1",
+ "blowfish",
+ "buf_redux",
+ "byteorder",
+ "cast5",
+ "cfb-mode",
+ "chrono",
+ "cipher 0.3.0",
+ "circular",
+ "clear_on_drop",
+ "crc24",
+ "derive_builder",
+ "des",
+ "digest 0.9.0",
+ "ed25519-dalek",
+ "flate2",
+ "generic-array 0.14.6",
+ "hex",
+ "lazy_static",
+ "log",
+ "md-5",
+ "nom 4.2.3",
+ "num-bigint-dig",
+ "num-derive",
+ "num-traits",
+ "rand 0.8.5",
+ "ripemd160",
+ "rsa",
+ "sha-1",
+ "sha2 0.9.9",
+ "sha3 0.9.1",
+ "signature",
+ "smallvec",
+ "thiserror",
+ "twofish",
+ "x25519-dalek",
+ "zeroize",
 ]
 
 [[package]]
@@ -4405,18 +4215,7 @@ version = "1.0.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc"
 dependencies = [
- "pin-project-internal 1.0.12",
-]
-
-[[package]]
-name = "pin-project-internal"
-version = "0.4.30"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "851c8d0ce9bebe43790dedfc86614c23494ac9f423dd618d3a61fc693eafe61e"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
+ "pin-project-internal",
 ]
 
 [[package]]
@@ -4442,24 +4241,36 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
 
+[[package]]
+name = "pkcs1"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a78f66c04ccc83dd4486fd46c33896f4e17b24a7a3a6400dedc48ed0ddd72320"
+dependencies = [
+ "der 0.5.1",
+ "pkcs8 0.8.0",
+ "zeroize",
+]
+
 [[package]]
 name = "pkcs8"
-version = "0.6.1"
+version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c9c2f795bc591cb3384cb64082a578b89207ac92bb89c9d98c1ea2ace7cd8110"
+checksum = "7cabda3fb821068a9a4fab19a683eac3af12edf0f34b94a8be53c4972b8149d0"
 dependencies = [
- "der 0.3.5",
- "spki 0.3.0",
+ "der 0.5.1",
+ "spki 0.5.4",
+ "zeroize",
 ]
 
 [[package]]
 name = "pkcs8"
-version = "0.7.6"
+version = "0.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ee3ef9b64d26bad0536099c816c6734379e45bbd5f14798def6809e5cc350447"
+checksum = "9eca2c590a5f85da82668fa685c09ce2888b9430e83299debf1f34b65fd4a4ba"
 dependencies = [
- "der 0.4.5",
- "spki 0.4.1",
+ "der 0.6.0",
+ "spki 0.6.0",
 ]
 
 [[package]]
@@ -4474,7 +4285,7 @@ version = "2.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "899b00b9c8ab553c743b3e11e87c5c7d423b2a2de229ba95b24a756344748011"
 dependencies = [
- "autocfg 1.1.0",
+ "autocfg",
  "cfg-if",
  "libc",
  "log",
@@ -4490,7 +4301,7 @@ checksum = "048aeb476be11a4b6ca432ca569e375810de9294ae78f4774e78ea98a9246ede"
 dependencies = [
  "cpufeatures",
  "opaque-debug 0.3.0",
- "universal-hash",
+ "universal-hash 0.4.1",
 ]
 
 [[package]]
@@ -4502,20 +4313,26 @@ dependencies = [
  "cfg-if",
  "cpufeatures",
  "opaque-debug 0.3.0",
- "universal-hash",
+ "universal-hash 0.4.1",
 ]
 
 [[package]]
-name = "ppv-lite86"
-version = "0.2.16"
+name = "polyval"
+version = "0.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872"
+checksum = "7ef234e08c11dfcb2e56f79fd70f6f2eb7f025c0ce2333e82f4f0518ecad30c6"
+dependencies = [
+ "cfg-if",
+ "cpufeatures",
+ "opaque-debug 0.3.0",
+ "universal-hash 0.5.0",
+]
 
 [[package]]
-name = "precomputed-hash"
-version = "0.1.1"
+name = "ppv-lite86"
+version = "0.2.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c"
+checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872"
 
 [[package]]
 name = "primitive-types"
@@ -4527,19 +4344,6 @@ dependencies = [
  "uint",
 ]
 
-[[package]]
-name = "primitive-types"
-version = "0.10.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "05e4722c697a58a99d5d06a08c30821d7c082a4632198de1eaa5a6c22ef42373"
-dependencies = [
- "fixed-hash",
- "impl-codec",
- "impl-rlp",
- "impl-serde",
- "uint",
-]
-
 [[package]]
 name = "proc-macro-crate"
 version = "1.2.1"
@@ -4561,7 +4365,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "syn",
- "version_check",
+ "version_check 0.9.4",
 ]
 
 [[package]]
@@ -4572,14 +4376,14 @@ checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869"
 dependencies = [
  "proc-macro2",
  "quote",
- "version_check",
+ "version_check 0.9.4",
 ]
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.43"
+version = "1.0.47"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0a2ca2c61bc9f3d74d2886294ab7b9853abd9c1ad903a3ac7815c58989bb7bab"
+checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725"
 dependencies = [
  "unicode-ident",
 ]
@@ -4593,28 +4397,28 @@ dependencies = [
  "proc-macro2",
  "quote",
  "syn",
- "version_check",
+ "version_check 0.9.4",
  "yansi",
 ]
 
 [[package]]
 name = "procfs"
-version = "0.12.0"
+version = "0.14.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0941606b9934e2d98a3677759a971756eb821f75764d0e0d26946d08e74d9104"
+checksum = "2dfb6451c91904606a1abe93e83a8ec851f45827fa84273f256ade45dc095818"
 dependencies = [
  "bitflags",
  "byteorder",
  "hex",
  "lazy_static",
- "libc",
+ "rustix",
 ]
 
 [[package]]
 name = "prometheus"
-version = "0.13.2"
+version = "0.13.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "45c8babc29389186697fe5a2a4859d697825496b83db5d0b65271cdc0488e88c"
+checksum = "449811d15fbdf5ceb5c1144416066429cf82316e2ec8ce0c1f6f8a02e7bbcf8c"
 dependencies = [
  "cfg-if",
  "fnv",
@@ -4629,21 +4433,21 @@ dependencies = [
 
 [[package]]
 name = "prometheus-client"
-version = "0.15.1"
+version = "0.18.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c9a896938cc6018c64f279888b8c7559d3725210d5db9a3a1ee6bc7188d51d34"
+checksum = "83cd1b99916654a69008fd66b4f9397fbe08e6e51dfe23d4417acf5d3b8cb87c"
 dependencies = [
  "dtoa",
  "itoa",
- "owning_ref",
+ "parking_lot 0.12.1",
  "prometheus-client-derive-text-encode",
 ]
 
 [[package]]
 name = "prometheus-client-derive-text-encode"
-version = "0.2.0"
+version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e8e12d01b9d66ad9eb4529c57666b6263fc1993cb30261d83ead658fdd932652"
+checksum = "66a455fbcb954c1a7decf3c586e860fd7889cddf4b8e164be736dbac95a953cd"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -4652,67 +4456,114 @@ dependencies = [
 
 [[package]]
 name = "prost"
-version = "0.8.0"
+version = "0.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "de5e2533f59d08fcf364fd374ebda0692a70bd6d7e66ef97f306f45c6c5d8020"
+checksum = "444879275cb4fd84958b1a1d5420d15e6fcf7c235fe47f053c9c2a80aceb6001"
 dependencies = [
  "bytes",
- "prost-derive 0.8.0",
+ "prost-derive 0.9.0",
 ]
 
 [[package]]
 name = "prost"
-version = "0.9.0"
+version = "0.10.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "444879275cb4fd84958b1a1d5420d15e6fcf7c235fe47f053c9c2a80aceb6001"
+checksum = "71adf41db68aa0daaefc69bb30bcd68ded9b9abaad5d1fbb6304c4fb390e083e"
 dependencies = [
  "bytes",
- "prost-derive 0.9.0",
+ "prost-derive 0.10.1",
+]
+
+[[package]]
+name = "prost"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "399c3c31cdec40583bb68f0b18403400d01ec4289c383aa047560439952c4dd7"
+dependencies = [
+ "bytes",
+ "prost-derive 0.11.0",
 ]
 
 [[package]]
 name = "prost-build"
-version = "0.8.0"
+version = "0.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "355f634b43cdd80724ee7848f95770e7e70eefa6dcf14fea676216573b8fd603"
+checksum = "62941722fb675d463659e49c4f3fe1fe792ff24fe5bbaa9c08cd3b98a1c354f5"
 dependencies = [
  "bytes",
  "heck 0.3.3",
  "itertools",
+ "lazy_static",
  "log",
  "multimap",
- "petgraph 0.5.1",
- "prost 0.8.0",
- "prost-types 0.8.0",
+ "petgraph",
+ "prost 0.9.0",
+ "prost-types 0.9.0",
+ "regex",
  "tempfile",
  "which",
 ]
 
 [[package]]
 name = "prost-build"
-version = "0.9.0"
+version = "0.10.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62941722fb675d463659e49c4f3fe1fe792ff24fe5bbaa9c08cd3b98a1c354f5"
+checksum = "8ae5a4388762d5815a9fc0dea33c56b021cdc8dde0c55e0c9ca57197254b0cab"
 dependencies = [
  "bytes",
- "heck 0.3.3",
+ "cfg-if",
+ "cmake",
+ "heck 0.4.0",
  "itertools",
  "lazy_static",
  "log",
  "multimap",
- "petgraph 0.6.2",
- "prost 0.9.0",
- "prost-types 0.9.0",
+ "petgraph",
+ "prost 0.10.4",
+ "prost-types 0.10.1",
+ "regex",
+ "tempfile",
+ "which",
+]
+
+[[package]]
+name = "prost-build"
+version = "0.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f835c582e6bd972ba8347313300219fed5bfa52caf175298d860b61ff6069bb"
+dependencies = [
+ "bytes",
+ "heck 0.4.0",
+ "itertools",
+ "lazy_static",
+ "log",
+ "multimap",
+ "petgraph",
+ "prost 0.11.0",
+ "prost-types 0.11.1",
  "regex",
  "tempfile",
  "which",
 ]
 
+[[package]]
+name = "prost-codec"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "011ae9ff8359df7915f97302d591cdd9e0e27fbd5a4ddc5bd13b71079bb20987"
+dependencies = [
+ "asynchronous-codec",
+ "bytes",
+ "prost 0.11.0",
+ "thiserror",
+ "unsigned-varint",
+]
+
 [[package]]
 name = "prost-derive"
-version = "0.8.0"
+version = "0.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "600d2f334aa05acb02a755e217ef1ab6dea4d51b58b7846588b747edec04efba"
+checksum = "f9cc1a3263e07e0bf68e96268f37665207b49560d98739662cdfaae215c720fe"
 dependencies = [
  "anyhow",
  "itertools",
@@ -4723,9 +4574,9 @@ dependencies = [
 
 [[package]]
 name = "prost-derive"
-version = "0.9.0"
+version = "0.10.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f9cc1a3263e07e0bf68e96268f37665207b49560d98739662cdfaae215c720fe"
+checksum = "7b670f45da57fb8542ebdbb6105a925fe571b67f9e7ed9f47a06a84e72b4e7cc"
 dependencies = [
  "anyhow",
  "itertools",
@@ -4735,13 +4586,16 @@ dependencies = [
 ]
 
 [[package]]
-name = "prost-types"
-version = "0.8.0"
+name = "prost-derive"
+version = "0.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "603bbd6394701d13f3f25aada59c7de9d35a6a5887cfc156181234a44002771b"
+checksum = "7345d5f0e08c0536d7ac7229952590239e77abf0a0100a1b1d890add6ea96364"
 dependencies = [
- "bytes",
- "prost 0.8.0",
+ "anyhow",
+ "itertools",
+ "proc-macro2",
+ "quote",
+ "syn",
 ]
 
 [[package]]
@@ -4754,11 +4608,31 @@ dependencies = [
  "prost 0.9.0",
 ]
 
+[[package]]
+name = "prost-types"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2d0a014229361011dc8e69c8a1ec6c2e8d0f2af7c91e3ea3f5b2170298461e68"
+dependencies = [
+ "bytes",
+ "prost 0.10.4",
+]
+
+[[package]]
+name = "prost-types"
+version = "0.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4dfaa718ad76a44b3415e6c4d53b17c8f99160dcb3a99b10470fce8ad43f6e3e"
+dependencies = [
+ "bytes",
+ "prost 0.11.0",
+]
+
 [[package]]
 name = "protobuf"
-version = "2.27.1"
+version = "2.28.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cf7e6d18738ecd0902d30d1ad232c9125985a3422929b16c65517b38adc14f96"
+checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94"
 
 [[package]]
 name = "quick-error"
@@ -4766,15 +4640,6 @@ version = "1.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0"
 
-[[package]]
-name = "quick-protobuf"
-version = "0.8.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6ca6639207ac869e31cca06b8adbc7676278f22b321e51115766009b4f192dbb"
-dependencies = [
- "byteorder",
-]
-
 [[package]]
 name = "quote"
 version = "1.0.21"
@@ -4784,12 +4649,6 @@ dependencies = [
  "proc-macro2",
 ]
 
-[[package]]
-name = "radium"
-version = "0.5.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "941ba9d78d8e2f7ce474c015eea4d9c6d25b6a3327f9832ee29a4de27f91bbb8"
-
 [[package]]
 name = "radium"
 version = "0.6.2"
@@ -4883,7 +4742,7 @@ version = "0.6.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
 dependencies = [
- "getrandom 0.2.7",
+ "getrandom 0.2.8",
 ]
 
 [[package]]
@@ -4913,31 +4772,20 @@ dependencies = [
  "bitflags",
 ]
 
-[[package]]
-name = "redox_users"
-version = "0.4.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b"
-dependencies = [
- "getrandom 0.2.7",
- "redox_syscall",
- "thiserror",
-]
-
 [[package]]
 name = "ref-cast"
-version = "1.0.9"
+version = "1.0.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ed13bcd201494ab44900a96490291651d200730904221832b9547d24a87d332b"
+checksum = "12a733f1746c929b4913fe48f8697fcf9c55e3304ba251a79ffb41adfeaf49c2"
 dependencies = [
  "ref-cast-impl",
 ]
 
 [[package]]
 name = "ref-cast-impl"
-version = "1.0.9"
+version = "1.0.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5234cd6063258a5e32903b53b1b6ac043a0541c8adc1f610f67b0326c7a578fa"
+checksum = "5887de4a01acafd221861463be6113e6e87275e79804e56779f4cdc131c60368"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -4981,11 +4829,11 @@ dependencies = [
 
 [[package]]
 name = "reqwest"
-version = "0.11.11"
+version = "0.11.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b75aa69a3f06bbcc66ede33af2af253c6f7a86b1ca0033f60c580a27074fbf92"
+checksum = "431949c384f4e2ae07605ccaa56d1d9d2ecdb5cadd4f9577ccfab29f2e5149fc"
 dependencies = [
- "base64 0.13.0",
+ "base64 0.13.1",
  "bytes",
  "encoding_rs",
  "futures-core",
@@ -4997,10 +4845,10 @@ dependencies = [
  "hyper-tls",
  "ipnet",
  "js-sys",
- "lazy_static",
  "log",
  "mime",
  "native-tls",
+ "once_cell",
  "percent-encoding",
  "pin-project-lite",
  "serde",
@@ -5027,6 +4875,17 @@ dependencies = [
  "quick-error",
 ]
 
+[[package]]
+name = "rfc6979"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "88c86280f057430a52f4861551b092a01b419b8eacefc7c995eacb9dc132fe32"
+dependencies = [
+ "crypto-bigint 0.4.9",
+ "hmac",
+ "zeroize",
+]
+
 [[package]]
 name = "ring"
 version = "0.16.20"
@@ -5053,27 +4912,6 @@ dependencies = [
  "opaque-debug 0.3.0",
 ]
 
-[[package]]
-name = "rlp"
-version = "0.5.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "999508abb0ae792aabed2460c45b89106d97fe4adac593bdaef433c2605847b5"
-dependencies = [
- "bytes",
- "rustc-hex",
-]
-
-[[package]]
-name = "rlp-derive"
-version = "0.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e33d7b2abe0c340d8797fe2907d3f20d3b5ea5908683618bfe80df7f621f672a"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
 [[package]]
 name = "rocket"
 version = "0.5.0-rc.2"
@@ -5104,12 +4942,12 @@ dependencies = [
  "serde_json",
  "state",
  "tempfile",
- "time 0.3.14",
+ "time 0.3.15",
  "tokio",
  "tokio-stream",
  "tokio-util 0.7.4",
  "ubyte",
- "version_check",
+ "version_check 0.9.4",
  "yansi",
 ]
 
@@ -5151,38 +4989,45 @@ dependencies = [
  "smallvec",
  "stable-pattern",
  "state",
- "time 0.3.14",
+ "time 0.3.15",
  "tokio",
  "uncased",
 ]
 
 [[package]]
 name = "rsa"
-version = "0.3.0"
+version = "0.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3648b669b10afeab18972c105e284a7b953a669b0be3514c27f9b17acab2f9cd"
+checksum = "4cf22754c49613d2b3b119f0e5d46e34a2c628a937e3024b8762de4e7d8c710b"
 dependencies = [
  "byteorder",
- "digest 0.9.0",
- "lazy_static",
+ "digest 0.10.5",
  "num-bigint-dig",
  "num-integer",
  "num-iter",
  "num-traits",
- "pem",
- "rand 0.7.3",
- "sha2 0.9.9",
- "simple_asn1 0.4.1",
+ "pkcs1",
+ "pkcs8 0.8.0",
+ "rand_core 0.6.4",
+ "smallvec",
  "subtle",
- "thiserror",
  "zeroize",
 ]
 
 [[package]]
-name = "rustc-hex"
-version = "2.1.0"
+name = "rtnetlink"
+version = "0.10.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3e75f6a532d0fd9f7f13144f392b6ad56a32696bfcd9c78f797f16bbb6f072d6"
+checksum = "322c53fd76a18698f1c27381d58091de3a043d356aa5bd0d510608b565f469a0"
+dependencies = [
+ "async-global-executor",
+ "futures",
+ "log",
+ "netlink-packet-route",
+ "netlink-proto",
+ "nix",
+ "thiserror",
+]
 
 [[package]]
 name = "rustc_version"
@@ -5193,13 +5038,27 @@ dependencies = [
  "semver",
 ]
 
+[[package]]
+name = "rustix"
+version = "0.35.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fbb2fda4666def1433b1b05431ab402e42a1084285477222b72d6c564c417cef"
+dependencies = [
+ "bitflags",
+ "errno",
+ "io-lifetimes",
+ "libc",
+ "linux-raw-sys",
+ "windows-sys 0.36.1",
+]
+
 [[package]]
 name = "rustls"
 version = "0.19.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "35edb675feee39aec9c99fa5ff985081995a06d594114ae14cbe797ad7b7a6d7"
 dependencies = [
- "base64 0.13.0",
+ "base64 0.13.1",
  "log",
  "ring",
  "sct",
@@ -5226,12 +5085,12 @@ checksum = "97477e48b4cf8603ad5f7aaf897467cf42ab4218a38ef76fb14c2d6773a6d6a8"
 
 [[package]]
 name = "rw-stream-sink"
-version = "0.2.1"
+version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4da5fcb054c46f5a5dff833b129285a93d3f0179531735e6c866e8cc307d2020"
+checksum = "26338f5e09bb721b85b135ea05af7767c90b52f6de4f087d4f4a3a9d64e7dc04"
 dependencies = [
  "futures",
- "pin-project 0.4.30",
+ "pin-project",
  "static_assertions",
 ]
 
@@ -5247,6 +5106,12 @@ version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6518fc26bced4d53678a22d6e423e9d8716377def84545fe328236e3af070e7f"
 
+[[package]]
+name = "safemem"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ef703b7cb59335eae2eb93ceb664c0eb7ea6bf567079d843e09420219668e072"
+
 [[package]]
 name = "schannel"
 version = "0.1.20"
@@ -5254,7 +5119,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "88d6731146462ea25d9244b2ed5fd1d716d25c52e4d54aa4fb0f3c4e9854dbe2"
 dependencies = [
  "lazy_static",
- "windows-sys",
+ "windows-sys 0.36.1",
 ]
 
 [[package]]
@@ -5269,6 +5134,12 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
 
+[[package]]
+name = "scratch"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9c8132065adcfd6e02db789d9285a0deb2f3fcb04002865ab67d5fb103533898"
+
 [[package]]
 name = "sct"
 version = "0.6.1"
@@ -5279,6 +5150,20 @@ dependencies = [
  "untrusted",
 ]
 
+[[package]]
+name = "sec1"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3be24c1842290c45df0a7bf069e0c268a747ad05a192f2fd7dcfdbc1cba40928"
+dependencies = [
+ "base16ct",
+ "der 0.6.0",
+ "generic-array 0.14.6",
+ "pkcs8 0.9.0",
+ "subtle",
+ "zeroize",
+]
+
 [[package]]
 name = "security-framework"
 version = "2.7.0"
@@ -5304,66 +5189,15 @@ dependencies = [
 
 [[package]]
 name = "semver"
-version = "1.0.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e25dfac463d778e353db5be2449d1cce89bd6fd23c9f1ea21310ce6e5a1b29c4"
-
-[[package]]
-name = "sequoia-openpgp"
-version = "1.10.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4ee32fced98917f2c03d571658934aadae9b1527133ae9c7ac3cadb9d8252a05"
-dependencies = [
- "aes 0.6.0",
- "anyhow",
- "base64 0.13.0",
- "block-modes",
- "block-padding 0.2.1",
- "blowfish",
- "buffered-reader",
- "cast5",
- "chrono",
- "cipher 0.2.5",
- "des",
- "digest 0.9.0",
- "dyn-clone",
- "eax",
- "ecdsa 0.11.1",
- "ed25519-dalek",
- "flate2",
- "generic-array 0.14.6",
- "getrandom 0.2.7",
- "idea",
- "idna 0.2.3",
- "lalrpop",
- "lalrpop-util",
- "lazy_static",
- "libc",
- "md-5",
- "memsec",
- "num-bigint-dig",
- "p256 0.8.1",
- "rand 0.7.3",
- "rand_core 0.6.4",
- "regex",
- "regex-syntax",
- "ripemd160",
- "rsa",
- "sha-1",
- "sha1collisiondetection",
- "sha2 0.9.9",
- "thiserror",
- "twofish",
- "typenum",
- "x25519-dalek",
- "xxhash-rust",
-]
+version = "1.0.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e25dfac463d778e353db5be2449d1cce89bd6fd23c9f1ea21310ce6e5a1b29c4"
 
 [[package]]
 name = "serde"
-version = "1.0.144"
+version = "1.0.146"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0f747710de3dcd43b88c9168773254e809d8ddbdf9653b84e2554ab219f17860"
+checksum = "6df50b7a60a0ad48e1b42eb38373eac8ff785d619fb14db917b4e63d5439361f"
 dependencies = [
  "serde_derive",
 ]
@@ -5388,9 +5222,9 @@ dependencies = [
 
 [[package]]
 name = "serde_derive"
-version = "1.0.144"
+version = "1.0.146"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "94ed3a816fb1d101812f83e789f888322c34e291f894f19590dc310963e87a00"
+checksum = "a714fd32ba1d66047ce7d53dabd809e9922d538f9047de13cc4cffca47b36205"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -5410,9 +5244,9 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.85"
+version = "1.0.87"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e55a28e3aaef9d5ce0506d0a14dbba8054ddc7e499ef522dd8b26859ec9d4a44"
+checksum = "6ce777b7b150d76b9cf60d28b55f5847135a003f7d7350c6be7a773508ce7d45"
 dependencies = [
  "itoa",
  "ryu",
@@ -5437,10 +5271,26 @@ version = "1.14.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "678b5a069e50bf00ecd22d0cd8ddf7c236f68581b03db652061ed5eb13a312ff"
 dependencies = [
- "base64 0.13.0",
+ "base64 0.13.1",
+ "hex",
+ "serde",
+ "serde_with_macros 1.5.2",
+]
+
+[[package]]
+name = "serde_with"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "368f2d60d049ea019a84dcd6687b0d1e0030fe663ae105039bdf967ed5e6a9a7"
+dependencies = [
+ "base64 0.13.1",
+ "chrono",
  "hex",
+ "indexmap",
  "serde",
- "serde_with_macros",
+ "serde_json",
+ "serde_with_macros 2.0.1",
+ "time 0.3.15",
 ]
 
 [[package]]
@@ -5455,6 +5305,18 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "serde_with_macros"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1ccadfacf6cf10faad22bbadf55986bdd0856edfb5d9210aa1dcf1f516e84e93"
+dependencies = [
+ "darling 0.14.1",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "sha-1"
 version = "0.9.8"
@@ -5479,16 +5341,6 @@ dependencies = [
  "digest 0.10.5",
 ]
 
-[[package]]
-name = "sha1collisiondetection"
-version = "0.2.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c66558a774ef5044cb4a834db5f5c7f95e139d2341d7f502fe6034afa7082461"
-dependencies = [
- "digest 0.9.0",
- "generic-array 0.14.6",
-]
-
 [[package]]
 name = "sha2"
 version = "0.8.2"
@@ -5539,9 +5391,9 @@ dependencies = [
 
 [[package]]
 name = "sha3"
-version = "0.10.5"
+version = "0.10.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e2904bea16a1ae962b483322a1c7b81d976029203aea1f461e51cd7705db7ba9"
+checksum = "bdf0c33fae925bdc080598b84bc15c55e7b9a4a43b3c704da051f977469691c9"
 dependencies = [
  "digest 0.10.5",
  "keccak",
@@ -5567,25 +5419,14 @@ dependencies = [
 
 [[package]]
 name = "signature"
-version = "1.3.2"
+version = "1.6.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f2807892cfa58e081aa1f1111391c7a0649d4fa127a4ffbe34bcbfb35a1171a4"
+checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c"
 dependencies = [
- "digest 0.9.0",
+ "digest 0.10.5",
  "rand_core 0.6.4",
 ]
 
-[[package]]
-name = "simple_asn1"
-version = "0.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "692ca13de57ce0613a363c8c2f1de925adebc81b04c923ac60c5488bb44abe4b"
-dependencies = [
- "chrono",
- "num-bigint 0.2.6",
- "num-traits",
-]
-
 [[package]]
 name = "simple_asn1"
 version = "0.5.4"
@@ -5593,31 +5434,55 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8eb4ea60fb301dc81dfc113df680571045d375ab7345d171c5dc7d7e13107a80"
 dependencies = [
  "chrono",
- "num-bigint 0.4.3",
+ "num-bigint",
  "num-traits",
  "thiserror",
 ]
 
 [[package]]
-name = "siphasher"
-version = "0.3.10"
+name = "siwe"
+version = "0.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de"
+checksum = "f24fe2b646c33a670e7d79a232bffb41821fed28b1870a8bd1a47e6ae686ace6"
+dependencies = [
+ "hex",
+ "http",
+ "iri-string 0.4.1",
+ "k256",
+ "rand 0.8.5",
+ "sha3 0.9.1",
+ "thiserror",
+ "time 0.3.15",
+]
 
 [[package]]
 name = "siwe"
-version = "0.4.0"
+version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8bf4de418b0989028f138b74db880525e1dbab7fdbeeb4ba8ad96f62bc9ed9a3"
+checksum = "ec4cc2eafb2354c1aeaeac5f53b7726ca4ea5ad16314a0b5ebacb6676f599c2b"
 dependencies = [
  "hex",
  "http",
- "iri-string 0.4.1",
+ "iri-string 0.6.0",
  "k256",
  "rand 0.8.5",
- "sha3 0.9.1",
+ "sha3 0.10.6",
+ "thiserror",
+ "time 0.3.15",
+]
+
+[[package]]
+name = "siwe-recap"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "afe6254970fe99274b440ba4c5e9891550939e7a3c10274029b2e8a9ab2d5c8b"
+dependencies = [
+ "base64 0.12.3",
+ "iri-string 0.6.0",
+ "serde",
+ "serde_json",
+ "siwe 0.5.0",
  "thiserror",
- "time 0.3.14",
 ]
 
 [[package]]
@@ -5626,7 +5491,7 @@ version = "0.4.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef"
 dependencies = [
- "autocfg 1.1.0",
+ "autocfg",
 ]
 
 [[package]]
@@ -5647,9 +5512,9 @@ dependencies = [
 
 [[package]]
 name = "smallvec"
-version = "1.9.0"
+version = "1.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2fd0db749597d91ff862fd1d55ea87f7855a744a8425a64695b6fca237d1dad1"
+checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0"
 
 [[package]]
 name = "snow"
@@ -5657,7 +5522,7 @@ version = "0.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "774d05a3edae07ce6d68ea6984f3c05e9bba8927e3dd591e3b479e5b03213d0d"
 dependencies = [
- "aes-gcm",
+ "aes-gcm 0.9.4",
  "blake2",
  "chacha20poly1305",
  "curve25519-dalek 4.0.0-pre.1",
@@ -5692,20 +5557,22 @@ checksum = "7f6002a767bff9e83f8eeecf883ecb8011875a21ae8da43bffb817a57e78cc09"
 
 [[package]]
 name = "spki"
-version = "0.3.0"
+version = "0.5.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9dae7e047abc519c96350e9484a96c6bf1492348af912fd3446dd2dc323f6268"
+checksum = "44d01ac02a6ccf3e07db148d2be087da624fea0221a16152ed01f0496a6b0a27"
 dependencies = [
- "der 0.3.5",
+ "base64ct",
+ "der 0.5.1",
 ]
 
 [[package]]
 name = "spki"
-version = "0.4.1"
+version = "0.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5c01a0c15da1b0b0e1494112e7af814a678fec9bd157881b49beac661e9b6f32"
+checksum = "67cf02bbac7a337dc36e4f5a693db6c21e7863f45070f7064577eb4367a3212b"
 dependencies = [
- "der 0.4.5",
+ "base64ct",
+ "der 0.6.0",
 ]
 
 [[package]]
@@ -5722,10 +5589,9 @@ dependencies = [
 [[package]]
 name = "ssi"
 version = "0.4.0"
-source = "git+https://github.com/spruceid/ssi?branch=main#d19575d2d4251ddc200d54845a59174971964147"
+source = "git+https://github.com/spruceid/ssi?branch=main#c4a2051c9ed8fdaa503bec39175b13ca24523812"
 dependencies = [
  "ssi-caips",
- "ssi-contexts",
  "ssi-core",
  "ssi-crypto",
  "ssi-dids",
@@ -5744,7 +5610,7 @@ dependencies = [
 [[package]]
 name = "ssi-caips"
 version = "0.1.0"
-source = "git+https://github.com/spruceid/ssi?branch=main#d19575d2d4251ddc200d54845a59174971964147"
+source = "git+https://github.com/spruceid/ssi?branch=main#c4a2051c9ed8fdaa503bec39175b13ca24523812"
 dependencies = [
  "bs58",
  "ssi-jwk",
@@ -5754,12 +5620,12 @@ dependencies = [
 [[package]]
 name = "ssi-contexts"
 version = "0.1.3"
-source = "git+https://github.com/spruceid/ssi?branch=main#d19575d2d4251ddc200d54845a59174971964147"
+source = "git+https://github.com/spruceid/ssi?branch=main#c4a2051c9ed8fdaa503bec39175b13ca24523812"
 
 [[package]]
 name = "ssi-core"
 version = "0.1.0"
-source = "git+https://github.com/spruceid/ssi?branch=main#d19575d2d4251ddc200d54845a59174971964147"
+source = "git+https://github.com/spruceid/ssi?branch=main#c4a2051c9ed8fdaa503bec39175b13ca24523812"
 dependencies = [
  "async-trait",
  "serde",
@@ -5769,14 +5635,13 @@ dependencies = [
 [[package]]
 name = "ssi-crypto"
 version = "0.1.0"
-source = "git+https://github.com/spruceid/ssi?branch=main#d19575d2d4251ddc200d54845a59174971964147"
+source = "git+https://github.com/spruceid/ssi?branch=main#c4a2051c9ed8fdaa503bec39175b13ca24523812"
 dependencies = [
  "bs58",
  "digest 0.9.0",
  "k256",
  "keccak-hash",
  "ripemd160",
- "serde",
  "sha2 0.10.6",
  "thiserror",
  "zeroize",
@@ -5785,7 +5650,7 @@ dependencies = [
 [[package]]
 name = "ssi-dids"
 version = "0.1.0"
-source = "git+https://github.com/spruceid/ssi?branch=main#d19575d2d4251ddc200d54845a59174971964147"
+source = "git+https://github.com/spruceid/ssi?branch=main#c4a2051c9ed8fdaa503bec39175b13ca24523812"
 dependencies = [
  "anyhow",
  "async-std",
@@ -5794,11 +5659,8 @@ dependencies = [
  "chrono",
  "derive_builder",
  "hex",
- "http",
  "multibase 0.8.0",
  "percent-encoding",
- "pest",
- "pest_derive",
  "reqwest",
  "serde",
  "serde_json",
@@ -5813,7 +5675,7 @@ dependencies = [
 [[package]]
 name = "ssi-json-ld"
 version = "0.1.0"
-source = "git+https://github.com/spruceid/ssi?branch=main#d19575d2d4251ddc200d54845a59174971964147"
+source = "git+https://github.com/spruceid/ssi?branch=main#c4a2051c9ed8fdaa503bec39175b13ca24523812"
 dependencies = [
  "async-std",
  "chrono",
@@ -5833,21 +5695,22 @@ dependencies = [
 [[package]]
 name = "ssi-jwk"
 version = "0.1.0"
-source = "git+https://github.com/spruceid/ssi?branch=main#d19575d2d4251ddc200d54845a59174971964147"
+source = "git+https://github.com/spruceid/ssi?branch=main#c4a2051c9ed8fdaa503bec39175b13ca24523812"
 dependencies = [
  "base64 0.12.3",
  "blake2b_simd 0.5.11",
  "bs58",
  "ed25519-dalek",
+ "getrandom 0.2.8",
  "k256",
  "lazy_static",
- "num-bigint 0.4.3",
- "p256 0.9.0",
+ "num-bigint",
+ "p256",
  "rand 0.7.3",
  "rand 0.8.5",
  "rsa",
  "serde",
- "simple_asn1 0.5.4",
+ "simple_asn1",
  "ssi-crypto",
  "thiserror",
  "zeroize",
@@ -5856,20 +5719,19 @@ dependencies = [
 [[package]]
 name = "ssi-jws"
 version = "0.1.0"
-source = "git+https://github.com/spruceid/ssi?branch=main#d19575d2d4251ddc200d54845a59174971964147"
+source = "git+https://github.com/spruceid/ssi?branch=main#c4a2051c9ed8fdaa503bec39175b13ca24523812"
 dependencies = [
  "base64 0.12.3",
- "blake2b_simd 0.5.11",
+ "blake2",
  "clear_on_drop",
  "ed25519-dalek",
  "k256",
- "p256 0.9.0",
- "rand 0.7.3",
+ "p256",
+ "rand 0.8.5",
  "rsa",
  "serde",
  "serde_json",
- "sha2 0.9.9",
- "ssi-core",
+ "sha2 0.10.6",
  "ssi-crypto",
  "ssi-jwk",
  "thiserror",
@@ -5878,7 +5740,7 @@ dependencies = [
 [[package]]
 name = "ssi-jwt"
 version = "0.1.0"
-source = "git+https://github.com/spruceid/ssi?branch=main#d19575d2d4251ddc200d54845a59174971964147"
+source = "git+https://github.com/spruceid/ssi?branch=main#c4a2051c9ed8fdaa503bec39175b13ca24523812"
 dependencies = [
  "chrono",
  "serde",
@@ -5891,17 +5753,14 @@ dependencies = [
 [[package]]
 name = "ssi-ldp"
 version = "0.1.0"
-source = "git+https://github.com/spruceid/ssi?branch=main#d19575d2d4251ddc200d54845a59174971964147"
+source = "git+https://github.com/spruceid/ssi?branch=main#c4a2051c9ed8fdaa503bec39175b13ca24523812"
 dependencies = [
  "async-trait",
  "bs58",
  "chrono",
  "hex",
- "k256",
- "keccak-hash",
  "lazy_static",
  "multibase 0.8.0",
- "p256 0.9.0",
  "serde",
  "serde_jcs",
  "serde_json",
@@ -5913,15 +5772,13 @@ dependencies = [
  "ssi-json-ld",
  "ssi-jwk",
  "ssi-jws",
- "ssi-jwt",
- "ssi-tzkey",
  "thiserror",
 ]
 
 [[package]]
 name = "ssi-ssh"
 version = "0.1.0"
-source = "git+https://github.com/spruceid/ssi?branch=main#d19575d2d4251ddc200d54845a59174971964147"
+source = "git+https://github.com/spruceid/ssi?branch=main#c4a2051c9ed8fdaa503bec39175b13ca24523812"
 dependencies = [
  "sshkeys",
  "ssi-jwk",
@@ -5931,7 +5788,7 @@ dependencies = [
 [[package]]
 name = "ssi-tzkey"
 version = "0.1.0"
-source = "git+https://github.com/spruceid/ssi?branch=main#d19575d2d4251ddc200d54845a59174971964147"
+source = "git+https://github.com/spruceid/ssi?branch=main#c4a2051c9ed8fdaa503bec39175b13ca24523812"
 dependencies = [
  "bs58",
  "ed25519-dalek",
@@ -5943,14 +5800,14 @@ dependencies = [
 [[package]]
 name = "ssi-ucan"
 version = "0.1.0"
-source = "git+https://github.com/spruceid/ssi?branch=main#d19575d2d4251ddc200d54845a59174971964147"
+source = "git+https://github.com/spruceid/ssi?branch=main#c4a2051c9ed8fdaa503bec39175b13ca24523812"
 dependencies = [
  "base64 0.12.3",
  "chrono",
- "libipld",
+ "libipld 0.13.1",
  "serde",
  "serde_json",
- "serde_with",
+ "serde_with 1.14.0",
  "ssi-caips",
  "ssi-core",
  "ssi-dids",
@@ -5963,16 +5820,20 @@ dependencies = [
 [[package]]
 name = "ssi-vc"
 version = "0.1.0"
-source = "git+https://github.com/spruceid/ssi?branch=main#d19575d2d4251ddc200d54845a59174971964147"
+source = "git+https://github.com/spruceid/ssi?branch=main#c4a2051c9ed8fdaa503bec39175b13ca24523812"
 dependencies = [
  "async-trait",
  "base64 0.12.3",
- "bitvec 0.20.4",
+ "bitvec",
+ "cacaos",
  "chrono",
  "flate2",
+ "libipld 0.14.0",
+ "multihash",
  "reqwest",
  "serde",
  "serde_json",
+ "siwe-recap",
  "ssi-core",
  "ssi-dids",
  "ssi-json-ld",
@@ -5986,7 +5847,7 @@ dependencies = [
 [[package]]
 name = "ssi-zcap-ld"
 version = "0.1.0"
-source = "git+https://github.com/spruceid/ssi?branch=main#d19575d2d4251ddc200d54845a59174971964147"
+source = "git+https://github.com/spruceid/ssi?branch=main#c4a2051c9ed8fdaa503bec39175b13ca24523812"
 dependencies = [
  "async-trait",
  "serde",
@@ -6008,12 +5869,6 @@ dependencies = [
  "memchr",
 ]
 
-[[package]]
-name = "stable_deref_trait"
-version = "1.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"
-
 [[package]]
 name = "state"
 version = "0.5.3"
@@ -6029,19 +5884,6 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
 
-[[package]]
-name = "string_cache"
-version = "0.8.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "213494b7a2b503146286049378ce02b482200519accc31872ee8be91fa820a08"
-dependencies = [
- "new_debug_unreachable",
- "once_cell",
- "parking_lot 0.12.1",
- "phf_shared",
- "precomputed-hash",
-]
-
 [[package]]
 name = "strsim"
 version = "0.9.3"
@@ -6062,9 +5904,9 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601"
 
 [[package]]
 name = "syn"
-version = "1.0.100"
+version = "1.0.103"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "52205623b1b0f064a4e71182c3b18ae902267282930c6d5462c91b859668426e"
+checksum = "a864042229133ada95abf3b54fdc62ef5ccabe9515b64717bcb9a1919e59445d"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -6083,6 +5925,27 @@ dependencies = [
  "unicode-xid",
 ]
 
+[[package]]
+name = "system-configuration"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d75182f12f490e953596550b65ee31bda7c8e043d9386174b353bda50838c3fd"
+dependencies = [
+ "bitflags",
+ "core-foundation",
+ "system-configuration-sys",
+]
+
+[[package]]
+name = "system-configuration-sys"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
+]
+
 [[package]]
 name = "tap"
 version = "1.0.1"
@@ -6114,30 +5977,28 @@ dependencies = [
 ]
 
 [[package]]
-name = "term"
-version = "0.7.0"
+name = "termcolor"
+version = "1.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f"
+checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755"
 dependencies = [
- "dirs-next",
- "rustversion",
- "winapi",
+ "winapi-util",
 ]
 
 [[package]]
 name = "thiserror"
-version = "1.0.35"
+version = "1.0.37"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c53f98874615aea268107765aa1ed8f6116782501d18e53d08b471733bea6c85"
+checksum = "10deb33631e3c9018b9baf9dcbbc4f737320d2b576bac10f6aefa048fa407e3e"
 dependencies = [
  "thiserror-impl",
 ]
 
 [[package]]
 name = "thiserror-impl"
-version = "1.0.35"
+version = "1.0.37"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f8b463991b4eab2d801e724172285ec4195c650e8ec79b149e6c2a8e6dd3f783"
+checksum = "982d17546b47146b28f7c22e3d08465f6b8903d0ea13c1660d9d84a6e7adcdbb"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -6188,13 +6049,14 @@ dependencies = [
 
 [[package]]
 name = "time"
-version = "0.3.14"
+version = "0.3.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3c3f9a28b618c3a6b9251b6908e9c99e04b9e5c02e6581ccbb67d59c34ef7f9b"
+checksum = "d634a985c4d4238ec39cacaed2e7ae552fbd3c476b552c1deac3021b7d7eaf0c"
 dependencies = [
  "itoa",
  "libc",
  "num_threads",
+ "serde",
  "time-macros",
 ]
 
@@ -6230,17 +6092,16 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
 
 [[package]]
 name = "tokio"
-version = "1.21.1"
+version = "1.21.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0020c875007ad96677dcc890298f4b942882c5d4eb7cc8f439fc3bf813dc9c95"
+checksum = "a9e03c497dc955702ba729190dc4aac6f2a0ce97f913e5b1b5912fc5039d9099"
 dependencies = [
- "autocfg 1.1.0",
+ "autocfg",
  "bytes",
  "libc",
  "memchr",
  "mio",
  "num_cpus",
- "once_cell",
  "pin-project-lite",
  "signal-hook-registry",
  "socket2",
@@ -6294,9 +6155,9 @@ dependencies = [
 
 [[package]]
 name = "tokio-stream"
-version = "0.1.10"
+version = "0.1.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f6edf2d6bc038a43d31353570e27270603f4648d18f5ed10c0e179abe43255af"
+checksum = "d660770404473ccd7bc9f8b28494a811bc18542b915c0855c51e8f419d5223ce"
 dependencies = [
  "futures-core",
  "pin-project-lite",
@@ -6348,7 +6209,7 @@ checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c"
 dependencies = [
  "futures-core",
  "futures-util",
- "pin-project 1.0.12",
+ "pin-project",
  "pin-project-lite",
  "tokio",
  "tower-layer",
@@ -6358,9 +6219,9 @@ dependencies = [
 
 [[package]]
 name = "tower-layer"
-version = "0.3.1"
+version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "343bc9466d3fe6b0f960ef45960509f84480bf4fd96f92901afe7ff3df9d3a62"
+checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0"
 
 [[package]]
 name = "tower-service"
@@ -6370,9 +6231,9 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52"
 
 [[package]]
 name = "tracing"
-version = "0.1.36"
+version = "0.1.37"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2fce9567bd60a67d08a16488756721ba392f24f29006402881e43b19aac64307"
+checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8"
 dependencies = [
  "cfg-if",
  "log",
@@ -6383,9 +6244,9 @@ dependencies = [
 
 [[package]]
 name = "tracing-attributes"
-version = "0.1.22"
+version = "0.1.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "11c75893af559bc8e10716548bdef5cb2b983f8e637db9d0e15126b61b484ee2"
+checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -6394,26 +6255,14 @@ dependencies = [
 
 [[package]]
 name = "tracing-core"
-version = "0.1.29"
+version = "0.1.30"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5aeea4303076558a00714b823f9ad67d58a3bbda1df83d8827d21193156e22f7"
+checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a"
 dependencies = [
  "once_cell",
  "valuable",
 ]
 
-[[package]]
-name = "tracing-futures"
-version = "0.2.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2"
-dependencies = [
- "futures",
- "futures-task",
- "pin-project 1.0.12",
- "tracing",
-]
-
 [[package]]
 name = "tracing-log"
 version = "0.1.3"
@@ -6451,12 +6300,12 @@ dependencies = [
 
 [[package]]
 name = "tracing-subscriber"
-version = "0.3.15"
+version = "0.3.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "60db860322da191b40952ad9affe65ea23e7dd6a5c442c2c42865810c6ab8e6b"
+checksum = "a6176eae26dd70d0c919749377897b54a9276bd7061339665dd68777926b5a70"
 dependencies = [
- "ansi_term",
  "matchers",
+ "nu-ansi-term",
  "once_cell",
  "regex",
  "serde",
@@ -6481,9 +6330,9 @@ dependencies = [
 
 [[package]]
 name = "trust-dns-proto"
-version = "0.21.2"
+version = "0.22.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c31f240f59877c3d4bb3b3ea0ec5a6a0cff07323580ff8c7a605cd7d08b255d"
+checksum = "4f7f83d1e4a0e4358ac54c5c3681e5d7da5efc5a7a632c90bb6d6669ddd9bc26"
 dependencies = [
  "async-trait",
  "cfg-if",
@@ -6495,32 +6344,32 @@ dependencies = [
  "idna 0.2.3",
  "ipnet",
  "lazy_static",
- "log",
  "rand 0.8.5",
  "smallvec",
  "thiserror",
  "tinyvec",
  "tokio",
+ "tracing",
  "url",
 ]
 
 [[package]]
 name = "trust-dns-resolver"
-version = "0.21.2"
+version = "0.22.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e4ba72c2ea84515690c9fcef4c6c660bb9df3036ed1051686de84605b74fd558"
+checksum = "aff21aa4dcefb0a1afbfac26deb0adc93888c7d295fb63ab273ef276ba2b7cfe"
 dependencies = [
  "cfg-if",
  "futures-util",
  "ipconfig",
  "lazy_static",
- "log",
  "lru-cache",
  "parking_lot 0.12.1",
  "resolv-conf",
  "smallvec",
  "thiserror",
  "tokio",
+ "tracing",
  "trust-dns-proto",
 ]
 
@@ -6532,12 +6381,12 @@ checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642"
 
 [[package]]
 name = "twofish"
-version = "0.5.0"
+version = "0.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0028f5982f23ecc9a1bc3008ead4c664f843ed5d78acd3d213b99ff50c441bc2"
+checksum = "728f6b7e784825d272fe9d2a77e44063f4197a570cbedc6fdcc90a6ddac91296"
 dependencies = [
  "byteorder",
- "cipher 0.2.5",
+ "cipher 0.3.0",
  "opaque-debug 0.3.0",
 ]
 
@@ -6556,17 +6405,11 @@ dependencies = [
  "serde",
 ]
 
-[[package]]
-name = "ucd-trie"
-version = "0.1.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9e79c4d996edb816c91e4308506774452e55e95c3c9de07b6729e17e15a5ef81"
-
 [[package]]
 name = "uint"
-version = "0.9.3"
+version = "0.9.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "12f03af7ccf01dd611cc450a0d10dbc9b745770d096473e2faf0ca6e2d66d1e0"
+checksum = "a45526d29728d135c2900b0d30573fe3ee79fceb12ef534c7bb30e810a91b601"
 dependencies = [
  "byteorder",
  "crunchy",
@@ -6581,7 +6424,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "09b01702b0fd0b3fadcf98e098780badda8742d4f4a7676615cad90e8ac73622"
 dependencies = [
  "serde",
- "version_check",
+ "version_check 0.9.4",
 ]
 
 [[package]]
@@ -6592,9 +6435,9 @@ checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992"
 
 [[package]]
 name = "unicode-ident"
-version = "1.0.4"
+version = "1.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dcc811dc4066ac62f84f11307873c4850cb653bfa9b1719cee2bd2204a4bc5dd"
+checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3"
 
 [[package]]
 name = "unicode-normalization"
@@ -6611,6 +6454,12 @@ version = "1.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0fdbf052a0783de01e944a6ce7a8cb939e295b1e7be835a1112c3b9a7f047a5a"
 
+[[package]]
+name = "unicode-width"
+version = "0.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b"
+
 [[package]]
 name = "unicode-xid"
 version = "0.2.4"
@@ -6628,10 +6477,14 @@ dependencies = [
 ]
 
 [[package]]
-name = "unsigned-varint"
-version = "0.5.1"
+name = "universal-hash"
+version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f7fdeedbf205afadfe39ae559b75c3240f24e257d0ca27e85f85cb82aa19ac35"
+checksum = "7d3160b73c9a19f7e2939a2fdad446c57c1bbbbf4d919d3213ff1267a580d8b5"
+dependencies = [
+ "crypto-common",
+ "subtle",
+]
 
 [[package]]
 name = "unsigned-varint"
@@ -6674,11 +6527,12 @@ checksum = "e8db7427f936968176eaa7cdf81b7f98b980b18495ec28f1b5791ac3bfe3eea9"
 
 [[package]]
 name = "uuid"
-version = "1.1.2"
+version = "1.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dd6469f4314d5f1ffec476e05f17cc9a78bc7a27a6a857842170bdf8d6f98d2f"
+checksum = "feb41e78f93363bb2df8b0e86a2ca30eed7806ea16ea0c790d757cf93f79be83"
 dependencies = [
- "getrandom 0.2.7",
+ "getrandom 0.2.8",
+ "wasm-bindgen",
 ]
 
 [[package]]
@@ -6694,7 +6548,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2209b78d1249f7e6f3293657c9779fe31ced465df091bbd433a1cf88e916ec55"
 dependencies = [
  "ctor",
- "version_check",
+ "version_check 0.9.4",
 ]
 
 [[package]]
@@ -6703,6 +6557,12 @@ version = "0.2.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
 
+[[package]]
+name = "version_check"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "914b1a6776c4c929a602fafd8bc742e06365d4bcbe48c30f9cca5824f70dc9dd"
+
 [[package]]
 name = "version_check"
 version = "0.9.4"
@@ -6877,6 +6737,15 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
 
+[[package]]
+name = "winapi-util"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178"
+dependencies = [
+ "winapi",
+]
+
 [[package]]
 name = "winapi-x86_64-pc-windows-gnu"
 version = "0.4.0"
@@ -6896,6 +6765,19 @@ dependencies = [
  "windows_x86_64_msvc 0.32.0",
 ]
 
+[[package]]
+name = "windows"
+version = "0.34.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "45296b64204227616fdbf2614cefa4c236b98ee64dfaaaa435207ed99fe7829f"
+dependencies = [
+ "windows_aarch64_msvc 0.34.0",
+ "windows_i686_gnu 0.34.0",
+ "windows_i686_msvc 0.34.0",
+ "windows_x86_64_gnu 0.34.0",
+ "windows_x86_64_msvc 0.34.0",
+]
+
 [[package]]
 name = "windows-sys"
 version = "0.36.1"
@@ -6909,66 +6791,153 @@ dependencies = [
  "windows_x86_64_msvc 0.36.1",
 ]
 
+[[package]]
+name = "windows-sys"
+version = "0.42.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7"
+dependencies = [
+ "windows_aarch64_gnullvm",
+ "windows_aarch64_msvc 0.42.0",
+ "windows_i686_gnu 0.42.0",
+ "windows_i686_msvc 0.42.0",
+ "windows_x86_64_gnu 0.42.0",
+ "windows_x86_64_gnullvm",
+ "windows_x86_64_msvc 0.42.0",
+]
+
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.42.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e"
+
 [[package]]
 name = "windows_aarch64_msvc"
 version = "0.32.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d8e92753b1c443191654ec532f14c199742964a061be25d77d7a96f09db20bf5"
 
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.34.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "17cffbe740121affb56fad0fc0e421804adf0ae00891205213b5cecd30db881d"
+
 [[package]]
 name = "windows_aarch64_msvc"
 version = "0.36.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47"
 
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.42.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4"
+
 [[package]]
 name = "windows_i686_gnu"
 version = "0.32.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6a711c68811799e017b6038e0922cb27a5e2f43a2ddb609fe0b6f3eeda9de615"
 
+[[package]]
+name = "windows_i686_gnu"
+version = "0.34.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2564fde759adb79129d9b4f54be42b32c89970c18ebf93124ca8870a498688ed"
+
 [[package]]
 name = "windows_i686_gnu"
 version = "0.36.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6"
 
+[[package]]
+name = "windows_i686_gnu"
+version = "0.42.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7"
+
 [[package]]
 name = "windows_i686_msvc"
 version = "0.32.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "146c11bb1a02615db74680b32a68e2d61f553cc24c4eb5b4ca10311740e44172"
 
+[[package]]
+name = "windows_i686_msvc"
+version = "0.34.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9cd9d32ba70453522332c14d38814bceeb747d80b3958676007acadd7e166956"
+
 [[package]]
 name = "windows_i686_msvc"
 version = "0.36.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024"
 
+[[package]]
+name = "windows_i686_msvc"
+version = "0.42.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246"
+
 [[package]]
 name = "windows_x86_64_gnu"
 version = "0.32.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c912b12f7454c6620635bbff3450962753834be2a594819bd5e945af18ec64bc"
 
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.34.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cfce6deae227ee8d356d19effc141a509cc503dfd1f850622ec4b0f84428e1f4"
+
 [[package]]
 name = "windows_x86_64_gnu"
 version = "0.36.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1"
 
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.42.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bf7b1b21b5362cbc318f686150e5bcea75ecedc74dd157d874d754a2ca44b0ed"
+
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.42.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028"
+
 [[package]]
 name = "windows_x86_64_msvc"
 version = "0.32.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "504a2476202769977a040c6364301a3f65d0cc9e3fb08600b2bda150a0488316"
 
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.34.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d19538ccc21819d01deaf88d6a17eae6596a12e9aafdbb97916fb49896d89de9"
+
 [[package]]
 name = "windows_x86_64_msvc"
 version = "0.36.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680"
 
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.42.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5"
+
 [[package]]
 name = "winreg"
 version = "0.7.0"
@@ -7010,12 +6979,6 @@ version = "0.13.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "114ba2b24d2167ef6d67d7d04c8cc86522b87f490025f39f0303b7db5bf5e3d8"
 
-[[package]]
-name = "xxhash-rust"
-version = "0.8.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "074914ea4eec286eb8d1fd745768504f420a1f7b7919185682a4a267bed7d2e7"
-
 [[package]]
 name = "yamux"
 version = "0.10.2"
@@ -7038,9 +7001,9 @@ checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec"
 
 [[package]]
 name = "zeroize"
-version = "1.4.3"
+version = "1.5.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d68d9dcec5f9b43a30d38c49f91dfedfaac384cb8f085faca366c26207dd1619"
+checksum = "c394b5bd0c6f669e7275d9c20aa90ae064cb22e75a1cad54e1b34088034b149f"
 dependencies = [
  "zeroize_derive",
 ]
diff --git a/Cargo.toml b/Cargo.toml
index f5bc8bbf..0a8704b9 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -19,8 +19,7 @@ base64 = "0.13"
 bincode = "1.3"
 bs58 = "0.4"
 cached = "0.34"
-ethers-core = "0.6"
-futures = { default-features = false, version = "0.3.9", features = ["alloc", "std"] }
+futures = { default-features = false, version = "0.3", features = ["alloc", "std"] }
 hex = "0.4"
 hyper = "0.14" # Prometheus server
 iri-string = "0.5"
diff --git a/lib/Cargo.toml b/lib/Cargo.toml
index 6c5b3e15..d13d3a3c 100644
--- a/lib/Cargo.toml
+++ b/lib/Cargo.toml
@@ -5,15 +5,15 @@ edition = "2021"
 
 [dependencies]
 async-trait = "0.1"
-iri-string = "0.5"
-libipld = "0.13"
+iri-string = "0.6"
+libipld = "0.14"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 serde_with = "1.0"
 thiserror = "1"
 base64 = "0.13"
-cacaos = { git = "https://github.com/spruceid/cacao-rs" }
-capgrok = "0.2"
+cacaos = "0.5"
+capgrok = "0.3"
 lazy_static = "1.4"
 did-method-key = { default-features = false, git = "https://github.com/spruceid/ssi", branch = "main" }
 did-tz = { default-features = false, git = "https://github.com/spruceid/ssi", branch = "main" }

From b30841387ac66085a269bd1a7a06ac3960f8721b Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Fri, 21 Oct 2022 17:25:19 +0200
Subject: [PATCH 09/73] remove SupportedCodecs, unused

---
 lib/src/resource.rs | 12 ++--------
 src/cas.rs          |  1 -
 src/codec.rs        | 55 ---------------------------------------------
 src/ipfs.rs         |  1 -
 src/orbit.rs        |  1 -
 5 files changed, 2 insertions(+), 68 deletions(-)
 delete mode 100644 src/codec.rs

diff --git a/lib/src/resource.rs b/lib/src/resource.rs
index cf721139..9360e49f 100644
--- a/lib/src/resource.rs
+++ b/lib/src/resource.rs
@@ -18,14 +18,6 @@ use std::io::{Read, Seek, Write};
 use std::{convert::TryFrom, fmt, str::FromStr};
 use thiserror::Error;
 
-#[derive(Clone, Copy, Debug)]
-pub enum SupportedCodecs {
-    Raw = 0x55,
-    Json = 0x0200,
-    MsgPack = 0x0201,
-    Cbor = 0x51,
-}
-
 #[derive(Clone, Hash, PartialEq, Debug, Eq, SerializeDisplay, DeserializeFromStr)]
 pub struct OrbitId {
     suffix: String,
@@ -51,7 +43,7 @@ impl OrbitId {
 
     pub fn get_cid(&self) -> Cid {
         Cid::new_v1(
-            SupportedCodecs::Raw as u64,
+            0x55, // raw codec
             Code::Blake2b256.digest(self.to_string().as_bytes()),
         )
     }
@@ -131,7 +123,7 @@ impl ResourceId {
 
     pub fn get_cid(&self) -> Cid {
         Cid::new_v1(
-            SupportedCodecs::Raw as u64,
+            0x55, // raw codec
             Code::Blake2b256.digest(self.to_string().as_bytes()),
         )
     }
diff --git a/src/cas.rs b/src/cas.rs
index b4c54866..437be14d 100644
--- a/src/cas.rs
+++ b/src/cas.rs
@@ -1,4 +1,3 @@
-use super::codec::SupportedCodecs;
 use anyhow::Result;
 use kepler_lib::libipld::cid::Cid;
 use rocket::{
diff --git a/src/codec.rs b/src/codec.rs
deleted file mode 100644
index 6addb25a..00000000
--- a/src/codec.rs
+++ /dev/null
@@ -1,55 +0,0 @@
-use rocket::{
-    data::{Capped, ToByteUnit},
-    form::{DataField, FromFormField, Result},
-    http::ContentType,
-    request::{FromRequest, Outcome, Request},
-};
-
-#[derive(Clone, Copy, Debug)]
-pub enum SupportedCodecs {
-    Raw = 0x55,
-    Json = 0x0200,
-    MsgPack = 0x0201,
-    Cbor = 0x51,
-}
-
-#[derive(Debug)]
-pub struct PutContent {
-    pub codec: SupportedCodecs,
-    // TODO dont use a Vec, but passing the datastream results in a hang
-    pub content: Capped<Vec<u8>>,
-}
-
-impl From<&ContentType> for SupportedCodecs {
-    fn from(c: &ContentType) -> Self {
-        if c.is_json() {
-            Self::Json
-        } else if c.is_msgpack() {
-            Self::MsgPack
-        } else {
-            Self::Raw
-        }
-    }
-}
-
-#[rocket::async_trait]
-impl<'r> FromRequest<'r> for SupportedCodecs {
-    type Error = anyhow::Error;
-
-    async fn from_request(req: &'r Request<'_>) -> Outcome<Self, Self::Error> {
-        Outcome::Success(match req.content_type() {
-            Some(t) => Self::from(t),
-            None => Self::Raw,
-        })
-    }
-}
-
-#[rocket::async_trait]
-impl<'r> FromFormField<'r> for PutContent {
-    async fn from_data(field: DataField<'r, '_>) -> Result<'r, Self> {
-        Ok(PutContent {
-            codec: (&field.content_type).into(),
-            content: field.data.open(1u8.megabytes()).into_bytes().await?,
-        })
-    }
-}
diff --git a/src/ipfs.rs b/src/ipfs.rs
index e8d1d428..cb9656bc 100644
--- a/src/ipfs.rs
+++ b/src/ipfs.rs
@@ -9,7 +9,6 @@ use kepler_lib::libipld::{
 use libp2p::{core::transport::MemoryTransport, futures::TryStreamExt, swarm::Swarm as TSwarm};
 use std::{future::Future, sync::mpsc::Receiver};
 
-use super::{cas::ContentAddressedStorage, codec::SupportedCodecs};
 use crate::{
     config,
     kv::behaviour::{Behaviour, Event as BehaviourEvent},
diff --git a/src/orbit.rs b/src/orbit.rs
index 81d5042c..6a425899 100644
--- a/src/orbit.rs
+++ b/src/orbit.rs
@@ -1,7 +1,6 @@
 use crate::{
     capabilities::{store::Store as CapStore, Service as CapService},
     cas::ContentAddressedStorage,
-    codec::SupportedCodecs,
     config,
     kv::{behaviour::BehaviourProcess, Service as KVService, Store},
     manifest::Manifest,

From 62f48cbbeefeb3e1a47c0b15a7e0fbfb555cb606 Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Fri, 21 Oct 2022 17:25:59 +0200
Subject: [PATCH 10/73] fixes, stub kv behaviour

---
 src/capabilities/store.rs  |  5 ++++-
 src/kv/behaviour.rs        | 32 +++++++++++++++++---------------
 src/storage/file_system.rs | 12 +++++-------
 3 files changed, 26 insertions(+), 23 deletions(-)

diff --git a/src/capabilities/store.rs b/src/capabilities/store.rs
index 12c2c782..0a27bc00 100644
--- a/src/capabilities/store.rs
+++ b/src/capabilities/store.rs
@@ -362,7 +362,10 @@ impl Store<B> {
 }
 
 #[rocket::async_trait]
-impl CapStore for Store {
+impl<B> CapStore for Store<B>
+where
+    B: ImmutableStore,
+{
     async fn get_cap(&self, c: &Cid) -> Result<Option<Delegation>> {
         // annoyingly ipfs will error if it cant find something, so we probably dont want to error here
         Ok(self.get_obj(c).await.ok().map(|d| d.base))
diff --git a/src/kv/behaviour.rs b/src/kv/behaviour.rs
index 2f92bc47..ca848620 100644
--- a/src/kv/behaviour.rs
+++ b/src/kv/behaviour.rs
@@ -9,8 +9,8 @@ use std::{
 use libp2p::{
     core::{connection::ConnectionId, ConnectedPoint, Multiaddr, PeerId},
     swarm::{
-        handler::DummyConnectionHandler, IntoConnectionHandler, NetworkBehaviour,
-        NetworkBehaviourAction, PollParameters,
+        handler::ConnectionHandler as DummyConnectionHandler, IntoConnectionHandler,
+        NetworkBehaviour, NetworkBehaviourAction, PollParameters,
     },
 };
 use void::Void;
@@ -80,7 +80,7 @@ impl NetworkBehaviour for Behaviour {
 pub struct BehaviourProcess(Arc<AbortOnDrop<()>>);
 
 impl BehaviourProcess {
-    pub fn new(store: Store, mut receiver: Receiver<Event>) -> Self {
+    pub fn new<B>(store: Store<B>, mut receiver: Receiver<Event>) -> Self {
         Self(Arc::new(AbortOnDrop::new(tokio::spawn(async move {
             while let Ok(Ok((event, returned_receiver))) =
                 tokio::task::spawn_blocking(move || receiver.recv().map(|ev| (ev, receiver))).await
@@ -88,20 +88,22 @@ impl BehaviourProcess {
                 receiver = returned_receiver;
                 match event {
                     Event::ConnectionEstablished(peer_id) => {
-                        if let Err(e) = store.ipfs.pubsub_add_peer(peer_id).await {
-                            tracing::error!("failed to add new peer to allowed pubsub peers: {}", e)
-                        }
-                        if let Err(e) = store.request_heads().await {
-                            tracing::error!("failed to request heads from peers: {}", e)
-                        }
+                        todo!("synchronise state with peer");
+                        // if let Err(e) = store.ipfs.pubsub_add_peer(peer_id).await {
+                        //     tracing::error!("failed to add new peer to allowed pubsub peers: {}", e)
+                        // }
+                        // if let Err(e) = store.request_heads().await {
+                        //     tracing::error!("failed to request heads from peers: {}", e)
+                        // }
                     }
                     Event::ConnectionTerminated(peer_id) => {
-                        if let Err(e) = store.ipfs.pubsub_remove_peer(peer_id).await {
-                            tracing::error!(
-                                "failed to remove disconnected peer from allowed pubsub peers: {}",
-                                e
-                            )
-                        }
+                        todo!("remove peer from set/deny connection");
+                        // if let Err(e) = store.ipfs.pubsub_remove_peer(peer_id).await {
+                        //     tracing::error!(
+                        //         "failed to remove disconnected peer from allowed pubsub peers: {}",
+                        //         e
+                        //     )
+                        // }
                     }
                 }
             }
diff --git a/src/storage/file_system.rs b/src/storage/file_system.rs
index 483315a7..2508aac2 100644
--- a/src/storage/file_system.rs
+++ b/src/storage/file_system.rs
@@ -3,10 +3,7 @@ use kepler_lib::libipld::cid::{
     multibase::{encode, Base},
     multihash::Multihash,
 };
-use std::{
-    io::{Error, ErrorKind},
-    path::PathBuf,
-};
+use std::{io::ErrorKind, path::PathBuf};
 use tokio::{
     fs::{remove_file, File},
     io::copy,
@@ -21,11 +18,12 @@ impl FileSystemStore {
         Self { path }
     }
 
-    fn get_path(mh: &Multihash) -> PathBuf {
+    fn get_path(&self, mh: &Multihash) -> PathBuf {
         self.path.join(encode(Base::Base64Url, mh))
     }
 }
 
+#[async_trait]
 impl ImmutableStore for FileSystemStore {
     type Error = std::io::Error;
     type Readable = File;
@@ -42,14 +40,14 @@ impl ImmutableStore for FileSystemStore {
     async fn remove(&self, id: &Multihash) -> Result<Option<()>, Self::Error> {
         match remove_file(self.get_path(id)).await {
             Ok(()) => Ok(Some(())),
-            Err(e) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),
+            Err(e) if e.kind() == ErrorKind::NotFound => Ok(None),
             Err(e) => Err(e),
         }
     }
     async fn read(&self, id: &Multihash) -> Result<Option<Self::Readable>, Self::Error> {
         match File::open(self.get_path(id)).await {
             Ok(f) => Ok(Some(f)),
-            Err(e) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),
+            Err(e) if e.kind() == ErrorKind::NotFound => Ok(None),
             Err(e) => Err(e),
         }
     }

From cfa1feac0698a551194d91bd85b5690907529200 Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Mon, 24 Oct 2022 14:18:51 +0200
Subject: [PATCH 11/73] impl EitherStore to combine stores, make Orbit generic

---
 src/orbit.rs          | 22 ++++-------
 src/storage/either.rs | 86 +++++++++++++++++++++++++++++++++++++++++++
 src/storage/mod.rs    | 13 +++----
 3 files changed, 99 insertions(+), 22 deletions(-)
 create mode 100644 src/storage/either.rs

diff --git a/src/orbit.rs b/src/orbit.rs
index 6a425899..5384de96 100644
--- a/src/orbit.rs
+++ b/src/orbit.rs
@@ -1,9 +1,9 @@
 use crate::{
     capabilities::{store::Store as CapStore, Service as CapService},
-    cas::ContentAddressedStorage,
     config,
     kv::{behaviour::BehaviourProcess, Service as KVService, Store},
     manifest::Manifest,
+    storage::{BlockStores, ImmutableStore},
 };
 use anyhow::{anyhow, Result};
 use kepler_lib::libipld::cid::{
@@ -63,12 +63,11 @@ impl OrbitTasks {
 #[derive(Clone)]
 pub struct Orbit<B> {
     pub service: KVService<B>,
-    _tasks: OrbitTasks,
     pub manifest: Manifest,
     pub capabilities: CapService<B>,
 }
 
-impl Orbit {
+impl<B> Orbit<B> {
     async fn new(
         config: &config::Config,
         kp: Ed25519Keypair,
@@ -91,21 +90,16 @@ impl Orbit {
         .await?;
         let capabilities = CapService::start(cap_store).await?;
 
-        let behaviour_process = BehaviourProcess::new(service.store.clone(), receiver);
-
-        let tasks = OrbitTasks::new(behaviour_process);
-
         Ok(Orbit {
             service,
             manifest,
-            _tasks: tasks,
             capabilities,
         })
     }
 
-    pub async fn connect(&self, node: MultiaddrWithPeerId) -> anyhow::Result<()> {
-        self.service.store.ipfs.connect(node).await
-    }
+    // pub async fn connect(&self, node: MultiaddrWithPeerId) -> anyhow::Result<()> {
+    // self.service.store.ipfs.connect(node).await
+    // }
 }
 
 // Using Option to distinguish when the orbit already exists from a hard error
@@ -115,7 +109,7 @@ pub async fn create_orbit(
     auth: &[u8],
     relay: (PeerId, Multiaddr),
     kp: Ed25519Keypair,
-) -> Result<Option<Orbit>> {
+) -> Result<Option<Orbit<BlockStores>>> {
     let md = match Manifest::resolve_dyn(id, None).await? {
         Some(m) => m,
         _ => return Ok(None),
@@ -140,7 +134,7 @@ pub async fn load_orbit(
     id_cid: Cid,
     config: &config::Config,
     relay: (PeerId, Multiaddr),
-) -> Result<Option<Orbit>> {
+) -> Result<Option<Orbit<BlockStores>>> {
     let storage_utils = StorageUtils::new(config.storage.blocks.clone());
     if !storage_utils.exists(id_cid).await? {
         return Ok(None);
@@ -157,7 +151,7 @@ async fn load_orbit_inner(
     orbit: Cid,
     config: config::Config,
     relay: (PeerId, Multiaddr),
-) -> Result<Orbit> {
+) -> Result<Orbit<BlockStores>> {
     let storage_utils = StorageUtils::new(config.storage.blocks.clone());
     let id = storage_utils
         .orbit_id(orbit)
diff --git a/src/storage/either.rs b/src/storage/either.rs
new file mode 100644
index 00000000..489cb862
--- /dev/null
+++ b/src/storage/either.rs
@@ -0,0 +1,86 @@
+use super::ImmutableStore;
+use core::pin::Pin;
+use futures::{
+    io::{AsyncRead, Error},
+    task::{Context, Poll},
+};
+use kepler_lib::libipld::cid::multihash::Multihash;
+
+#[derive(Debug)]
+pub enum EitherStore<L, R> {
+    Left(L),
+    Right(R),
+}
+
+#[derive(Debug)]
+pub enum AsyncReadEither<L, R>
+where
+    L: ImmutableStore,
+    R: ImmutableStore,
+{
+    Left(L::Readable),
+    Right(R::Readable),
+}
+
+impl<L, R> AsyncRead for AsyncReadEither<L, R>
+where
+    L: ImmutableStore,
+    R: ImmutableStore,
+{
+    fn poll_read(
+        self: Pin<&mut Self>,
+        cx: &mut Context<'_>,
+        buf: &mut [u8],
+    ) -> Poll<Result<usize, Error>> {
+        match self {
+            Self::L(l) => l.poll_read(cx, buf),
+            Self::R(r) => r.poll_read(cx, buf),
+        }
+    }
+}
+
+#[derive(thiserror::Error)]
+pub enum EitherStoreError<L, R>
+where
+    L: ImmutableStore,
+    R: ImmutableStore,
+{
+    #[error(transparent)]
+    Left(L::Error),
+    #[error(transparent)]
+    Right(R::Error),
+}
+
+#[async_trait]
+impl<L, R> ImmutableStore for EitherStore<L, R>
+where
+    L: ImmutableStore,
+    R: ImmutableStore,
+{
+    type Readable = AsyncReadEither<L, R>;
+    type Error = EitherStoreError<L, R>;
+    async fn contains(&self, id: &Multihash) -> Result<bool, Self::Error> {
+        match self {
+            Self::Left(l) => l.contains(id).await.map_err(Self::Error::Left),
+            Self::Right(r) => r.contains(id).await.map_err(Self::Error::Right),
+        }
+    }
+    async fn write(&self, data: impl futures::io::AsyncRead) -> Result<Multihash, Self::Error> {
+        match self {
+            Self::Left(l) => l.write(data).await.map_err(Self::Error::Left),
+            Self::Right(r) => r.write(data).await.map_err(Self::Error::Right),
+        }
+    }
+    async fn remove(&self, id: &Multihash) -> Result<Option<()>, Self::Error> {
+        match self {
+            Self::Left(l) => l.remove(id).await.map_err(Self::Error::Left),
+            Self::Right(r) => r.remove(id).await.map_err(Self::Error::Right),
+        }
+    }
+    async fn read(&self, id: &Multihash) -> Result<Option<Self::Readable>, Self::Error> {
+        match self {
+            Self::Left(l) => l.read(id).await.map_err(Self::Error::Left),
+            Self::Right(r) => r.read(id).await.map_err(Self::Error::Right),
+        }
+    }
+}
diff --git a/src/storage/mod.rs b/src/storage/mod.rs
index 26af5ae8..2f45759c 100644
--- a/src/storage/mod.rs
+++ b/src/storage/mod.rs
@@ -12,6 +12,7 @@ use std::{collections::HashMap, path::PathBuf, str::FromStr};
 use tracing::instrument;
 
 mod dynamodb;
+mod either;
 mod file_system;
 mod indexes;
 mod s3;
@@ -20,7 +21,6 @@ mod utils;
 pub use indexes::KV;
 
 use crate::config;
-use dynamodb::References;
 
 pub struct StorageUtils {
     config: config::BlockStorage,
@@ -29,11 +29,8 @@ pub struct StorageUtils {
 #[derive(Debug)]
 pub struct Repo;
 
-#[derive(Debug)]
-pub enum BlockStores {
-    S3(Box<s3::S3BlockStore>),
-    Local(Box<file_system::FileSystemStore>),
-}
+pub type BlockStores =
+    either::EitherStore<Box<s3::S3BlockStore>, Box<file_system::FileSystemStore>>;
 
 #[derive(Debug)]
 pub enum DataStores {
@@ -238,7 +235,7 @@ impl StorageUtils {
     }
 }
 
-#[derive(Error)]
+#[derive(thiserror::Error)]
 pub enum VecReadError<E> {
     #[error(transparent)]
     Store(#[from] E),
@@ -247,7 +244,7 @@ pub enum VecReadError<E> {
 }
 
 #[async_trait]
-trait ImmutableStore {
+pub trait ImmutableStore {
     type Error;
     type Readable: futures::io::AsyncRead;
     async fn contains(&self, id: &Multihash) -> Result<bool, Self::Error>;

From 22e16f2535b7b912cf650cb6a3a880413918c090 Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Mon, 24 Oct 2022 14:20:31 +0200
Subject: [PATCH 12/73] minor core fixes

---
 src/capabilities/mod.rs   | 4 ++--
 src/capabilities/store.rs | 3 ++-
 src/kv/mod.rs             | 3 +--
 src/kv/store.rs           | 7 +++----
 src/lib.rs                | 1 -
 5 files changed, 8 insertions(+), 10 deletions(-)

diff --git a/src/capabilities/mod.rs b/src/capabilities/mod.rs
index 15b52730..0901b213 100644
--- a/src/capabilities/mod.rs
+++ b/src/capabilities/mod.rs
@@ -3,7 +3,7 @@ pub mod store;
 use crate::orbit::AbortOnDrop;
 use anyhow::Result;
 use kepler_lib::libipld::{cbor::DagCborCodec, codec::Decode, multibase::Base, Cid};
-use libp2p::identity::PeerId;
+use libp2p::core::PeerId;
 use rocket::futures::{Stream, StreamExt};
 use store::{CapsMessage, Store};
 
@@ -27,7 +27,7 @@ impl<B> std::ops::Deref for Service<B> {
     }
 }
 
-impl Service<B> {
+impl<B> Service<B> {
     fn new(store: Store<B>) -> Self {
         Self { store }
     }
diff --git a/src/capabilities/store.rs b/src/capabilities/store.rs
index 0a27bc00..0dc4f21d 100644
--- a/src/capabilities/store.rs
+++ b/src/capabilities/store.rs
@@ -3,6 +3,7 @@ use crate::{
     indexes::{AddRemoveSetStore, HeadStore},
     ipfs::Block,
     kv::to_block_raw,
+    storage::ImmutableStore,
 };
 use anyhow::Result;
 use async_recursion::async_recursion;
@@ -38,7 +39,7 @@ pub struct Store<B> {
     invocation_heads: HeadStore,
 }
 
-impl Store<B> {
+impl<B> Store<B> {
     pub async fn new(oid: &OrbitId, blocks: B, config: config::IndexStorage) -> Result<Self> {
         let id = oid
             .clone()
diff --git a/src/kv/mod.rs b/src/kv/mod.rs
index 152095ff..011b04f8 100644
--- a/src/kv/mod.rs
+++ b/src/kv/mod.rs
@@ -5,7 +5,6 @@ use kepler_lib::libipld::{
 use libp2p::core::PeerId;
 use rocket::futures::{Stream, StreamExt};
 use serde::{Deserialize, Serialize};
-use std::sync::Arc;
 
 pub mod behaviour;
 mod entries;
@@ -23,7 +22,7 @@ pub struct Service<B> {
     pub store: Store<B>,
 }
 
-impl Service<B> {
+impl<B> Service<B> {
     fn new(store: Store<B>) -> Self {
         Self { store }
     }
diff --git a/src/kv/store.rs b/src/kv/store.rs
index 4c54a584..d7c6b2ef 100644
--- a/src/kv/store.rs
+++ b/src/kv/store.rs
@@ -1,5 +1,4 @@
 use crate::indexes::{AddRemoveSetStore, HeadStore};
-use crate::kv::entries::{read_from_store, write_to_store};
 use crate::kv::{Object, ObjectBuilder, Service};
 use crate::storage::ImmutableStore;
 use anyhow::Result;
@@ -10,7 +9,7 @@ use rocket::{futures::future::try_join_all, tokio::io::AsyncRead};
 use std::{collections::BTreeMap, convert::TryFrom};
 use tracing::{debug, instrument};
 
-use super::{to_block, Block, KVMessage, ObjectReader};
+use super::{to_block, Block, KVMessage};
 use crate::config;
 
 #[derive(DagCbor)]
@@ -106,7 +105,7 @@ pub struct Store<B> {
     heads: HeadStore,
 }
 
-impl Store<B> {
+impl<B> Store<B> {
     pub async fn new(orbit_id: Cid, blocks: B, config: config::IndexStorage) -> Result<Self> {
         let index = AddRemoveSetStore::new(orbit_id, "kv".to_string(), config.clone()).await?;
         // heads tracking store
@@ -148,7 +147,7 @@ impl Store<B> {
     }
 }
 
-impl Store<B>
+impl<B> Store<B>
 where
     B: ImmutableStore,
 {
diff --git a/src/lib.rs b/src/lib.rs
index 650ed9e7..1a355f68 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -14,7 +14,6 @@ pub mod auth_guards;
 pub mod authorization;
 pub mod capabilities;
 pub mod cas;
-pub mod codec;
 pub mod config;
 pub mod indexes;
 pub mod ipfs;

From 2a18ad953d64d018cea634bcdcb9a7e3e0fd9c1f Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Mon, 24 Oct 2022 14:21:36 +0200
Subject: [PATCH 13/73] hollow out relay.rs

---
 src/relay.rs | 72 +++++++++++++++++++++++++---------------------------
 1 file changed, 34 insertions(+), 38 deletions(-)

diff --git a/src/relay.rs b/src/relay.rs
index b2161bea..935259af 100644
--- a/src/relay.rs
+++ b/src/relay.rs
@@ -1,5 +1,4 @@
 use anyhow::Result;
-use ipfs::{p2p::transport::TransportBuilder, Ipfs, IpfsOptions, TestTypes, UninitializedIpfs};
 use libp2p::core::{
     identity::Keypair, multiaddr::multiaddr, transport::MemoryTransport, Multiaddr, PeerId,
 };
@@ -10,50 +9,47 @@ use crate::orbit::AbortOnDrop;
 pub struct RelayNode {
     pub port: u16,
     pub id: PeerId,
-    _task: AbortOnDrop<()>,
-    _ipfs: Ipfs<TestTypes>,
 }
 
 impl RelayNode {
     pub async fn new(port: u16, keypair: Keypair) -> Result<Self> {
-        let local_public_key = keypair.public();
-        let id = local_public_key.to_peer_id();
-        let relay_tcp_addr = Self::_external(port);
-        let relay_mem_addr = Self::_internal(port);
-
-        let (transport_builder, relay_behaviour) = TransportBuilder::new(keypair.clone())?
-            .or(MemoryTransport::default())
-            .relay();
-
-        let ipfs_opts = IpfsOptions {
-            ipfs_path: std::env::temp_dir(),
-            keypair,
-            bootstrap: vec![],
-            mdns: false,
-            kad_protocol: "/kepler/relay".to_string().into(),
-            listening_addrs: vec![relay_tcp_addr, relay_mem_addr],
-            span: None,
-        };
-
-        // TestTypes designates an in-memory Ipfs instance, but this peer won't store data anyway.
-        let (_ipfs, ipfs_task) =
-            UninitializedIpfs::new(ipfs_opts, transport_builder.build(), Some(relay_behaviour))
-                .start()
-                .await?;
-
-        tracing::debug!(
-            "opened relay: {} at {}, {}",
-            id,
-            Self::_internal(port),
-            Self::_external(port),
-        );
-
-        let task = spawn(ipfs_task);
+        // let local_public_key = keypair.public();
+        // let id = local_public_key.to_peer_id();
+        // let relay_tcp_addr = Self::_external(port);
+        // let relay_mem_addr = Self::_internal(port);
+
+        // let (transport_builder, relay_behaviour) = TransportBuilder::new(keypair.clone())?
+        //     .or(MemoryTransport::default())
+        //     .relay();
+
+        // let ipfs_opts = IpfsOptions {
+        //     ipfs_path: std::env::temp_dir(),
+        //     keypair,
+        //     bootstrap: vec![],
+        //     mdns: false,
+        //     kad_protocol: "/kepler/relay".to_string().into(),
+        //     listening_addrs: vec![relay_tcp_addr, relay_mem_addr],
+        //     span: None,
+        // };
+
+        // // TestTypes designates an in-memory Ipfs instance, but this peer won't store data anyway.
+        // let (_ipfs, ipfs_task) =
+        //     UninitializedIpfs::new(ipfs_opts, transport_builder.build(), Some(relay_behaviour))
+        //         .start()
+        //         .await?;
+
+        // tracing::debug!(
+        //     "opened relay: {} at {}, {}",
+        //     id,
+        //     Self::_internal(port),
+        //     Self::_external(port),
+        // );
+
+        // let task = spawn(ipfs_task);
         Ok(Self {
             port,
-            _task: AbortOnDrop::new(task),
+            // _task: AbortOnDrop::new(task),
             id,
-            _ipfs,
         })
     }
 

From 53bcce764c622e3c9557beb89a702be2e58617d4 Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Mon, 24 Oct 2022 14:26:08 +0200
Subject: [PATCH 14/73] remove src/ipfs.rs

---
 src/authorization.rs      |  2 +-
 src/capabilities/store.rs |  2 +-
 src/ipfs.rs               | 20 --------------------
 src/kv/entries.rs         |  6 +++---
 src/kv/mod.rs             |  2 +-
 src/lib.rs                |  4 +++-
 6 files changed, 9 insertions(+), 27 deletions(-)
 delete mode 100644 src/ipfs.rs

diff --git a/src/authorization.rs b/src/authorization.rs
index 8663568a..5eaa647e 100644
--- a/src/authorization.rs
+++ b/src/authorization.rs
@@ -1,6 +1,6 @@
 use crate::{
     capabilities::store::{FromBlock, ToBlock},
-    ipfs::Block,
+    Block,
 };
 use anyhow::Result;
 use kepler_lib::libipld::Cid;
diff --git a/src/capabilities/store.rs b/src/capabilities/store.rs
index 0dc4f21d..9474d331 100644
--- a/src/capabilities/store.rs
+++ b/src/capabilities/store.rs
@@ -1,9 +1,9 @@
 use crate::{
     authorization::{CapStore, Delegation, Invocation, Revocation, Verifiable},
     indexes::{AddRemoveSetStore, HeadStore},
-    ipfs::Block,
     kv::to_block_raw,
     storage::ImmutableStore,
+    Block,
 };
 use anyhow::Result;
 use async_recursion::async_recursion;
diff --git a/src/ipfs.rs b/src/ipfs.rs
deleted file mode 100644
index cb9656bc..00000000
--- a/src/ipfs.rs
+++ /dev/null
@@ -1,20 +0,0 @@
-use anyhow::Result;
-use kepler_lib::libipld::{
-    block::Block as OBlock,
-    cid::{multibase::Base, Cid},
-    multihash::Code,
-    raw::RawCodec,
-    store::DefaultParams,
-};
-use libp2p::{core::transport::MemoryTransport, futures::TryStreamExt, swarm::Swarm as TSwarm};
-use std::{future::Future, sync::mpsc::Receiver};
-
-use crate::{
-    config,
-    kv::behaviour::{Behaviour, Event as BehaviourEvent},
-    storage::{Repo, StorageUtils},
-};
-
-pub type KeplerParams = DefaultParams;
-pub type Block = OBlock<KeplerParams>;
-pub type Swarm = TSwarm<Types>;
diff --git a/src/kv/entries.rs b/src/kv/entries.rs
index 9cb7d1dc..00241441 100644
--- a/src/kv/entries.rs
+++ b/src/kv/entries.rs
@@ -1,7 +1,7 @@
 use super::{to_block, to_block_raw};
-use crate::ipfs::{Block, KeplerParams};
+use crate::Block;
 use anyhow::Result;
-use kepler_lib::libipld::{cid::Cid, store::StoreParams, DagCbor};
+use kepler_lib::libipld::{cid::Cid, DagCbor};
 use libp2p::futures::stream::BoxStream;
 use std::{
     collections::BTreeMap,
@@ -75,7 +75,7 @@ mod test {
     async fn write() -> Result<(), anyhow::Error> {
         tracing_try_init(&config::Logging::default());
         let tmp = tempdir::TempDir::new("test_streams")?;
-        let data = vec![3u8; KeplerParams::MAX_BLOCK_SIZE * 3];
+        let data = vec![3u8; 1000000000 * 3];
 
         let mut config = IpfsOptions::inmemory_with_generated_keys();
         config.ipfs_path = tmp.path().into();
diff --git a/src/kv/mod.rs b/src/kv/mod.rs
index 011b04f8..5049fb41 100644
--- a/src/kv/mod.rs
+++ b/src/kv/mod.rs
@@ -10,7 +10,7 @@ pub mod behaviour;
 mod entries;
 mod store;
 
-use super::{ipfs::Block, orbit::AbortOnDrop};
+use super::{orbit::AbortOnDrop, Block};
 
 pub use entries::{Object, ObjectBuilder};
 pub use store::Store;
diff --git a/src/lib.rs b/src/lib.rs
index 1a355f68..f8c35056 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -7,6 +7,7 @@ extern crate anyhow;
 extern crate tokio;
 
 use anyhow::Result;
+use kepler_lib::libipld::{block::Block as OBlock, store::DefaultParams};
 use rocket::{fairing::AdHoc, figment::Figment, http::Header, Build, Rocket};
 
 pub mod allow_list;
@@ -16,7 +17,6 @@ pub mod capabilities;
 pub mod cas;
 pub mod config;
 pub mod indexes;
-pub mod ipfs;
 pub mod kv;
 pub mod manifest;
 pub mod orbit;
@@ -35,6 +35,8 @@ use relay::RelayNode;
 use routes::{delegate, invoke, open_host_key, relay_addr, util_routes::*};
 use std::{collections::HashMap, sync::RwLock};
 
+pub type Block = OBlock<DefaultParams>;
+
 pub async fn app(config: &Figment) -> Result<Rocket<Build>> {
     let kepler_config = config.extract::<config::Config>()?;
 

From 6c644fc376ad8a94b7552e93e9426be992a5f902 Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Wed, 26 Oct 2022 16:30:11 +0200
Subject: [PATCH 15/73] fix siwe and ssi versions

---
 lib/Cargo.toml        | 20 ++++++++++----------
 sdk/Cargo.toml        |  4 ++--
 sdk/src/session.rs    |  6 +++---
 sdk/src/siwe_utils.rs |  2 +-
 4 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/lib/Cargo.toml b/lib/Cargo.toml
index d13d3a3c..a73a96dd 100644
--- a/lib/Cargo.toml
+++ b/lib/Cargo.toml
@@ -13,16 +13,16 @@ serde_with = "1.0"
 thiserror = "1"
 base64 = "0.13"
 cacaos = "0.5"
-capgrok = "0.3"
+capgrok = "=0.3.0"
 lazy_static = "1.4"
-did-method-key = { default-features = false, git = "https://github.com/spruceid/ssi", branch = "main" }
-did-tz = { default-features = false, git = "https://github.com/spruceid/ssi", branch = "main" }
-did-ethr = { default-features = false, git = "https://github.com/spruceid/ssi", branch = "main" }
-did-pkh = { default-features = false, git = "https://github.com/spruceid/ssi", branch = "main" }
-did-web = { default-features = false, git = "https://github.com/spruceid/ssi", branch = "main" }
-did-webkey = { default-features = false, git = "https://github.com/spruceid/ssi", branch = "main" }
-did-onion = { default-features = false, git = "https://github.com/spruceid/ssi", branch = "main" }
-did-ion = { default-features = false, git = "https://github.com/spruceid/ssi", branch = "main" }
+did-method-key = { default-features = false, git = "https://github.com/spruceid/ssi", branch = "feat/ucan-ipld-version" }
+did-tz = { default-features = false, git = "https://github.com/spruceid/ssi", branch = "feat/ucan-ipld-version" }
+did-ethr = { default-features = false, git = "https://github.com/spruceid/ssi", branch = "feat/ucan-ipld-version" }
+did-pkh = { default-features = false, git = "https://github.com/spruceid/ssi", branch = "feat/ucan-ipld-version" }
+did-web = { default-features = false, git = "https://github.com/spruceid/ssi", branch = "feat/ucan-ipld-version" }
+did-webkey = { default-features = false, git = "https://github.com/spruceid/ssi", branch = "feat/ucan-ipld-version" }
+did-onion = { default-features = false, git = "https://github.com/spruceid/ssi", branch = "feat/ucan-ipld-version" }
+did-ion = { default-features = false, git = "https://github.com/spruceid/ssi", branch = "feat/ucan-ipld-version" }
 
 [target.'cfg(target_arch = "wasm32")'.dependencies]
 uuid = { version = "1", features = ["v4", "js"] }
@@ -32,7 +32,7 @@ uuid = { version = "1", features = ["v4"] }
 
 [dependencies.ssi]
 git = "https://github.com/spruceid/ssi"
-branch = "main"
+branch = "feat/ucan-ipld-version"
 default-features = false
 features = ["ed25519", "rsa"]
 
diff --git a/sdk/Cargo.toml b/sdk/Cargo.toml
index 58b26ae7..a57cc6d7 100644
--- a/sdk/Cargo.toml
+++ b/sdk/Cargo.toml
@@ -5,10 +5,10 @@ edition = "2021"
 
 [dependencies]
 base64 = "0.13"
-chrono = { version = "0.4" }
+time = "0.3"
 hex = "0.4"
 http = "0.2"
-iri-string = "0.4"
+iri-string = "0.6"
 kepler-lib = { path = "../lib" }
 serde = { version = "1.0", features = ["derive"] }
 serde_with = { version = "1" }
diff --git a/sdk/src/session.rs b/sdk/src/session.rs
index 13c3b747..3e3e7a83 100644
--- a/sdk/src/session.rs
+++ b/sdk/src/session.rs
@@ -1,10 +1,9 @@
 use crate::authorization::DelegationHeaders;
-use chrono::Utc;
 use http::uri::Authority;
 use kepler_lib::{
     authorization::{make_invocation, InvocationError as ZcapError, KeplerInvocation},
     cacaos::{
-        siwe::{nonce::generate_nonce, Message, TimeStamp, Version as SIWEVersion},
+        siwe::{generate_nonce, Message, TimeStamp, Version as SIWEVersion},
         siwe_cacao::SIWESignature,
     },
     capgrok::Builder,
@@ -16,6 +15,7 @@ use kepler_lib::{
 use serde::{Deserialize, Serialize};
 use serde_with::{serde_as, DisplayFromStr};
 use std::collections::HashMap;
+use time::OffsetDateTime;
 
 #[serde_as]
 #[derive(Deserialize, Clone)]
@@ -140,7 +140,7 @@ impl Session {
             &self.jwk,
             self.verification_method,
             // 60 seconds in the future
-            (Utc::now().timestamp_nanos() as f64 / 1e+9_f64) + 60.0,
+            (OffsetDateTime::now_utc().nanosecond() as f64 / 1e+9_f64) + 60.0,
             None,
             None,
         )
diff --git a/sdk/src/siwe_utils.rs b/sdk/src/siwe_utils.rs
index ec7ae98a..ec6a8e58 100644
--- a/sdk/src/siwe_utils.rs
+++ b/sdk/src/siwe_utils.rs
@@ -1,7 +1,7 @@
 use http::uri::Authority;
 use kepler_lib::authorization::KeplerDelegation;
 use kepler_lib::cacaos::{
-    siwe::{nonce::generate_nonce, Message, TimeStamp, Version},
+    siwe::{generate_nonce, Message, TimeStamp, Version},
     siwe_cacao::{SIWESignature, SiweCacao},
 };
 use kepler_lib::capgrok::Builder;

From 0459cf72ba4bf45ecdd7447af3e69905573a6956 Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Wed, 26 Oct 2022 16:46:32 +0200
Subject: [PATCH 16/73] fix up storage trait, stub out impls

---
 src/storage/either.rs      | 103 +++++++++++-------
 src/storage/file_system.rs |  16 ++-
 src/storage/mod.rs         |  58 ++++++++---
 src/storage/s3.rs          | 207 +++++++++++++++----------------------
 4 files changed, 206 insertions(+), 178 deletions(-)

diff --git a/src/storage/either.rs b/src/storage/either.rs
index 489cb862..95ab92ab 100644
--- a/src/storage/either.rs
+++ b/src/storage/either.rs
@@ -6,81 +6,108 @@ use futures::{
 };
 use kepler_lib::libipld::cid::multihash::Multihash;
 
-#[derive(Debug)]
-pub enum EitherStore<L, R> {
-    Left(L),
-    Right(R),
+#[derive(Debug, Clone)]
+pub enum EitherStore<A, B> {
+    A(A),
+    B(B),
 }
 
-#[derive(Debug)]
-pub enum AsyncReadEither<L, R>
+#[derive(Debug, Clone)]
+pub enum AsyncReadEither<A, B>
 where
-    L: ImmutableStore,
-    R: ImmutableStore,
+    A: ImmutableStore,
+    B: ImmutableStore,
 {
-    Left(L::Readable),
-    Right(R::Readable),
+    A(A::Readable),
+    B(B::Readable),
 }
 
-impl<L, R> AsyncRead for AsyncReadEither<L, R>
+impl<A, B> AsyncRead for AsyncReadEither<A, B>
 where
-    L: ImmutableStore,
-    R: ImmutableStore,
+    A: ImmutableStore,
+    B: ImmutableStore,
 {
+    #[inline]
     fn poll_read(
         self: Pin<&mut Self>,
         cx: &mut Context<'_>,
         buf: &mut [u8],
     ) -> Poll<Result<usize, Error>> {
-        match self {
-            Self::L(l) => l.poll_read(cx, buf),
-            Self::R(r) => r.poll_read(cx, buf),
+        // it actually seems like this is only possible with unsafe :(
+        // TODO use pin-project crate
+        unsafe {
+            match self.get_unchecked_mut() {
+                Self::A(l) => AsyncRead::poll_read(Pin::new_unchecked(l), cx, buf),
+                Self::B(r) => AsyncRead::poll_read(Pin::new_unchecked(r), cx, buf),
+            }
+        }
+    }
+    #[inline]
+    fn poll_read_vectored(
+        self: Pin<&mut Self>,
+        cx: &mut Context<'_>,
+        bufs: &mut [std::io::IoSliceMut<'_>],
+    ) -> Poll<Result<usize, Error>> {
+        unsafe {
+            match self.get_unchecked_mut() {
+                Self::A(l) => AsyncRead::poll_read_vectored(Pin::new_unchecked(l), cx, bufs),
+                Self::B(r) => AsyncRead::poll_read_vectored(Pin::new_unchecked(r), cx, bufs),
+            }
         }
     }
 }
 
-#[derive(thiserror::Error)]
-pub enum EitherStoreError<L, R>
+#[derive(thiserror::Error, Debug)]
+pub enum EitherStoreError<A, B>
 where
-    L: ImmutableStore,
-    R: ImmutableStore,
+    A: ImmutableStore,
+    B: ImmutableStore,
 {
-    #[error(transparent)]
-    Left(L::Error),
-    #[error(transparent)]
-    Right(R::Error),
+    A(A::Error),
+    B(B::Error),
 }
 
 #[async_trait]
-impl<L, R> ImmutableStore for EitherStore<L, R>
+impl<A, B> ImmutableStore for EitherStore<A, B>
 where
-    L: ImmutableStore,
-    R: ImmutableStore,
+    A: ImmutableStore + Sync,
+    B: ImmutableStore + Sync,
 {
-    type Readable = AsyncReadEither<L, R>;
-    type Error = EitherStoreError<L, R>;
+    type Readable = AsyncReadEither<A, B>;
+    type Error = EitherStoreError<A, B>;
     async fn contains(&self, id: &Multihash) -> Result<bool, Self::Error> {
         match self {
-            Self::Left(l) => l.contains(id).await.map_err(Self::Error::Left),
-            Self::Right(r) => r.contains(id).await.map_err(Self::Error::Right),
+            Self::A(l) => l.contains(id).await.map_err(Self::Error::A),
+            Self::B(r) => r.contains(id).await.map_err(Self::Error::B),
         }
     }
-    async fn write(&self, data: impl futures::io::AsyncRead) -> Result<Multihash, Self::Error> {
+    async fn write(
+        &self,
+        data: impl futures::io::AsyncRead + Send,
+    ) -> Result<Multihash, Self::Error> {
         match self {
-            Self::Left(l) => l.write(data).await.map_err(Self::Error::Left),
-            Self::Right(r) => r.write(data).await.map_err(Self::Error::Right),
+            Self::A(l) => l.write(data).await.map_err(Self::Error::A),
+            Self::B(r) => r.write(data).await.map_err(Self::Error::B),
         }
     }
     async fn remove(&self, id: &Multihash) -> Result<Option<()>, Self::Error> {
         match self {
-            Self::Left(l) => l.remove(id).await.map_err(Self::Error::Left),
-            Self::Right(r) => r.remove(id).await.map_err(Self::Error::Right),
+            Self::A(l) => l.remove(id).await.map_err(Self::Error::A),
+            Self::B(r) => r.remove(id).await.map_err(Self::Error::B),
         }
     }
     async fn read(&self, id: &Multihash) -> Result<Option<Self::Readable>, Self::Error> {
         match self {
-            Self::Left(l) => l.read(id).await.map_err(Self::Error::Left),
-            Self::Right(r) => r.read(id).await.map_err(Self::Error::Right),
+            Self::A(l) => l
+                .read(id)
+                .await
+                .map(|o| o.map(Self::Readable::A))
+                .map_err(Self::Error::A),
+            Self::B(r) => r
+                .read(id)
+                .await
+                .map(|o| o.map(Self::Readable::B))
+                .map_err(Self::Error::B),
         }
     }
 }
diff --git a/src/storage/file_system.rs b/src/storage/file_system.rs
index 2508aac2..2dd53ede 100644
--- a/src/storage/file_system.rs
+++ b/src/storage/file_system.rs
@@ -8,7 +8,9 @@ use tokio::{
     fs::{remove_file, File},
     io::copy,
 };
+use tokio_util::compat::{Compat, TokioAsyncReadCompatExt};
 
+#[derive(Debug)]
 pub struct FileSystemStore {
     path: PathBuf,
 }
@@ -19,15 +21,21 @@ impl FileSystemStore {
     }
 
     fn get_path(&self, mh: &Multihash) -> PathBuf {
-        self.path.join(encode(Base::Base64Url, mh))
+        self.path.join(encode(Base::Base64Url, mh.to_bytes()))
     }
 }
 
 #[async_trait]
 impl ImmutableStore for FileSystemStore {
     type Error = std::io::Error;
-    type Readable = File;
-    async fn write(&self, data: impl futures::io::AsyncRead) -> Result<Multihash, Self::Error> {
+    type Readable = Compat<File>;
+    async fn contains(&self, id: &Multihash) -> Result<bool, Self::Error> {
+        Ok(self.get_path(id).exists())
+    }
+    async fn write(
+        &self,
+        data: impl futures::io::AsyncRead + Send,
+    ) -> Result<Multihash, Self::Error> {
         todo!();
         // write into tmp then rename, to name after the hash
         // need to stream data through a hasher into the file and return hash
@@ -46,7 +54,7 @@ impl ImmutableStore for FileSystemStore {
     }
     async fn read(&self, id: &Multihash) -> Result<Option<Self::Readable>, Self::Error> {
         match File::open(self.get_path(id)).await {
-            Ok(f) => Ok(Some(f)),
+            Ok(f) => Ok(Some(f.compat())),
             Err(e) if e.kind() == ErrorKind::NotFound => Ok(None),
             Err(e) => Err(e),
         }
diff --git a/src/storage/mod.rs b/src/storage/mod.rs
index 2f45759c..285de96b 100644
--- a/src/storage/mod.rs
+++ b/src/storage/mod.rs
@@ -47,7 +47,8 @@ impl StorageUtils {
         match &self.config {
             config::BlockStorage::S3(r) => {
                 let client = s3::S3BlockStore::new_(r.clone(), Cid::default());
-                client.init().await
+                // client.init().await
+                Ok(())
             }
             config::BlockStorage::Local(r) => {
                 if !r.path.is_dir() {
@@ -240,7 +241,7 @@ pub enum VecReadError<E> {
     #[error(transparent)]
     Store(#[from] E),
     #[error(transparent)]
-    Read(#[from] futures::io::Error),
+    Read(futures::io::Error),
 }
 
 #[async_trait]
@@ -248,21 +249,30 @@ pub trait ImmutableStore {
     type Error;
     type Readable: futures::io::AsyncRead;
     async fn contains(&self, id: &Multihash) -> Result<bool, Self::Error>;
-    async fn write(&self, data: impl futures::io::AsyncRead) -> Result<Multihash, Self::Error>;
+    async fn write(
+        &self,
+        data: impl futures::io::AsyncRead + Send,
+    ) -> Result<Multihash, Self::Error>;
     async fn remove(&self, id: &Multihash) -> Result<Option<()>, Self::Error>;
     async fn read(&self, id: &Multihash) -> Result<Option<Self::Readable>, Self::Error>;
     async fn read_to_vec(
         &self,
         id: &Multihash,
-    ) -> Result<Option<Vec<u8>>, VecReadError<Self::Error>> {
-        Ok(match self.read(id).await? {
-            Some(r) => {
-                let mut v = Vec::new();
-                r.read_all(&mut v).await?;
-                Some(v)
-            }
-            _ => None,
-        })
+    ) -> Result<Option<Vec<u8>>, VecReadError<Self::Error>>
+    where
+        Self::Readable: Send,
+    {
+        use futures::io::AsyncReadExt;
+        let r = match self.read(id).await? {
+            None => return Ok(None),
+            Some(r) => r,
+        };
+        let mut v = Vec::new();
+        Box::pin(r)
+            .read_to_end(&mut v)
+            .await
+            .map_err(VecReadError::Read)?;
+        Ok(Some(v))
     }
 }
 
@@ -272,6 +282,30 @@ trait StoreSeek: ImmutableStore {
     async fn seek(&self, id: &Cid) -> Result<Option<Self::Seekable>, Self::Error>;
 }
 
+#[async_trait]
+impl<S> ImmutableStore for Box<S>
+where
+    S: ImmutableStore + Send + Sync,
+{
+    type Error = S::Error;
+    type Readable = S::Readable;
+    async fn contains(&self, id: &Multihash) -> Result<bool, Self::Error> {
+        self.contains(id).await
+    }
+    async fn write(
+        &self,
+        data: impl futures::io::AsyncRead + Send,
+    ) -> Result<Multihash, Self::Error> {
+        self.write(data).await
+    }
+    async fn remove(&self, id: &Multihash) -> Result<Option<()>, Self::Error> {
+        self.remove(id).await
+    }
+    async fn read(&self, id: &Multihash) -> Result<Option<Self::Readable>, Self::Error> {
+        self.read(id).await
+    }
+}
+
 #[async_trait]
 trait IdempotentHeightGroup {
     // write a height value for a Cid
diff --git a/src/storage/s3.rs b/src/storage/s3.rs
index 23c633fd..cdc2841f 100644
--- a/src/storage/s3.rs
+++ b/src/storage/s3.rs
@@ -1,25 +1,35 @@
-use anyhow::{Context, Error};
+use anyhow::Error;
 use aws_sdk_s3::{
     error::{GetObjectError, GetObjectErrorKind},
     types::{ByteStream, SdkError},
     Client, // Config,
+    Error as S3Error,
 };
-use aws_smithy_http::{body::SdkBody, endpoint::Endpoint};
-// use aws_types::{credentials::Credentials, region::Region};
-use futures::stream::{StreamExt, TryStreamExt};
-use ipfs::{
-    repo::{
-        BlockPut, BlockRm, BlockRmError, BlockStore, Column, DataStore, PinKind, PinMode, PinStore,
-    },
-    Block,
+use aws_smithy_http::{body::SdkBody, byte_stream::Error as ByteStreamError, endpoint::Endpoint};
+use futures::stream::{IntoAsyncRead, MapErr, TryStreamExt};
+use kepler_lib::libipld::cid::{
+    multibase::{encode, Base},
+    multihash::Multihash,
+    Cid,
 };
-use kepler_lib::libipld::cid::{multibase::Base, Cid};
 use regex::Regex;
 use rocket::{async_trait, http::hyper::Uri};
-use std::{path::PathBuf, str::FromStr};
+use std::{io::Error as IoError, path::PathBuf, str::FromStr};
 
-use super::dynamodb::{DynamoPinStore, References};
-use crate::config;
+use crate::{
+    config,
+    storage::{dynamodb::DynamoPinStore, ImmutableStore},
+};
+
+#[derive(Debug)]
+pub struct S3DataStore {
+    // TODO Remove is unused (orbit::delete is never called).
+    // When that changes we will need to use a mutex, either local or in Dynamo
+    pub client: Client,
+    pub bucket: String,
+    pub dynamodb: DynamoPinStore,
+    pub orbit: Cid,
+}
 
 // TODO we could use the same struct for both the block store and the data
 // (pin) store, but we need to remember that for now it will be two different
@@ -30,7 +40,7 @@ pub struct S3BlockStore {
     // When that changes we will need to use a mutex, either local or in Dynamo
     pub client: Client,
     pub bucket: String,
-    pub orbit: Cid,
+    pub orbit: String,
 }
 
 pub fn new_client(config: config::S3BlockStorage) -> Client {
@@ -49,6 +59,7 @@ impl S3DataStore {
         S3DataStore {
             client: new_client(config.clone()),
             bucket: config.bucket,
+            dynamodb: DynamoPinStore::new(config.dynamodb, orbit),
             orbit,
         }
     }
@@ -94,88 +105,65 @@ impl S3DataStore {
         Ok(())
     }
 }
+
 impl S3BlockStore {
     pub fn new_(config: config::S3BlockStorage, orbit: Cid) -> Self {
         S3BlockStore {
             client: new_client(config.clone()),
             bucket: config.bucket,
-            orbit,
+            orbit: orbit.to_string(),
         }
     }
 }
 
-fn path_to_config(path: PathBuf) -> (config::S3BlockStorage, Cid) {
-    let re =
-            Regex::new(r"^/s3bucket/(?P<bucket>.*)/s3endpoint/(?P<s3endpoint>.*)/dynamotable/(?P<table>.*)/dynamoendpoint/(?P<dynamoendpoint>.*)/orbitcid/(?P<orbit>.*)/(blockstore|datastore)$")
-                .unwrap();
-    let fields = re.captures(path.to_str().unwrap()).unwrap();
-    let s3_bucket = fields.name("bucket").unwrap().as_str().to_string();
-    let s3_endpoint = Some(fields.name("s3endpoint").unwrap().as_str())
-        .filter(|s| !s.is_empty())
-        .map(|e| Uri::from_str(e).unwrap());
-    let dynamo_table = fields.name("table").unwrap().as_str().to_string();
-    let dynamo_endpoint = Some(fields.name("dynamoendpoint").unwrap().as_str())
-        .filter(|s| !s.is_empty())
-        .map(|e| Uri::from_str(e).unwrap());
-    let orbit = Cid::from_str(fields.name("orbit").unwrap().as_str()).unwrap();
-
-    let config = config::S3BlockStorage {
-        bucket: s3_bucket,
-        endpoint: s3_endpoint,
-        dynamodb: config::DynamoStorage {
-            table: dynamo_table,
-            endpoint: dynamo_endpoint,
-        },
-    };
-    (config, orbit)
+pub fn convert(e: ByteStreamError) -> IoError {
+    e.into()
 }
 
 #[async_trait]
-impl BlockStore for S3BlockStore {
-    fn new(path: PathBuf) -> Self {
-        let (config, orbit) = path_to_config(path);
-        S3BlockStore::new_(config, orbit)
-    }
-
-    async fn init(&self) -> Result<(), Error> {
-        self.dynamodb
-            .healthcheck()
-            .await
-            .context("Failed healthcheck for DynamoDB")?;
-        self.client
-            .head_bucket()
-            .bucket(self.bucket.clone())
-            .send()
-            .await
-            .context("Failed healthcheck for S3")?;
-        Ok(())
+impl ImmutableStore for S3BlockStore {
+    type Error = S3Error;
+    type Readable = IntoAsyncRead<MapErr<ByteStream, fn(ByteStreamError) -> IoError>>;
+    async fn contains(&self, id: &Multihash) -> Result<bool, Self::Error> {
+        todo!()
     }
-
-    async fn open(&self) -> Result<(), Error> {
-        Ok(())
+    async fn write(
+        &self,
+        data: impl futures::io::AsyncRead + Send,
+    ) -> Result<Multihash, Self::Error> {
+        // write to a dummy ID (in fs or in s3)
+        // get the hash
+        // write or copy to correct ID (hash)
+        todo!();
+        // write into tmp then rename, to name after the hash
+        // need to stream data through a hasher into the file and return hash
+        // match File::open(path.join(cid.to_string())),await {
+        //     Ok(f) => copy(data, file).await
+        //     Err(e) if error.kind() == IoErrorKind::NotFound => Ok(None),
+        //     Err(e) => Err(e),
+        // }
     }
-
-    async fn contains(&self, cid: &Cid) -> Result<bool, Error> {
-        self.dynamodb.is_pinned(cid).await
+    async fn remove(&self, id: &Multihash) -> Result<Option<()>, Self::Error> {
+        todo!()
     }
-
-    async fn get(&self, cid: &Cid) -> Result<Option<Block>, Error> {
+    async fn read(&self, id: &Multihash) -> Result<Option<Self::Readable>, Self::Error> {
         let res = self
             .client
             .get_object()
             .bucket(self.bucket.clone())
             .key(format!(
                 "{}/{}",
-                self.orbit.to_string_of_base(Base::Base58Btc)?,
-                cid
+                self.orbit,
+                encode(Base::Base64Url, &id.to_bytes())
             ))
             .send()
             .await;
         match res {
-            Ok(o) => Ok(Some(Block::new(
-                *cid,
-                o.body.collect().await?.into_bytes().to_vec(),
-            )?)),
+            Ok(o) => Ok(Some(
+                o.body
+                    .map_err(convert as fn(ByteStreamError) -> IoError)
+                    .into_async_read(),
+            )),
             Err(SdkError::ServiceError {
                 err:
                     GetObjectError {
@@ -187,59 +175,30 @@ impl BlockStore for S3BlockStore {
             Err(e) => Err(e.into()),
         }
     }
+}
 
-    async fn put(&self, block: Block) -> Result<(Cid, BlockPut), Error> {
-        let res = self
-            .client
-            .put_object()
-            .bucket(self.bucket.clone())
-            .key(format!(
-                "{}/{}",
-                self.orbit.to_string_of_base(Base::Base58Btc)?,
-                block.cid()
-            ))
-            .body(ByteStream::new(SdkBody::from(block.data())))
-            .send()
-            .await;
-
-        match res {
-            // can't tell if the object already existed
-            Ok(_) => Ok((*block.cid(), BlockPut::NewBlock)),
-            Err(e) => Err(e.into()),
-        }
-    }
-
-    async fn remove(&self, cid: &Cid) -> Result<Result<BlockRm, BlockRmError>, Error> {
-        // TODO when is that called, should the pin store call this?
-        let res = self
-            .client
-            .delete_object()
-            .bucket(self.bucket.clone())
-            .key(format!(
-                "{}/{}",
-                self.orbit.to_string_of_base(Base::Base58Btc)?,
-                cid
-            ))
-            .send()
-            .await;
-
-        match res {
-            // Cannot tell if the object existed in the first place.
-            Ok(_) => Ok(Ok(BlockRm::Removed(*cid))),
-            Err(e) => Err(e.into()),
-        }
-    }
-
-    async fn list(&self) -> Result<Vec<Cid>, Error> {
-        self.dynamodb
-            .list(None)
-            .await
-            .map(|r| r.map(|rr| rr.0))
-            .try_collect()
-            .await
-    }
+fn path_to_config(path: PathBuf) -> (config::S3BlockStorage, Cid) {
+    let re =
+            Regex::new(r"^/s3bucket/(?P<bucket>.*)/s3endpoint/(?P<s3endpoint>.*)/dynamotable/(?P<table>.*)/dynamoendpoint/(?P<dynamoendpoint>.*)/orbitcid/(?P<orbit>.*)/(blockstore|datastore)$")
+                .unwrap();
+    let fields = re.captures(path.to_str().unwrap()).unwrap();
+    let s3_bucket = fields.name("bucket").unwrap().as_str().to_string();
+    let s3_endpoint = Some(fields.name("s3endpoint").unwrap().as_str())
+        .filter(|s| !s.is_empty())
+        .map(|e| Uri::from_str(e).unwrap());
+    let dynamo_table = fields.name("table").unwrap().as_str().to_string();
+    let dynamo_endpoint = Some(fields.name("dynamoendpoint").unwrap().as_str())
+        .filter(|s| !s.is_empty())
+        .map(|e| Uri::from_str(e).unwrap());
+    let orbit = Cid::from_str(fields.name("orbit").unwrap().as_str()).unwrap();
 
-    async fn wipe(&self) {
-        unimplemented!("wipe")
-    }
+    let config = config::S3BlockStorage {
+        bucket: s3_bucket,
+        endpoint: s3_endpoint,
+        dynamodb: config::DynamoStorage {
+            table: dynamo_table,
+            endpoint: dynamo_endpoint,
+        },
+    };
+    (config, orbit)
 }

From 960700794d195eb2058c1ad2a28ced0bea26177a Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Wed, 26 Oct 2022 17:57:35 +0200
Subject: [PATCH 17/73] add clone and stderror for stores

---
 src/storage/either.rs      | 18 ++++++++++++------
 src/storage/file_system.rs |  4 +++-
 src/storage/mod.rs         |  2 +-
 src/storage/s3.rs          |  2 +-
 4 files changed, 17 insertions(+), 9 deletions(-)

diff --git a/src/storage/either.rs b/src/storage/either.rs
index 95ab92ab..3f82d6ac 100644
--- a/src/storage/either.rs
+++ b/src/storage/either.rs
@@ -7,7 +7,11 @@ use futures::{
 use kepler_lib::libipld::cid::multihash::Multihash;
 
 #[derive(Debug, Clone)]
-pub enum EitherStore<A, B> {
+pub enum EitherStore<A, B>
+where
+    A: ImmutableStore + Sync,
+    B: ImmutableStore + Sync,
+{
     A(A),
     B(B),
 }
@@ -60,11 +64,13 @@ where
 #[derive(thiserror::Error, Debug)]
 pub enum EitherStoreError<A, B>
 where
-    A: ImmutableStore,
-    B: ImmutableStore,
+    A: std::error::Error,
+    B: std::error::Error,
 {
-    A(A::Error),
-    B(B::Error),
+    #[error(transparent)]
+    A(A),
+    #[error(transparent)]
+    B(B),
 }
 
 #[async_trait]
@@ -74,7 +80,7 @@ where
     B: ImmutableStore + Sync,
 {
     type Readable = AsyncReadEither<A, B>;
-    type Error = EitherStoreError<A, B>;
+    type Error = EitherStoreError<A::Error, B::Error>;
     async fn contains(&self, id: &Multihash) -> Result<bool, Self::Error> {
         match self {
             Self::A(l) => l.contains(id).await.map_err(Self::Error::A),
diff --git a/src/storage/file_system.rs b/src/storage/file_system.rs
index 2dd53ede..731bdcf2 100644
--- a/src/storage/file_system.rs
+++ b/src/storage/file_system.rs
@@ -10,7 +10,7 @@ use tokio::{
 };
 use tokio_util::compat::{Compat, TokioAsyncReadCompatExt};
 
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub struct FileSystemStore {
     path: PathBuf,
 }
@@ -36,6 +36,8 @@ impl ImmutableStore for FileSystemStore {
         &self,
         data: impl futures::io::AsyncRead + Send,
     ) -> Result<Multihash, Self::Error> {
+        // TODO lock file to prevent overlapping writes
+        // only open to write if not existing AND not being written to right now
         todo!();
         // write into tmp then rename, to name after the hash
         // need to stream data through a hasher into the file and return hash
diff --git a/src/storage/mod.rs b/src/storage/mod.rs
index 285de96b..01ccc402 100644
--- a/src/storage/mod.rs
+++ b/src/storage/mod.rs
@@ -246,7 +246,7 @@ pub enum VecReadError<E> {
 
 #[async_trait]
 pub trait ImmutableStore {
-    type Error;
+    type Error: std::error::Error;
     type Readable: futures::io::AsyncRead;
     async fn contains(&self, id: &Multihash) -> Result<bool, Self::Error>;
     async fn write(
diff --git a/src/storage/s3.rs b/src/storage/s3.rs
index cdc2841f..742310f3 100644
--- a/src/storage/s3.rs
+++ b/src/storage/s3.rs
@@ -34,7 +34,7 @@ pub struct S3DataStore {
 // TODO we could use the same struct for both the block store and the data
 // (pin) store, but we need to remember that for now it will be two different
 // objects in rust-ipfs
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub struct S3BlockStore {
     // TODO Remove is unused (orbit::delete is never called).
     // When that changes we will need to use a mutex, either local or in Dynamo

From e2e90ddf8ac74bd025ace8d748f47d3a656a2661 Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Wed, 26 Oct 2022 20:17:11 +0200
Subject: [PATCH 18/73] bump aws and tokio-util versions

---
 Cargo.toml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index 0a8704b9..3e7ab26d 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -10,11 +10,11 @@ license = "Apache-2.0"
 [dependencies]
 anyhow = "=1.0.59"
 async-recursion = "0.3"
-aws-config = "0.9"
-aws-sdk-dynamodb = "0.9"
-aws-sdk-s3 = "0.9"
-aws-types = "0.9"
-aws-smithy-http = "0.39"
+aws-config = "0.49"
+aws-sdk-dynamodb = "0.19"
+aws-sdk-s3 = "0.19"
+aws-types = "0.49"
+aws-smithy-http = "0.49"
 base64 = "0.13"
 bincode = "1.3"
 bs58 = "0.4"
@@ -40,7 +40,7 @@ thiserror = "1"
 time = "0.3"
 tokio = { version = "1", features = ["rt", "macros", "rt-multi-thread"] }
 tokio-stream = { version = "0.1", features = ["fs"] }
-tokio-util = "0.6.9"
+tokio-util = { version = "0.7", features = ["compat"] }
 tracing = "0.1"
 tracing-log = "0.1"
 tracing-opentelemetry = "0.17.2"

From 874b629b93a6e98e05fd43d5974e0a49b6e3e542 Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Wed, 26 Oct 2022 20:18:53 +0200
Subject: [PATCH 19/73] make everything to do with ImmutableStore Send + Sync

---
 src/storage/either.rs | 14 +++++---------
 src/storage/mod.rs    |  8 ++++----
 src/storage/s3.rs     |  1 +
 3 files changed, 10 insertions(+), 13 deletions(-)

diff --git a/src/storage/either.rs b/src/storage/either.rs
index 3f82d6ac..736a6a38 100644
--- a/src/storage/either.rs
+++ b/src/storage/either.rs
@@ -9,8 +9,8 @@ use kepler_lib::libipld::cid::multihash::Multihash;
 #[derive(Debug, Clone)]
 pub enum EitherStore<A, B>
 where
-    A: ImmutableStore + Sync,
-    B: ImmutableStore + Sync,
+    A: ImmutableStore,
+    B: ImmutableStore,
 {
     A(A),
     B(B),
@@ -62,11 +62,7 @@ where
 }
 
 #[derive(thiserror::Error, Debug)]
-pub enum EitherStoreError<A, B>
-where
-    A: std::error::Error,
-    B: std::error::Error,
-{
+pub enum EitherStoreError<A, B> {
     #[error(transparent)]
     A(A),
     #[error(transparent)]
@@ -76,8 +72,8 @@ where
 #[async_trait]
 impl<A, B> ImmutableStore for EitherStore<A, B>
 where
-    A: ImmutableStore + Sync,
-    B: ImmutableStore + Sync,
+    A: ImmutableStore,
+    B: ImmutableStore,
 {
     type Readable = AsyncReadEither<A, B>;
     type Error = EitherStoreError<A::Error, B::Error>;
diff --git a/src/storage/mod.rs b/src/storage/mod.rs
index 01ccc402..073ab6cb 100644
--- a/src/storage/mod.rs
+++ b/src/storage/mod.rs
@@ -236,7 +236,7 @@ impl StorageUtils {
     }
 }
 
-#[derive(thiserror::Error)]
+#[derive(thiserror::Error, Debug)]
 pub enum VecReadError<E> {
     #[error(transparent)]
     Store(#[from] E),
@@ -245,9 +245,9 @@ pub enum VecReadError<E> {
 }
 
 #[async_trait]
-pub trait ImmutableStore {
-    type Error: std::error::Error;
-    type Readable: futures::io::AsyncRead;
+pub trait ImmutableStore: Send + Sync {
+    type Error: std::error::Error + Send + Sync;
+    type Readable: futures::io::AsyncRead + Send + Sync;
     async fn contains(&self, id: &Multihash) -> Result<bool, Self::Error>;
     async fn write(
         &self,
diff --git a/src/storage/s3.rs b/src/storage/s3.rs
index 742310f3..c869b4dd 100644
--- a/src/storage/s3.rs
+++ b/src/storage/s3.rs
@@ -15,6 +15,7 @@ use kepler_lib::libipld::cid::{
 use regex::Regex;
 use rocket::{async_trait, http::hyper::Uri};
 use std::{io::Error as IoError, path::PathBuf, str::FromStr};
+use tokio_util::compat::TokioAsyncReadCompatExt;
 
 use crate::{
     config,

From 57f9f3dc72b384b9ad984aeac9c365ce9888749b Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Wed, 26 Oct 2022 20:20:46 +0200
Subject: [PATCH 20/73] fix up the KV mod to use ImmutableStore

---
 src/kv/behaviour.rs |   4 +-
 src/kv/entries.rs   |  57 +++----
 src/kv/mod.rs       | 382 ++++++++++++++++++++++----------------------
 src/kv/store.rs     |  51 +++---
 4 files changed, 245 insertions(+), 249 deletions(-)

diff --git a/src/kv/behaviour.rs b/src/kv/behaviour.rs
index ca848620..111b5182 100644
--- a/src/kv/behaviour.rs
+++ b/src/kv/behaviour.rs
@@ -9,7 +9,7 @@ use std::{
 use libp2p::{
     core::{connection::ConnectionId, ConnectedPoint, Multiaddr, PeerId},
     swarm::{
-        handler::ConnectionHandler as DummyConnectionHandler, IntoConnectionHandler,
+        dummy::ConnectionHandler as DummyConnectionHandler, IntoConnectionHandler,
         NetworkBehaviour, NetworkBehaviourAction, PollParameters,
     },
 };
@@ -36,7 +36,7 @@ impl NetworkBehaviour for Behaviour {
     type OutEvent = ();
 
     fn new_handler(&mut self) -> Self::ConnectionHandler {
-        DummyConnectionHandler::default()
+        DummyConnectionHandler
     }
 
     fn inject_event(&mut self, _peer_id: PeerId, _connection: ConnectionId, _event: Void) {}
diff --git a/src/kv/entries.rs b/src/kv/entries.rs
index 00241441..a0860366 100644
--- a/src/kv/entries.rs
+++ b/src/kv/entries.rs
@@ -1,17 +1,8 @@
-use super::{to_block, to_block_raw};
+use super::to_block;
 use crate::Block;
 use anyhow::Result;
 use kepler_lib::libipld::{cid::Cid, DagCbor};
-use libp2p::futures::stream::BoxStream;
-use std::{
-    collections::BTreeMap,
-    io::{self, Cursor, ErrorKind, Write},
-};
-
-use rocket::futures::{StreamExt, TryStreamExt};
-use rocket::tokio::io::AsyncRead;
-use tokio_stream::iter;
-use tokio_util::io::{ReaderStream, StreamReader};
+use std::collections::BTreeMap;
 
 #[derive(DagCbor, PartialEq, Eq, Debug, Clone)]
 pub struct Object {
@@ -68,34 +59,34 @@ impl ObjectBuilder {
 #[cfg(test)]
 mod test {
 
-    use super::*;
-    use crate::{config, kv::DagCborCodec, tracing::tracing_try_init};
+    // use super::*;
+    // use crate::{config, kv::DagCborCodec, tracing::tracing_try_init};
 
-    #[tokio::test(flavor = "multi_thread")]
-    async fn write() -> Result<(), anyhow::Error> {
-        tracing_try_init(&config::Logging::default());
-        let tmp = tempdir::TempDir::new("test_streams")?;
-        let data = vec![3u8; 1000000000 * 3];
+    // #[tokio::test(flavor = "multi_thread")]
+    // async fn write() -> Result<(), anyhow::Error> {
+    //     tracing_try_init(&config::Logging::default());
+    //     let tmp = tempdir::TempDir::new("test_streams")?;
+    //     let data = vec![3u8; 1000000000 * 3];
 
-        let mut config = IpfsOptions::inmemory_with_generated_keys();
-        config.ipfs_path = tmp.path().into();
+    //     let mut config = IpfsOptions::inmemory_with_generated_keys();
+    //     config.ipfs_path = tmp.path().into();
 
-        let (ipfs, task) = config.create_uninitialised_ipfs()?.start().await?;
-        let _join_handle = tokio::spawn(task);
+    //     let (ipfs, task) = config.create_uninitialised_ipfs()?.start().await?;
+    //     let _join_handle = tokio::spawn(task);
 
-        let o = write_to_store(&ipfs, Cursor::new(data.clone())).await?;
+    //     let o = write_to_store(&ipfs, Cursor::new(data.clone())).await?;
 
-        let content = ipfs
-            .get_block(&o)
-            .await?
-            .decode::<DagCborCodec, Vec<(Cid, u32)>>()?;
+    //     let content = ipfs
+    //         .get_block(&o)
+    //         .await?
+    //         .decode::<DagCborCodec, Vec<(Cid, u32)>>()?;
 
-        let mut read = read_from_store(ipfs, content);
+    //     let mut read = read_from_store(ipfs, content);
 
-        let mut out = Vec::new();
-        tokio::io::copy(&mut read, &mut out).await?;
+    //     let mut out = Vec::new();
+    //     tokio::io::copy(&mut read, &mut out).await?;
 
-        assert_eq!(out.len(), data.len());
-        Ok(())
-    }
+    //     assert_eq!(out.len(), data.len());
+    //     Ok(())
+    // }
 }
diff --git a/src/kv/mod.rs b/src/kv/mod.rs
index 5049fb41..d9337f54 100644
--- a/src/kv/mod.rs
+++ b/src/kv/mod.rs
@@ -94,9 +94,9 @@ async fn kv_task<B>(
                 Ok((p, KVMessage::Heads(heads))) => {
                     debug!("{} received new heads from {}", peer_id, p);
                     // sync heads
-                    if let Err(e) = store.try_merge_heads(heads.into_iter()).await {
-                        error!("failed to merge heads {}", e);
-                    };
+                    // if let Err(e) = store.try_merge_heads(heads.into_iter()).await {
+                    //     error!("failed to merge heads {}", e);
+                    // };
                 }
                 Ok((p, KVMessage::StateReq)) => {
                     // debug!("{} requests state", p);
@@ -115,192 +115,192 @@ async fn kv_task<B>(
 
 #[cfg(test)]
 mod test {
-    use libp2p::{identity::Keypair, multiaddr::Protocol};
-
-    use super::*;
-    use crate::{config, relay::test::test_relay};
-    use std::{
-        collections::BTreeMap, convert::TryFrom, path::PathBuf, str::FromStr, time::Duration,
-    };
-
-    async fn create_store<I>(
-        id: &Cid,
-        path: PathBuf,
-        keypair: Keypair,
-        allowed_peers: I,
-    ) -> Result<(Store, behaviour::BehaviourProcess), anyhow::Error>
-    where
-        I: IntoIterator<Item = PeerId> + 'static,
-    {
-        std::fs::create_dir(path.clone())?;
-        let config = config::Config {
-            storage: config::Storage {
-                blocks: config::BlockStorage::Local(config::LocalBlockStorage {
-                    path: path.clone(),
-                }),
-                indexes: config::IndexStorage::Local(config::LocalIndexStorage {
-                    path: path.clone(),
-                }),
-            },
-            ..Default::default()
-        };
-        let (ipfs, ipfs_task, receiver) = create_ipfs(*id, &config, keypair, allowed_peers).await?;
-        tokio::spawn(ipfs_task);
-        let store = Store::new(*id, ipfs, config.storage.indexes).await?;
-        Ok((
-            store.clone(),
-            behaviour::BehaviourProcess::new(store, receiver),
-        ))
-    }
-
-    #[tokio::test(flavor = "multi_thread")]
-    async fn test() -> Result<(), anyhow::Error> {
-        let relay = test_relay().await?;
-        let relay_peer_id = relay.id.clone();
-        let relay_internal = relay.internal();
-
-        let tmp = tempdir::TempDir::new("test_streams")?;
-        let id =
-            Cid::from_str("bafkreieq5jui4j25lacwomsqgjeswwl3y5zcdrresptwgmfylxo2depppq").unwrap();
-
-        let alice_keypair = Keypair::generate_ed25519();
-        let alice_peer_id = alice_keypair.public().to_peer_id();
-        let bob_keypair = Keypair::generate_ed25519();
-        let bob_peer_id = bob_keypair.public().to_peer_id();
-
-        let (alice_store, _alice_behaviour_process) =
-            create_store(&id, tmp.path().join("alice"), alice_keypair, [bob_peer_id]).await?;
-        let (bob_store, _bob_behaviour_process) =
-            create_store(&id, tmp.path().join("bob"), bob_keypair, [bob_peer_id]).await?;
-
-        let alice_service = alice_store.start_service().await?;
-        let bob_service = bob_store.start_service().await?;
-
-        // Connect the peers to the relay.
-        alice_service
-            .ipfs
-            .connect(
-                MultiaddrWithoutPeerId::try_from(relay_internal.clone())?
-                    .with(relay_peer_id.clone()),
-            )
-            .await
-            .expect("alice failed to connect to relay");
-        bob_service
-            .ipfs
-            .connect(
-                MultiaddrWithoutPeerId::try_from(relay_internal.clone())?
-                    .with(relay_peer_id.clone()),
-            )
-            .await
-            .expect("bob failed to connect to relay");
-
-        // Connect the peers to eachother.
-        bob_service
-            .ipfs
-            .connect(
-                MultiaddrWithoutPeerId::try_from(
-                    relay
-                        .external()
-                        .with(Protocol::P2p(relay_peer_id.into()))
-                        .with(Protocol::P2pCircuit),
-                )?
-                .with(alice_peer_id.clone()),
-            )
-            .await
-            .expect("bob failed to connect to alice");
-
-        // TODO: Work out why there is a race condition, and fix it so we don't need this sleep between connecting and writing.
-        tokio::time::sleep(Duration::from_millis(50)).await;
-
-        let json = r#"{"hello":"there"}"#;
-        let key1 = "my_json.json";
-        let key2 = "my_dup_json.json";
-        let md: BTreeMap<String, String> =
-            [("content-type".to_string(), "application/json".to_string())]
-                .to_vec()
-                .into_iter()
-                .collect();
-        let dab = to_block(&id).unwrap();
-        let dummy_auth = *dab.cid();
-        alice_service.ipfs.put_block(dab).await?;
-
-        let kv_obj_1 = ObjectBuilder::new(key1.as_bytes().to_vec(), md.clone(), dummy_auth.clone());
-        let kv_obj_2 = ObjectBuilder::new(key2.as_bytes().to_vec(), md.clone(), dummy_auth.clone());
-
-        type RmItem = (Vec<u8>, Option<(u64, Cid)>, Cid);
-        let rm: Vec<RmItem> = vec![];
-        alice_service
-            .write(vec![(kv_obj_1, json.as_bytes())], rm.clone())
-            .await?;
-        bob_service
-            .write(vec![(kv_obj_2, json.as_bytes())], rm)
-            .await?;
-
-        {
-            // ensure only alice has kv_obj_1
-            let o = alice_service
-                .get(key1)
-                .await?
-                .expect("object 1 not found for alice");
-            assert_eq!(&o.key, key1.as_bytes());
-            assert_eq!(&o.metadata, &md);
-            assert_eq!(bob_service.get(key1).await?, None, "object 1 found for bob");
-        };
-        {
-            // ensure only bob has kv_obj_2
-            let o = bob_service
-                .get(key2)
-                .await?
-                .expect("object 2 not found for bob");
-            assert_eq!(&o.key, key2.as_bytes());
-            assert_eq!(&o.metadata, &md);
-            assert_eq!(
-                alice_service.get(key2).await?,
-                None,
-                "object 2 found for alice"
-            );
-        };
-
-        tokio::time::sleep(Duration::from_millis(500)).await;
-
-        assert_eq!(
-            bob_service
-                .get(key1)
-                .await?
-                .expect("object 1 not found for bob"),
-            alice_service
-                .get(key1)
-                .await?
-                .expect("object 1 not found for alice")
-        );
-
-        assert_eq!(
-            bob_service
-                .get(key2)
-                .await?
-                .expect("object 2 not found for bob"),
-            alice_service
-                .get(key2)
-                .await?
-                .expect("object 2 not found for alice")
-        );
-
-        // remove key1
-        let add: Vec<(&[u8], Cid)> = vec![];
-        alice_service
-            .index(add, vec![(key1.as_bytes().to_vec(), None, dummy_auth)])
-            .await?;
-
-        assert_eq!(
-            alice_service.get(key1).await?,
-            None,
-            "alice still has object 1"
-        );
-
-        std::thread::sleep(Duration::from_millis(500));
-
-        assert_eq!(bob_service.get(key1).await?, None, "bob still has object 1");
-
-        Ok(())
-    }
+    // use libp2p::{identity::Keypair, multiaddr::Protocol};
+
+    // use super::*;
+    // use crate::{config, relay::test::test_relay};
+    // use std::{
+    //     collections::BTreeMap, convert::TryFrom, path::PathBuf, str::FromStr, time::Duration,
+    // };
+
+    // async fn create_store<I>(
+    //     id: &Cid,
+    //     path: PathBuf,
+    //     keypair: Keypair,
+    //     allowed_peers: I,
+    // ) -> Result<(Store, behaviour::BehaviourProcess), anyhow::Error>
+    // where
+    //     I: IntoIterator<Item = PeerId> + 'static,
+    // {
+    //     std::fs::create_dir(path.clone())?;
+    //     let config = config::Config {
+    //         storage: config::Storage {
+    //             blocks: config::BlockStorage::Local(config::LocalBlockStorage {
+    //                 path: path.clone(),
+    //             }),
+    //             indexes: config::IndexStorage::Local(config::LocalIndexStorage {
+    //                 path: path.clone(),
+    //             }),
+    //         },
+    //         ..Default::default()
+    //     };
+    //     let (ipfs, ipfs_task, receiver) = create_ipfs(*id, &config, keypair, allowed_peers).await?;
+    //     tokio::spawn(ipfs_task);
+    //     let store = Store::new(*id, ipfs, config.storage.indexes).await?;
+    //     Ok((
+    //         store.clone(),
+    //         behaviour::BehaviourProcess::new(store, receiver),
+    //     ))
+    // }
+
+    // #[tokio::test(flavor = "multi_thread")]
+    // async fn test() -> Result<(), anyhow::Error> {
+    //     let relay = test_relay().await?;
+    //     let relay_peer_id = relay.id.clone();
+    //     let relay_internal = relay.internal();
+
+    //     let tmp = tempdir::TempDir::new("test_streams")?;
+    //     let id =
+    //         Cid::from_str("bafkreieq5jui4j25lacwomsqgjeswwl3y5zcdrresptwgmfylxo2depppq").unwrap();
+
+    //     let alice_keypair = Keypair::generate_ed25519();
+    //     let alice_peer_id = alice_keypair.public().to_peer_id();
+    //     let bob_keypair = Keypair::generate_ed25519();
+    //     let bob_peer_id = bob_keypair.public().to_peer_id();
+
+    //     let (alice_store, _alice_behaviour_process) =
+    //         create_store(&id, tmp.path().join("alice"), alice_keypair, [bob_peer_id]).await?;
+    //     let (bob_store, _bob_behaviour_process) =
+    //         create_store(&id, tmp.path().join("bob"), bob_keypair, [bob_peer_id]).await?;
+
+    //     let alice_service = alice_store.start_service().await?;
+    //     let bob_service = bob_store.start_service().await?;
+
+    //     // Connect the peers to the relay.
+    //     alice_service
+    //         .ipfs
+    //         .connect(
+    //             MultiaddrWithoutPeerId::try_from(relay_internal.clone())?
+    //                 .with(relay_peer_id.clone()),
+    //         )
+    //         .await
+    //         .expect("alice failed to connect to relay");
+    //     bob_service
+    //         .ipfs
+    //         .connect(
+    //             MultiaddrWithoutPeerId::try_from(relay_internal.clone())?
+    //                 .with(relay_peer_id.clone()),
+    //         )
+    //         .await
+    //         .expect("bob failed to connect to relay");
+
+    //     // Connect the peers to eachother.
+    //     bob_service
+    //         .ipfs
+    //         .connect(
+    //             MultiaddrWithoutPeerId::try_from(
+    //                 relay
+    //                     .external()
+    //                     .with(Protocol::P2p(relay_peer_id.into()))
+    //                     .with(Protocol::P2pCircuit),
+    //             )?
+    //             .with(alice_peer_id.clone()),
+    //         )
+    //         .await
+    //         .expect("bob failed to connect to alice");
+
+    //     // TODO: Work out why there is a race condition, and fix it so we don't need this sleep between connecting and writing.
+    //     tokio::time::sleep(Duration::from_millis(50)).await;
+
+    //     let json = r#"{"hello":"there"}"#;
+    //     let key1 = "my_json.json";
+    //     let key2 = "my_dup_json.json";
+    //     let md: BTreeMap<String, String> =
+    //         [("content-type".to_string(), "application/json".to_string())]
+    //             .to_vec()
+    //             .into_iter()
+    //             .collect();
+    //     let dab = to_block(&id).unwrap();
+    //     let dummy_auth = *dab.cid();
+    //     alice_service.ipfs.put_block(dab).await?;
+
+    //     let kv_obj_1 = ObjectBuilder::new(key1.as_bytes().to_vec(), md.clone(), dummy_auth.clone());
+    //     let kv_obj_2 = ObjectBuilder::new(key2.as_bytes().to_vec(), md.clone(), dummy_auth.clone());
+
+    //     type RmItem = (Vec<u8>, Option<(u64, Cid)>, Cid);
+    //     let rm: Vec<RmItem> = vec![];
+    //     alice_service
+    //         .write(vec![(kv_obj_1, json.as_bytes())], rm.clone())
+    //         .await?;
+    //     bob_service
+    //         .write(vec![(kv_obj_2, json.as_bytes())], rm)
+    //         .await?;
+
+    //     {
+    //         // ensure only alice has kv_obj_1
+    //         let o = alice_service
+    //             .get(key1)
+    //             .await?
+    //             .expect("object 1 not found for alice");
+    //         assert_eq!(&o.key, key1.as_bytes());
+    //         assert_eq!(&o.metadata, &md);
+    //         assert_eq!(bob_service.get(key1).await?, None, "object 1 found for bob");
+    //     };
+    //     {
+    //         // ensure only bob has kv_obj_2
+    //         let o = bob_service
+    //             .get(key2)
+    //             .await?
+    //             .expect("object 2 not found for bob");
+    //         assert_eq!(&o.key, key2.as_bytes());
+    //         assert_eq!(&o.metadata, &md);
+    //         assert_eq!(
+    //             alice_service.get(key2).await?,
+    //             None,
+    //             "object 2 found for alice"
+    //         );
+    //     };
+
+    //     tokio::time::sleep(Duration::from_millis(500)).await;
+
+    //     assert_eq!(
+    //         bob_service
+    //             .get(key1)
+    //             .await?
+    //             .expect("object 1 not found for bob"),
+    //         alice_service
+    //             .get(key1)
+    //             .await?
+    //             .expect("object 1 not found for alice")
+    //     );
+
+    //     assert_eq!(
+    //         bob_service
+    //             .get(key2)
+    //             .await?
+    //             .expect("object 2 not found for bob"),
+    //         alice_service
+    //             .get(key2)
+    //             .await?
+    //             .expect("object 2 not found for alice")
+    //     );
+
+    //     // remove key1
+    //     let add: Vec<(&[u8], Cid)> = vec![];
+    //     alice_service
+    //         .index(add, vec![(key1.as_bytes().to_vec(), None, dummy_auth)])
+    //         .await?;
+
+    //     assert_eq!(
+    //         alice_service.get(key1).await?,
+    //         None,
+    //         "alice still has object 1"
+    //     );
+
+    //     std::thread::sleep(Duration::from_millis(500));
+
+    //     assert_eq!(bob_service.get(key1).await?, None, "bob still has object 1");
+
+    //     Ok(())
+    // }
 }
diff --git a/src/kv/store.rs b/src/kv/store.rs
index d7c6b2ef..96f76fe9 100644
--- a/src/kv/store.rs
+++ b/src/kv/store.rs
@@ -2,14 +2,16 @@ use crate::indexes::{AddRemoveSetStore, HeadStore};
 use crate::kv::{Object, ObjectBuilder, Service};
 use crate::storage::ImmutableStore;
 use anyhow::Result;
-use async_recursion::async_recursion;
-use futures::stream::{self, StreamExt, TryStreamExt};
-use kepler_lib::libipld::{cbor::DagCborCodec, cid::Cid, multibase::Base, DagCbor};
-use rocket::{futures::future::try_join_all, tokio::io::AsyncRead};
+use futures::{
+    io::AsyncRead,
+    stream::{self, StreamExt, TryStreamExt},
+};
+use kepler_lib::libipld::{cid::Cid, multibase::Base, DagCbor};
+use rocket::futures::future::try_join_all;
 use std::{collections::BTreeMap, convert::TryFrom};
-use tracing::{debug, instrument};
+use tracing::instrument;
 
-use super::{to_block, Block, KVMessage};
+use super::{to_block, Block};
 use crate::config;
 
 #[derive(DagCbor)]
@@ -149,26 +151,29 @@ impl<B> Store<B> {
 
 impl<B> Store<B>
 where
-    B: ImmutableStore,
+    B: ImmutableStore + 'static,
 {
     #[instrument(name = "kv::get", skip_all)]
     pub async fn get<N>(&self, name: N) -> Result<Option<Object>>
     where
         N: AsRef<[u8]>,
     {
-        let cid = match self.index.element(&name).await? {
-            Some(Element(_, cid)) => Ok(
-                match self
-                    .index
-                    .is_tombstoned(&Version(name, cid).to_bytes())
+        match self.index.element(&name).await? {
+            Some(Element(_, cid)) => match self
+                .index
+                .is_tombstoned(&Version(name, cid).to_bytes())
+                .await?
+            {
+                false => self
+                    .blocks
+                    .read_to_vec(cid.hash())
                     .await?
-                {
-                    false => self.blocks.read_to_vec(cid).await,
-                    _ => return Ok(None),
-                },
-            ),
-            None => return Ok(None),
-        };
+                    .map(|v| Ok(Block::new(cid, v)?.decode()?))
+                    .transpose(),
+                _ => Ok(None),
+            },
+            None => Ok(None),
+        }
     }
 
     #[instrument(name = "kv::read", skip_all)]
@@ -180,7 +185,7 @@ where
             Ok(Some(content)) => content,
             _ => return Ok(None),
         };
-        match self.blocks.read(&kv_obj.value).await? {
+        match self.blocks.read(&kv_obj.value.hash()).await? {
             Some(r) => Ok(Some((kv_obj.metadata, r))),
             None => Err(anyhow!("Indexed contents missing from block store")),
         }
@@ -195,7 +200,7 @@ where
     ) -> Result<()>
     where
         N: AsRef<[u8]>,
-        R: AsyncRead + Unpin,
+        R: AsyncRead + Send,
     {
         tracing::debug!("writing tx");
         let indexes: Vec<(Vec<u8>, Cid)> = try_join_all(add.into_iter().map(|(o, r)| async {
@@ -204,7 +209,7 @@ where
             let cid = Cid::new_v1(0x55, self.blocks.write(r).await?);
             let obj = o.add_content(cid);
             let block = obj.to_block()?;
-            let obj_cid = Cid::new_v1(obj.cid().codec(), self.blocks.write(block.data()).await?);
+            let obj_cid = Cid::new_v1(block.cid().codec(), self.blocks.write(block.data()).await?);
             Ok((obj.key, obj_cid)) as Result<(Vec<u8>, Cid)>
         }))
         .await?
@@ -319,7 +324,7 @@ where
 
         Ok(())
     }
-    pub async fn start_service(self) -> Result<Service> {
+    pub async fn start_service(self) -> Result<Service<B>> {
         Service::start(self).await
     }
 }

From 2b2b57618cf6141285e9d5766b106f0cf09a7087 Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Wed, 26 Oct 2022 20:21:27 +0200
Subject: [PATCH 21/73] fix caps and auth to use immutablestore

---
 src/auth_guards.rs        | 23 +++++------
 src/authorization.rs      |  5 +--
 src/capabilities/mod.rs   | 18 ++++-----
 src/capabilities/store.rs | 82 ++++++++++++++++++++++++++++++---------
 4 files changed, 86 insertions(+), 42 deletions(-)

diff --git a/src/auth_guards.rs b/src/auth_guards.rs
index a8427b8a..411f6189 100644
--- a/src/auth_guards.rs
+++ b/src/auth_guards.rs
@@ -4,6 +4,7 @@ use crate::config;
 use crate::orbit::{create_orbit, load_orbit, Orbit};
 use crate::relay::RelayNode;
 use crate::routes::Metadata;
+use crate::storage::BlockStores;
 use anyhow::Result;
 use kepler_lib::{
     libipld::Cid,
@@ -216,12 +217,12 @@ impl<'l> FromRequest<'l> for DelegateAuthWrapper {
     }
 }
 
-pub enum InvokeAuthWrapper {
-    KV(Box<KVAction>),
+pub enum InvokeAuthWrapper<B> {
+    KV(Box<KVAction<B>>),
     Revocation,
 }
 
-impl InvokeAuthWrapper {
+impl<B> InvokeAuthWrapper<B> {
     pub fn prometheus_label(&self) -> &str {
         match self {
             InvokeAuthWrapper::Revocation => "revoke_delegation",
@@ -230,33 +231,33 @@ impl InvokeAuthWrapper {
     }
 }
 
-pub enum KVAction {
+pub enum KVAction<B> {
     Delete {
-        orbit: Orbit,
+        orbit: Orbit<B>,
         key: String,
         auth_ref: Cid,
     },
     Get {
-        orbit: Orbit,
+        orbit: Orbit<B>,
         key: String,
     },
     List {
-        orbit: Orbit,
+        orbit: Orbit<B>,
         prefix: String,
     },
     Metadata {
-        orbit: Orbit,
+        orbit: Orbit<B>,
         key: String,
     },
     Put {
-        orbit: Orbit,
+        orbit: Orbit<B>,
         key: String,
         metadata: Metadata,
         auth_ref: Cid,
     },
 }
 
-impl KVAction {
+impl<B> KVAction<B> {
     pub fn prometheus_label(&self) -> &str {
         match self {
             KVAction::Delete { .. } => "kv_delete",
@@ -269,7 +270,7 @@ impl KVAction {
 }
 
 #[async_trait]
-impl<'l> FromRequest<'l> for InvokeAuthWrapper {
+impl<'l> FromRequest<'l> for InvokeAuthWrapper<BlockStores> {
     type Error = anyhow::Error;
 
     async fn from_request(req: &'l Request<'_>) -> Outcome<Self, Self::Error> {
diff --git a/src/authorization.rs b/src/authorization.rs
index 5eaa647e..c26042b2 100644
--- a/src/authorization.rs
+++ b/src/authorization.rs
@@ -3,13 +3,13 @@ use crate::{
     Block,
 };
 use anyhow::Result;
-use kepler_lib::libipld::Cid;
 use kepler_lib::{
     authorization::{
         EncodingError, HeaderEncode, KeplerDelegation, KeplerInvocation, KeplerRevocation,
     },
     cacaos::siwe::Message,
     capgrok::{extract_capabilities, verify_statement, Capability as SiweCap, Set},
+    libipld::{multihash::Code, Cid},
     resolver::DID_METHODS,
     resource::{KRIParseError, ResourceId},
     ssi::ucan::Capability as UcanCap,
@@ -139,8 +139,7 @@ pub struct Invocation {
 
 impl ToBlock for Invocation {
     fn to_block(&self) -> Result<Block> {
-        self.invocation
-            .to_block(libipld::multihash::Code::Blake3_256)
+        self.invocation.to_block(Code::Blake3_256)
     }
 }
 
diff --git a/src/capabilities/mod.rs b/src/capabilities/mod.rs
index 0901b213..7d9e0f4a 100644
--- a/src/capabilities/mod.rs
+++ b/src/capabilities/mod.rs
@@ -50,9 +50,9 @@ async fn caps_task<B>(
                 }
                 Ok((_, CapsMessage::Invocation(cid))) => {
                     debug!("recieved invocation");
-                    if let Err(e) = store.try_merge_invocations([cid].into_iter()).await {
-                        debug!("failed to apply recieved invocation {}", e);
-                    }
+                    // if let Err(e) = store.try_merge_invocations([cid].into_iter()).await {
+                    //     debug!("failed to apply recieved invocation {}", e);
+                    // }
                 }
                 Ok((_, CapsMessage::StateReq)) => {
                     // if let Err(e) = store.broadcast_heads().await {
@@ -69,12 +69,12 @@ async fn caps_task<B>(
                         invocations,
                     },
                 )) => {
-                    if let Err(e) = store
-                        .try_merge_heads(updates.into_iter(), invocations.into_iter())
-                        .await
-                    {
-                        debug!("failed to merge heads {}", e);
-                    }
+                    // if let Err(e) = store
+                    //     .try_merge_heads(updates.into_iter(), invocations.into_iter())
+                    //     .await
+                    // {
+                    //     debug!("failed to merge heads {}", e);
+                    // }
                 }
                 Err(e) => {
                     debug!("cap service task error {}", e);
diff --git a/src/capabilities/store.rs b/src/capabilities/store.rs
index 9474d331..20ce88a3 100644
--- a/src/capabilities/store.rs
+++ b/src/capabilities/store.rs
@@ -39,7 +39,11 @@ pub struct Store<B> {
     invocation_heads: HeadStore,
 }
 
-impl<B> Store<B> {
+impl<B> Store<B>
+where
+    B: ImmutableStore,
+    B::Error: 'static,
+{
     pub async fn new(oid: &OrbitId, blocks: B, config: config::IndexStorage) -> Result<Self> {
         let id = oid
             .clone()
@@ -98,8 +102,12 @@ impl<B> Store<B> {
             delegate: event.delegate.iter().map(|d| *d.0.block.cid()).collect(),
             revoke: event.revoke.iter().map(|r| *r.0.block.cid()).collect(),
         };
+        let eb_block = eb.to_block()?;
 
-        let cid = self.blocks.write(eb.to_block()?.data()).await?;
+        let cid = Cid::new_v1(
+            eb_block.cid().codec(),
+            self.blocks.write(eb_block.data()).await?,
+        );
 
         // write element indexes
         try_join_all(event.delegate.into_iter().map(|d| async {
@@ -181,18 +189,28 @@ impl<B> Store<B> {
                     .collect::<Result<Vec<(WithBlock<LinkedUpdate>, WithBlock<Invocation>)>>>()?,
             })
             .await?;
-        self.broadcast_heads().await?;
+        // self.broadcast_heads().await?;
         Ok(cid)
     }
 
     async fn get_invocation(&self, i: &Cid) -> Result<Invocations> {
-        let update: InvocationsBlock = self.get_obj(i).await?.base;
+        let update: InvocationsBlock = self
+            .get_obj(i)
+            .await?
+            .ok_or_else(|| anyhow!("Incomplete Event"))?
+            .base;
 
         Ok(Invocations {
             prev: update.prev,
             invoke: try_join_all(update.invoke.iter().map(|c| async {
-                let link: WithBlock<LinkedUpdate> = self.get_obj(c).await?;
-                let inv: WithBlock<Invocation> = self.get_obj(&link.base.update).await?;
+                let link: WithBlock<LinkedUpdate> = self
+                    .get_obj(c)
+                    .await?
+                    .ok_or_else(|| anyhow!("Incomplete Event"))?;
+                let inv: WithBlock<Invocation> = self
+                    .get_obj(&link.base.update)
+                    .await?
+                    .ok_or_else(|| anyhow!("Incomplete Event"))?;
                 Result::<(WithBlock<LinkedUpdate>, WithBlock<Invocation>)>::Ok((link, inv))
             }))
             .await?,
@@ -212,7 +230,11 @@ impl<B> Store<B> {
             prev: event.prev,
             invoke: event.invoke.iter().map(|i| *i.0.block.cid()).collect(),
         };
-        let cid = self.blocks.write(eb.to_block()?.data()).await?;
+        let eb_block = eb.to_block()?;
+        let cid = Cid::new_v1(
+            eb_block.cid().codec(),
+            self.blocks.write(eb_block.data()).await?,
+        );
 
         for e in event.invoke.iter() {
             self.index
@@ -260,31 +282,50 @@ impl<B> Store<B> {
         })
     }
 
-    async fn get_event(&self, e: &Cid) -> Result<Event> {
-        let update: EventBlock = self.get_obj(e).await?.base;
+    async fn get_event(&self, e: &Cid) -> Result<Option<Event>> {
+        let update: EventBlock = match self.get_obj(e).await? {
+            Some(e) => e.base,
+            None => return Ok(None),
+        };
 
-        Ok(Event {
+        Ok(Some(Event {
             prev: update.prev,
             delegate: try_join_all(update.delegate.iter().map(|c| async {
-                let link: WithBlock<LinkedUpdate> = self.get_obj(c).await?;
-                let del: WithBlock<Delegation> = self.get_obj(&link.base.update).await?;
+                let link: WithBlock<LinkedUpdate> = self
+                    .get_obj(c)
+                    .await?
+                    .ok_or_else(|| anyhow!("Incomplete Event"))?;
+                let del: WithBlock<Delegation> = self
+                    .get_obj(&link.base.update)
+                    .await?
+                    .ok_or_else(|| anyhow!("Incomplete Event"))?;
                 Result::<(WithBlock<LinkedUpdate>, WithBlock<Delegation>)>::Ok((link, del))
             }))
             .await?,
             revoke: try_join_all(update.revoke.iter().map(|c| async {
-                let link: WithBlock<LinkedUpdate> = self.get_obj(c).await?;
-                let rev: WithBlock<Revocation> = self.get_obj(&link.base.update).await?;
+                let link: WithBlock<LinkedUpdate> = self
+                    .get_obj(c)
+                    .await?
+                    .ok_or_else(|| anyhow!("Incomplete Event"))?;
+                let rev: WithBlock<Revocation> = self
+                    .get_obj(&link.base.update)
+                    .await?
+                    .ok_or_else(|| anyhow!("Incomplete Event"))?;
                 Result::<(WithBlock<LinkedUpdate>, WithBlock<Revocation>)>::Ok((link, rev))
             }))
             .await?,
-        })
+        }))
     }
 
-    async fn get_obj<T>(&self, c: &Cid) -> Result<WithBlock<T>>
+    async fn get_obj<T>(&self, c: &Cid) -> Result<Option<WithBlock<T>>>
     where
         T: FromBlock,
     {
-        WithBlock::<T>::try_from(Block::new_v1(c, self.blocks.read_to_vec(c.hash()).await?))
+        self.blocks
+            .read_to_vec(c.hash())
+            .await?
+            .map(|v| Block::new(*c, v).and_then(WithBlock::try_from))
+            .transpose()
     }
 
     pub(crate) async fn try_merge_heads(
@@ -306,7 +347,10 @@ impl<B> Store<B> {
             if self.delegation_heads.get_height(&head).await?.is_some() {
                 return Ok(());
             };
-            let update = self.get_event(&head).await?;
+            let update = self
+                .get_event(&head)
+                .await?
+                .ok_or_else(|| anyhow!("Missing Event"))?;
 
             self.try_merge_updates(
                 stream::iter(update.prev.iter().map(Ok).collect::<Vec<Result<_>>>())
@@ -369,7 +413,7 @@ where
 {
     async fn get_cap(&self, c: &Cid) -> Result<Option<Delegation>> {
         // annoyingly ipfs will error if it cant find something, so we probably dont want to error here
-        Ok(self.get_obj(c).await.ok().map(|d| d.base))
+        self.get_obj(c).await.map(|o| o.map(|d| d.base))
     }
 }
 

From c5f63822b33ad8e8475c4fdd6f51e61e619af3f3 Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Wed, 26 Oct 2022 20:24:01 +0200
Subject: [PATCH 22/73] warnings

---
 src/capabilities/mod.rs    | 6 +-----
 src/capabilities/store.rs  | 2 +-
 src/storage/file_system.rs | 5 +----
 src/storage/s3.rs          | 1 -
 4 files changed, 3 insertions(+), 11 deletions(-)

diff --git a/src/capabilities/mod.rs b/src/capabilities/mod.rs
index 7d9e0f4a..b091e2fc 100644
--- a/src/capabilities/mod.rs
+++ b/src/capabilities/mod.rs
@@ -1,15 +1,11 @@
 pub mod store;
 
-use crate::orbit::AbortOnDrop;
 use anyhow::Result;
-use kepler_lib::libipld::{cbor::DagCborCodec, codec::Decode, multibase::Base, Cid};
+use kepler_lib::libipld::Cid;
 use libp2p::core::PeerId;
 use rocket::futures::{Stream, StreamExt};
 use store::{CapsMessage, Store};
 
-use std::io::Cursor;
-use std::sync::Arc;
-
 #[rocket::async_trait]
 pub trait Invoke<T> {
     async fn invoke(&self, invocation: &T) -> Result<Cid>;
diff --git a/src/capabilities/store.rs b/src/capabilities/store.rs
index 20ce88a3..3e193ebd 100644
--- a/src/capabilities/store.rs
+++ b/src/capabilities/store.rs
@@ -8,7 +8,7 @@ use crate::{
 use anyhow::Result;
 use async_recursion::async_recursion;
 use futures::stream::{self, TryStreamExt};
-use kepler_lib::libipld::{cbor::DagCborCodec, multibase::Base, multihash::Code, Cid, DagCbor};
+use kepler_lib::libipld::{cbor::DagCborCodec, multihash::Code, Cid, DagCbor};
 use kepler_lib::{
     authorization::{KeplerDelegation, KeplerInvocation, KeplerRevocation},
     cacaos::siwe_cacao::SiweCacao,
diff --git a/src/storage/file_system.rs b/src/storage/file_system.rs
index 731bdcf2..e2830a14 100644
--- a/src/storage/file_system.rs
+++ b/src/storage/file_system.rs
@@ -4,10 +4,7 @@ use kepler_lib::libipld::cid::{
     multihash::Multihash,
 };
 use std::{io::ErrorKind, path::PathBuf};
-use tokio::{
-    fs::{remove_file, File},
-    io::copy,
-};
+use tokio::fs::{remove_file, File};
 use tokio_util::compat::{Compat, TokioAsyncReadCompatExt};
 
 #[derive(Debug, Clone)]
diff --git a/src/storage/s3.rs b/src/storage/s3.rs
index c869b4dd..742310f3 100644
--- a/src/storage/s3.rs
+++ b/src/storage/s3.rs
@@ -15,7 +15,6 @@ use kepler_lib::libipld::cid::{
 use regex::Regex;
 use rocket::{async_trait, http::hyper::Uri};
 use std::{io::Error as IoError, path::PathBuf, str::FromStr};
-use tokio_util::compat::TokioAsyncReadCompatExt;
 
 use crate::{
     config,

From 36eefbe3dc88f86fb20593b7185e9f1db11b6d69 Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Wed, 26 Oct 2022 20:25:26 +0200
Subject: [PATCH 23/73] use ImmutableStore in Orbit

---
 src/orbit.rs | 16 ++++++++++------
 src/relay.rs |  4 ++--
 2 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/src/orbit.rs b/src/orbit.rs
index 5384de96..844e7bfb 100644
--- a/src/orbit.rs
+++ b/src/orbit.rs
@@ -13,14 +13,13 @@ use kepler_lib::libipld::cid::{
 use kepler_lib::resource::OrbitId;
 use libp2p::{
     core::Multiaddr,
-    identity::{ed25519::Keypair as Ed25519Keypair, Keypair, PublicKey},
+    identity::{ed25519::Keypair as Ed25519Keypair, PublicKey},
     PeerId,
 };
-use rocket::{futures::TryStreamExt, tokio::task::JoinHandle};
+use rocket::tokio::task::JoinHandle;
 
 use cached::proc_macro::cached;
-use std::{convert::TryFrom, ops::Deref, sync::Arc};
-use tokio::spawn;
+use std::{convert::TryFrom, ops::Deref};
 
 use super::storage::StorageUtils;
 
@@ -67,7 +66,10 @@ pub struct Orbit<B> {
     pub capabilities: CapService<B>,
 }
 
-impl<B> Orbit<B> {
+impl<B> Orbit<B>
+where
+    B: ImmutableStore,
+{
     async fn new(
         config: &config::Config,
         kp: Ed25519Keypair,
@@ -77,6 +79,8 @@ impl<B> Orbit<B> {
         let id = manifest.id().get_cid();
         let local_peer_id = PeerId::from_public_key(&PublicKey::Ed25519(kp.public()));
 
+        let storage_utils = StorageUtils::new(config.storage.blocks.clone());
+
         // TODO config into block store
         let blocks = todo!();
         let service_store = Store::new(id, blocks.clone(), config.storage.indexes.clone()).await?;
@@ -189,7 +193,7 @@ mod tests {
     use std::convert::TryInto;
     use tempdir::TempDir;
 
-    async fn op(md: Manifest) -> anyhow::Result<Orbit> {
+    async fn op(md: Manifest) -> anyhow::Result<Orbit<BlockStores>> {
         let dir = TempDir::new(&md.id().get_cid().to_string())
             .unwrap()
             .path()
diff --git a/src/relay.rs b/src/relay.rs
index 935259af..df6aa39b 100644
--- a/src/relay.rs
+++ b/src/relay.rs
@@ -13,8 +13,8 @@ pub struct RelayNode {
 
 impl RelayNode {
     pub async fn new(port: u16, keypair: Keypair) -> Result<Self> {
-        // let local_public_key = keypair.public();
-        // let id = local_public_key.to_peer_id();
+        let local_public_key = keypair.public();
+        let id = local_public_key.to_peer_id();
         // let relay_tcp_addr = Self::_external(port);
         // let relay_mem_addr = Self::_internal(port);
 

From a7643273f731dc00617a7b84727552adf7818075 Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Thu, 27 Oct 2022 08:36:09 +0200
Subject: [PATCH 24/73] fix up routes/responses to use AsyncRead

---
 src/routes/mod.rs | 44 ++++++++++++++++++++++++++++----------------
 1 file changed, 28 insertions(+), 16 deletions(-)

diff --git a/src/routes/mod.rs b/src/routes/mod.rs
index cb7bb75d..baaa922d 100644
--- a/src/routes/mod.rs
+++ b/src/routes/mod.rs
@@ -1,4 +1,5 @@
 use anyhow::Result;
+use futures::io::AsyncRead;
 use kepler_lib::libipld::Cid;
 use libp2p::{
     core::PeerId,
@@ -21,10 +22,12 @@ use tracing::{info_span, Instrument};
 
 use crate::{
     auth_guards::{DelegateAuthWrapper, InvokeAuthWrapper, KVAction},
-    kv::{ObjectBuilder, ObjectReader},
+    kv::ObjectBuilder,
     relay::RelayNode,
+    storage::{BlockStores, ImmutableStore},
     tracing::TracingSpan,
 };
+use tokio_util::compat::{FuturesAsyncReadCompatExt, TokioAsyncReadCompatExt};
 
 pub struct Metadata(pub BTreeMap<String, String>);
 
@@ -53,19 +56,22 @@ impl<'r> Responder<'r, 'static> for Metadata {
     }
 }
 
-pub struct KVResponse(ObjectReader, pub Metadata);
+pub struct KVResponse<R>(R, pub Metadata);
 
-impl KVResponse {
-    pub fn new(md: Metadata, reader: ObjectReader) -> Self {
+impl<R> KVResponse<R> {
+    pub fn new(md: Metadata, reader: R) -> Self {
         Self(reader, md)
     }
 }
 
-impl<'r> Responder<'r, 'static> for KVResponse {
+impl<'r, R> Responder<'r, 'static> for KVResponse<R>
+where
+    R: AsyncRead + Send,
+{
     fn respond_to(self, r: &'r Request<'_>) -> rocket::response::Result<'static> {
         Ok(Response::build_from(self.1.respond_to(r)?)
             // must ensure that Metadata::respond_to does not set the body of the response
-            .streamed_body(self.0)
+            .streamed_body(self.0.compat())
             .finalize())
     }
 }
@@ -117,10 +123,10 @@ impl<'r> Responder<'r, 'static> for DelegateAuthWrapper {
 
 #[post("/invoke", data = "<data>")]
 pub async fn invoke(
-    i: InvokeAuthWrapper,
+    i: InvokeAuthWrapper<BlockStores>,
     req_span: TracingSpan,
     data: Data<'_>,
-) -> Result<InvocationResponse, (Status, String)> {
+) -> Result<InvocationResponse<<BlockStores as ImmutableStore>::Readable>, (Status, String)> {
     let action_label = i.prometheus_label().to_string();
     let span = info_span!(parent: &req_span.0, "invoke", action = %action_label);
     // Instrumenting async block to handle yielding properly
@@ -141,10 +147,13 @@ pub async fn invoke(
     .await
 }
 
-pub async fn handle_kv_action(
-    action: KVAction,
-    data: Data<'_>,
-) -> Result<InvocationResponse, (Status, String)> {
+pub async fn handle_kv_action<'a, B>(
+    action: KVAction<B>,
+    data: Data<'a>,
+) -> Result<InvocationResponse<B::Readable>, (Status, String)>
+where
+    B: 'static + ImmutableStore,
+{
     match action {
         KVAction::Delete {
             orbit,
@@ -216,7 +225,7 @@ pub async fn handle_kv_action(
                 .write(
                     [(
                         ObjectBuilder::new(key.as_bytes().to_vec(), metadata.0, auth_ref),
-                        data.open(1u8.gigabytes()),
+                        data.open(1u8.gigabytes()).compat(),
                     )],
                     rm,
                 )
@@ -227,16 +236,19 @@ pub async fn handle_kv_action(
     }
 }
 
-pub enum InvocationResponse {
+pub enum InvocationResponse<R> {
     NotFound,
     EmptySuccess,
-    KVResponse(KVResponse),
+    KVResponse(KVResponse<R>),
     List(Vec<String>),
     Metadata(Metadata),
     Revoked,
 }
 
-impl<'r> Responder<'r, 'static> for InvocationResponse {
+impl<'r, R> Responder<'r, 'static> for InvocationResponse<R>
+where
+    R: AsyncRead + Send,
+{
     fn respond_to(self, request: &'r Request<'_>) -> rocket::response::Result<'static> {
         match self {
             InvocationResponse::NotFound => Option::<()>::None.respond_to(request),

From 75d43ddf5677909dd5086886fc8487b09c1bbc82 Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Thu, 27 Oct 2022 12:41:57 +0200
Subject: [PATCH 25/73] store minor fix

---
 src/capabilities/store.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/capabilities/store.rs b/src/capabilities/store.rs
index 3e193ebd..7e42e599 100644
--- a/src/capabilities/store.rs
+++ b/src/capabilities/store.rs
@@ -110,7 +110,7 @@ where
         );
 
         // write element indexes
-        try_join_all(event.delegate.into_iter().map(|d| async {
+        try_join_all(event.delegate.into_iter().map(|d| async move {
             // add backlink in index
             self.index
                 .set_element(&d.1.block.cid().to_bytes(), &d.0.block.cid().to_bytes())

From 5cece4fc00e8794a6adf21119d83a85c588b2f27 Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Mon, 31 Oct 2022 13:11:22 +0100
Subject: [PATCH 26/73] add StorageConfig trait, move config structs to
 respective modules

---
 src/storage/either.rs      | 43 ++++++++++++++++++++++++++--------
 src/storage/file_system.rs | 27 ++++++++++++++++++----
 src/storage/mod.rs         | 10 +++++++-
 src/storage/s3.rs          | 47 ++++++++++++++++++++++++++++----------
 4 files changed, 101 insertions(+), 26 deletions(-)

diff --git a/src/storage/either.rs b/src/storage/either.rs
index 736a6a38..efef4007 100644
--- a/src/storage/either.rs
+++ b/src/storage/either.rs
@@ -1,17 +1,13 @@
-use super::ImmutableStore;
+use super::{ImmutableStore, StorageConfig};
 use core::pin::Pin;
 use futures::{
     io::{AsyncRead, Error},
     task::{Context, Poll},
 };
-use kepler_lib::libipld::cid::multihash::Multihash;
+use kepler_lib::{libipld::cid::multihash::Multihash, resource::OrbitId};
 
 #[derive(Debug, Clone)]
-pub enum EitherStore<A, B>
-where
-    A: ImmutableStore,
-    B: ImmutableStore,
-{
+pub enum EitherStore<A, B> {
     A(A),
     B(B),
 }
@@ -62,7 +58,7 @@ where
 }
 
 #[derive(thiserror::Error, Debug)]
-pub enum EitherStoreError<A, B> {
+pub enum EitherError<A, B> {
     #[error(transparent)]
     A(A),
     #[error(transparent)]
@@ -76,7 +72,7 @@ where
     B: ImmutableStore,
 {
     type Readable = AsyncReadEither<A, B>;
-    type Error = EitherStoreError<A::Error, B::Error>;
+    type Error = EitherError<A::Error, B::Error>;
     async fn contains(&self, id: &Multihash) -> Result<bool, Self::Error> {
         match self {
             Self::A(l) => l.contains(id).await.map_err(Self::Error::A),
@@ -113,3 +109,32 @@ where
         }
     }
 }
+
+#[derive(Debug)]
+pub enum EitherConfig<A, B> {
+    A(A),
+    B(B),
+}
+
+#[async_trait]
+impl<A, B, SA, SB> StorageConfig<EitherStore<SA, SB>> for EitherConfig<A, B>
+where
+    A: StorageConfig<SA> + Sync,
+    B: StorageConfig<SB> + Sync,
+{
+    type Error = EitherError<A::Error, B::Error>;
+    async fn open(&self, orbit: &OrbitId) -> Result<EitherStore<SA, SB>, Self::Error> {
+        match self {
+            Self::A(a) => a
+                .open(orbit)
+                .await
+                .map(EitherStore::A)
+                .map_err(Self::Error::A),
+            Self::B(b) => b
+                .open(orbit)
+                .await
+                .map(EitherStore::B)
+                .map_err(Self::Error::B),
+        }
+    }
+}
diff --git a/src/storage/file_system.rs b/src/storage/file_system.rs
index e2830a14..a2ca578d 100644
--- a/src/storage/file_system.rs
+++ b/src/storage/file_system.rs
@@ -1,8 +1,12 @@
-use super::ImmutableStore;
-use kepler_lib::libipld::cid::{
-    multibase::{encode, Base},
-    multihash::Multihash,
+use super::{ImmutableStore, StorageConfig};
+use kepler_lib::{
+    libipld::cid::{
+        multibase::{encode, Base},
+        multihash::Multihash,
+    },
+    resource::OrbitId,
 };
+use serde::{Deserialize, Serialize};
 use std::{io::ErrorKind, path::PathBuf};
 use tokio::fs::{remove_file, File};
 use tokio_util::compat::{Compat, TokioAsyncReadCompatExt};
@@ -22,6 +26,21 @@ impl FileSystemStore {
     }
 }
 
+#[derive(Serialize, Deserialize, Debug, Clone, Hash, PartialEq, Eq)]
+pub struct FileSystemConfig {
+    pub path: PathBuf,
+}
+
+#[async_trait]
+impl StorageConfig<FileSystemStore> for FileSystemConfig {
+    type Error = std::convert::Infallible;
+    async fn open(&self, orbit: &OrbitId) -> Result<FileSystemStore, Self::Error> {
+        Ok(FileSystemStore::new(
+            self.path.join(orbit.get_cid().to_string()),
+        ))
+    }
+}
+
 #[async_trait]
 impl ImmutableStore for FileSystemStore {
     type Error = std::io::Error;
diff --git a/src/storage/mod.rs b/src/storage/mod.rs
index 073ab6cb..b869004c 100644
--- a/src/storage/mod.rs
+++ b/src/storage/mod.rs
@@ -38,6 +38,14 @@ pub enum DataStores {
     Local(Box<file_system::FileSystemStore>),
 }
 
+pub type BlockConfig = either::EitherConfig<s3::S3BlockConfig, file_system::FileSystemConfig>;
+
+#[async_trait]
+trait StorageConfig<S> {
+    type Error;
+    async fn open(&self, orbit: &OrbitId) -> Result<S, Self::Error>;
+}
+
 impl StorageUtils {
     pub fn new(config: config::BlockStorage) -> Self {
         Self { config }
@@ -46,7 +54,7 @@ impl StorageUtils {
     pub async fn healthcheck(&self) -> Result<()> {
         match &self.config {
             config::BlockStorage::S3(r) => {
-                let client = s3::S3BlockStore::new_(r.clone(), Cid::default());
+                let client = s3::S3BlockStore::new_(r, Cid::default());
                 // client.init().await
                 Ok(())
             }
diff --git a/src/storage/s3.rs b/src/storage/s3.rs
index 742310f3..eda45876 100644
--- a/src/storage/s3.rs
+++ b/src/storage/s3.rs
@@ -7,18 +7,23 @@ use aws_sdk_s3::{
 };
 use aws_smithy_http::{body::SdkBody, byte_stream::Error as ByteStreamError, endpoint::Endpoint};
 use futures::stream::{IntoAsyncRead, MapErr, TryStreamExt};
-use kepler_lib::libipld::cid::{
-    multibase::{encode, Base},
-    multihash::Multihash,
-    Cid,
+use kepler_lib::{
+    libipld::cid::{
+        multibase::{encode, Base},
+        multihash::Multihash,
+        Cid,
+    },
+    resource::OrbitId,
 };
 use regex::Regex;
 use rocket::{async_trait, http::hyper::Uri};
+use serde::{Deserialize, Serialize};
+use serde_with::{serde_as, DisplayFromStr};
 use std::{io::Error as IoError, path::PathBuf, str::FromStr};
 
 use crate::{
     config,
-    storage::{dynamodb::DynamoPinStore, ImmutableStore},
+    storage::{dynamodb::DynamoPinStore, ImmutableStore, StorageConfig},
 };
 
 #[derive(Debug)]
@@ -43,7 +48,25 @@ pub struct S3BlockStore {
     pub orbit: String,
 }
 
-pub fn new_client(config: config::S3BlockStorage) -> Client {
+#[serde_as]
+#[derive(Serialize, Deserialize, Debug, Clone, Hash, PartialEq, Eq)]
+pub struct S3BlockConfig {
+    pub bucket: String,
+    #[serde_as(as = "Option<DisplayFromStr>")]
+    #[serde(default)]
+    pub endpoint: Option<Uri>,
+    // pub dynamodb: DynamoStorage,
+}
+
+#[async_trait]
+impl StorageConfig<S3BlockStore> for S3BlockConfig {
+    type Error = std::convert::Infallible;
+    async fn open(&self, orbit: &OrbitId) -> Result<S3BlockStore, Self::Error> {
+        Ok(S3BlockStore::new_(self, orbit.get_cid()))
+    }
+}
+
+pub fn new_client(config: &S3BlockConfig) -> Client {
     let general_config = super::utils::aws_config();
     let sdk_config = aws_sdk_s3::config::Builder::from(&general_config);
     let sdk_config = match config.endpoint {
@@ -55,11 +78,11 @@ pub fn new_client(config: config::S3BlockStorage) -> Client {
 }
 
 impl S3DataStore {
-    pub fn new_(config: config::S3BlockStorage, orbit: Cid) -> Self {
+    pub fn new_(config: S3BlockConfig, orbit: Cid) -> Self {
         S3DataStore {
-            client: new_client(config.clone()),
+            client: new_client(&config),
             bucket: config.bucket,
-            dynamodb: DynamoPinStore::new(config.dynamodb, orbit),
+            dynamodb: todo!(),
             orbit,
         }
     }
@@ -107,10 +130,10 @@ impl S3DataStore {
 }
 
 impl S3BlockStore {
-    pub fn new_(config: config::S3BlockStorage, orbit: Cid) -> Self {
+    pub fn new_(config: &S3BlockConfig, orbit: Cid) -> Self {
         S3BlockStore {
-            client: new_client(config.clone()),
-            bucket: config.bucket,
+            client: new_client(config),
+            bucket: config.bucket.clone(),
             orbit: orbit.to_string(),
         }
     }

From 1a194a8c380b48ee876cf6cf42760442a8a4ec21 Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Tue, 1 Nov 2022 12:12:48 +0100
Subject: [PATCH 27/73] rename eitherstore -> either

---
 src/storage/either.rs | 26 ++++++--------------------
 src/storage/mod.rs    | 17 ++++++++---------
 2 files changed, 14 insertions(+), 29 deletions(-)

diff --git a/src/storage/either.rs b/src/storage/either.rs
index efef4007..9cc3e452 100644
--- a/src/storage/either.rs
+++ b/src/storage/either.rs
@@ -7,7 +7,7 @@ use futures::{
 use kepler_lib::{libipld::cid::multihash::Multihash, resource::OrbitId};
 
 #[derive(Debug, Clone)]
-pub enum EitherStore<A, B> {
+pub enum Either<A, B> {
     A(A),
     B(B),
 }
@@ -66,7 +66,7 @@ pub enum EitherError<A, B> {
 }
 
 #[async_trait]
-impl<A, B> ImmutableStore for EitherStore<A, B>
+impl<A, B> ImmutableStore for Either<A, B>
 where
     A: ImmutableStore,
     B: ImmutableStore,
@@ -110,31 +110,17 @@ where
     }
 }
 
-#[derive(Debug)]
-pub enum EitherConfig<A, B> {
-    A(A),
-    B(B),
-}
-
 #[async_trait]
-impl<A, B, SA, SB> StorageConfig<EitherStore<SA, SB>> for EitherConfig<A, B>
+impl<A, B, SA, SB> StorageConfig<Either<SA, SB>> for Either<A, B>
 where
     A: StorageConfig<SA> + Sync,
     B: StorageConfig<SB> + Sync,
 {
     type Error = EitherError<A::Error, B::Error>;
-    async fn open(&self, orbit: &OrbitId) -> Result<EitherStore<SA, SB>, Self::Error> {
+    async fn open(&self, orbit: &OrbitId) -> Result<Either<SA, SB>, Self::Error> {
         match self {
-            Self::A(a) => a
-                .open(orbit)
-                .await
-                .map(EitherStore::A)
-                .map_err(Self::Error::A),
-            Self::B(b) => b
-                .open(orbit)
-                .await
-                .map(EitherStore::B)
-                .map_err(Self::Error::B),
+            Self::A(a) => a.open(orbit).await.map(Either::A).map_err(Self::Error::A),
+            Self::B(b) => b.open(orbit).await.map(Either::B).map_err(Self::Error::B),
         }
     }
 }
diff --git a/src/storage/mod.rs b/src/storage/mod.rs
index b869004c..13081db0 100644
--- a/src/storage/mod.rs
+++ b/src/storage/mod.rs
@@ -26,11 +26,7 @@ pub struct StorageUtils {
     config: config::BlockStorage,
 }
 
-#[derive(Debug)]
-pub struct Repo;
-
-pub type BlockStores =
-    either::EitherStore<Box<s3::S3BlockStore>, Box<file_system::FileSystemStore>>;
+pub type BlockStores = either::Either<Box<s3::S3BlockStore>, Box<file_system::FileSystemStore>>;
 
 #[derive(Debug)]
 pub enum DataStores {
@@ -38,11 +34,11 @@ pub enum DataStores {
     Local(Box<file_system::FileSystemStore>),
 }
 
-pub type BlockConfig = either::EitherConfig<s3::S3BlockConfig, file_system::FileSystemConfig>;
+pub type BlockConfig = either::Either<s3::S3BlockConfig, file_system::FileSystemConfig>;
 
 #[async_trait]
-trait StorageConfig<S> {
-    type Error;
+pub trait StorageConfig<S> {
+    type Error: StdError;
     async fn open(&self, orbit: &OrbitId) -> Result<S, Self::Error>;
 }
 
@@ -252,9 +248,12 @@ pub enum VecReadError<E> {
     Read(futures::io::Error),
 }
 
+/// A Store implementing content-addressed storage
+/// Content is address by [Multihash][libipld::cid::multihash::Multihash] and represented as an
+/// [AsyncRead][futures::io::AsyncRead]-implementing type.
 #[async_trait]
 pub trait ImmutableStore: Send + Sync {
-    type Error: std::error::Error + Send + Sync;
+    type Error: StdError + Send + Sync;
     type Readable: futures::io::AsyncRead + Send + Sync;
     async fn contains(&self, id: &Multihash) -> Result<bool, Self::Error>;
     async fn write(

From 3fa79c299b6d5ac8078cc8e719105bb02e7c2eb8 Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Tue, 1 Nov 2022 13:36:15 +0100
Subject: [PATCH 28/73] add StorageConfig::create, Orbit::create and
 Orbit::open

---
 src/orbit.rs               | 79 +++++++++++++++++++++++++++-----------
 src/storage/either.rs      | 20 ++++++++--
 src/storage/file_system.rs | 24 ++++++++----
 src/storage/mod.rs         |  3 +-
 src/storage/s3.rs          |  5 ++-
 5 files changed, 96 insertions(+), 35 deletions(-)

diff --git a/src/orbit.rs b/src/orbit.rs
index 844e7bfb..9240620f 100644
--- a/src/orbit.rs
+++ b/src/orbit.rs
@@ -3,7 +3,7 @@ use crate::{
     config,
     kv::{behaviour::BehaviourProcess, Service as KVService, Store},
     manifest::Manifest,
-    storage::{BlockStores, ImmutableStore},
+    storage::{BlockStores, ImmutableStore, StorageConfig},
 };
 use anyhow::{anyhow, Result};
 use kepler_lib::libipld::cid::{
@@ -66,37 +66,70 @@ pub struct Orbit<B> {
     pub capabilities: CapService<B>,
 }
 
+#[derive(Clone, Debug)]
+pub struct PeerConfig {
+    kp: Ed25519Keypair,
+    manifest: Manifest,
+    relay: Option<(PeerId, Multiaddr)>,
+}
+
+#[derive(Clone, Debug)]
+pub struct StoreConfigs<B, I = config::IndexStorage> {
+    blocks: B,
+    index: I,
+}
+
 impl<B> Orbit<B>
 where
-    B: ImmutableStore,
+    B: ImmutableStore + Clone,
 {
-    async fn new(
-        config: &config::Config,
-        kp: Ed25519Keypair,
-        manifest: Manifest,
-        relay: Option<(PeerId, Multiaddr)>,
-    ) -> anyhow::Result<Self> {
-        let id = manifest.id().get_cid();
-        let local_peer_id = PeerId::from_public_key(&PublicKey::Ed25519(kp.public()));
-
-        let storage_utils = StorageUtils::new(config.storage.blocks.clone());
-
-        // TODO config into block store
-        let blocks = todo!();
-        let service_store = Store::new(id, blocks.clone(), config.storage.indexes.clone()).await?;
+    async fn open<C>(stores: &StoreConfigs<C>, peer: PeerConfig) -> anyhow::Result<Option<Self>>
+    where
+        C: StorageConfig<B>,
+        C::Error: 'static + Sync + Send,
+        B::Error: 'static,
+    {
+        let id = peer.manifest.id().get_cid();
+        let local_peer_id = PeerId::from_public_key(&PublicKey::Ed25519(peer.kp.public()));
+
+        let blocks = match stores.blocks.open(peer.manifest.id()).await? {
+            Some(b) => b,
+            None => return Ok(None),
+        };
+        let service_store = Store::new(id, blocks.clone(), stores.index.clone()).await?;
+        let service = KVService::start(service_store).await?;
+
+        let cap_store =
+            CapStore::new(peer.manifest.id(), blocks.clone(), stores.index.clone()).await?;
+        let capabilities = CapService::start(cap_store).await?;
+
+        Ok(Some(Orbit {
+            service,
+            manifest: peer.manifest,
+            capabilities,
+        }))
+    }
+
+    async fn create<C>(stores: &StoreConfigs<C>, peer: PeerConfig) -> anyhow::Result<Self>
+    where
+        C: StorageConfig<B>,
+        C::Error: 'static + Sync + Send,
+        B::Error: 'static,
+    {
+        let id = peer.manifest.id().get_cid();
+        let local_peer_id = PeerId::from_public_key(&PublicKey::Ed25519(peer.kp.public()));
+
+        let blocks = stores.blocks.create(peer.manifest.id()).await?;
+        let service_store = Store::new(id, blocks.clone(), stores.index.clone()).await?;
         let service = KVService::start(service_store).await?;
 
-        let cap_store = CapStore::new(
-            manifest.id(),
-            blocks.clone(),
-            config.storage.indexes.clone(),
-        )
-        .await?;
+        let cap_store =
+            CapStore::new(peer.manifest.id(), blocks.clone(), stores.index.clone()).await?;
         let capabilities = CapService::start(cap_store).await?;
 
         Ok(Orbit {
             service,
-            manifest,
+            manifest: peer.manifest,
             capabilities,
         })
     }
diff --git a/src/storage/either.rs b/src/storage/either.rs
index 9cc3e452..4a0c2d52 100644
--- a/src/storage/either.rs
+++ b/src/storage/either.rs
@@ -117,10 +117,24 @@ where
     B: StorageConfig<SB> + Sync,
 {
     type Error = EitherError<A::Error, B::Error>;
-    async fn open(&self, orbit: &OrbitId) -> Result<Either<SA, SB>, Self::Error> {
+    async fn open(&self, orbit: &OrbitId) -> Result<Option<Either<SA, SB>>, Self::Error> {
         match self {
-            Self::A(a) => a.open(orbit).await.map(Either::A).map_err(Self::Error::A),
-            Self::B(b) => b.open(orbit).await.map(Either::B).map_err(Self::Error::B),
+            Self::A(a) => a
+                .open(orbit)
+                .await
+                .map(|o| o.map(Either::A))
+                .map_err(Self::Error::A),
+            Self::B(b) => b
+                .open(orbit)
+                .await
+                .map(|o| o.map(Either::B))
+                .map_err(Self::Error::B),
+        }
+    }
+    async fn create(&self, orbit: &OrbitId) -> Result<Either<SA, SB>, Self::Error> {
+        match self {
+            Self::A(a) => a.create(orbit).await.map(Either::A).map_err(Self::Error::A),
+            Self::B(b) => b.create(orbit).await.map(Either::B).map_err(Self::Error::B),
         }
     }
 }
diff --git a/src/storage/file_system.rs b/src/storage/file_system.rs
index a2ca578d..b034341e 100644
--- a/src/storage/file_system.rs
+++ b/src/storage/file_system.rs
@@ -8,7 +8,7 @@ use kepler_lib::{
 };
 use serde::{Deserialize, Serialize};
 use std::{io::ErrorKind, path::PathBuf};
-use tokio::fs::{remove_file, File};
+use tokio::fs::{create_dir_all, remove_file, File};
 use tokio_util::compat::{Compat, TokioAsyncReadCompatExt};
 
 #[derive(Debug, Clone)]
@@ -28,16 +28,26 @@ impl FileSystemStore {
 
 #[derive(Serialize, Deserialize, Debug, Clone, Hash, PartialEq, Eq)]
 pub struct FileSystemConfig {
-    pub path: PathBuf,
+    path: PathBuf,
 }
 
 #[async_trait]
 impl StorageConfig<FileSystemStore> for FileSystemConfig {
-    type Error = std::convert::Infallible;
-    async fn open(&self, orbit: &OrbitId) -> Result<FileSystemStore, Self::Error> {
-        Ok(FileSystemStore::new(
-            self.path.join(orbit.get_cid().to_string()),
-        ))
+    type Error = std::io::Error;
+    async fn open(&self, orbit: &OrbitId) -> Result<Option<FileSystemStore>, Self::Error> {
+        let path = self.path.join(orbit.get_cid().to_string()).join("blocks");
+        if path.is_dir() {
+            Ok(Some(FileSystemStore::new(path)))
+        } else {
+            Ok(None)
+        }
+    }
+    async fn create(&self, orbit: &OrbitId) -> Result<FileSystemStore, Self::Error> {
+        let path = self.path.join(orbit.get_cid().to_string()).join("blocks");
+        if !path.is_dir() {
+            create_dir_all(&path).await?;
+        }
+        Ok(FileSystemStore::new(path))
     }
 }
 
diff --git a/src/storage/mod.rs b/src/storage/mod.rs
index 13081db0..9a3a20fd 100644
--- a/src/storage/mod.rs
+++ b/src/storage/mod.rs
@@ -39,7 +39,8 @@ pub type BlockConfig = either::Either<s3::S3BlockConfig, file_system::FileSystem
 #[async_trait]
 pub trait StorageConfig<S> {
     type Error: StdError;
-    async fn open(&self, orbit: &OrbitId) -> Result<S, Self::Error>;
+    async fn open(&self, orbit: &OrbitId) -> Result<Option<S>, Self::Error>;
+    async fn create(&self, orbit: &OrbitId) -> Result<S, Self::Error>;
 }
 
 impl StorageUtils {
diff --git a/src/storage/s3.rs b/src/storage/s3.rs
index eda45876..80b0620b 100644
--- a/src/storage/s3.rs
+++ b/src/storage/s3.rs
@@ -61,7 +61,10 @@ pub struct S3BlockConfig {
 #[async_trait]
 impl StorageConfig<S3BlockStore> for S3BlockConfig {
     type Error = std::convert::Infallible;
-    async fn open(&self, orbit: &OrbitId) -> Result<S3BlockStore, Self::Error> {
+    async fn open(&self, orbit: &OrbitId) -> Result<Option<S3BlockStore>, Self::Error> {
+        Ok(Some(S3BlockStore::new_(self, orbit.get_cid())))
+    }
+    async fn create(&self, orbit: &OrbitId) -> Result<S3BlockStore, Self::Error> {
         Ok(S3BlockStore::new_(self, orbit.get_cid()))
     }
 }

From b22942fbea880d7ec167004807d3ef62b03e175a Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Tue, 1 Nov 2022 17:47:00 +0100
Subject: [PATCH 29/73] add OrbitPeerConfig and builder

---
 Cargo.toml   |  1 +
 src/orbit.rs | 53 +++++++++++++++++++++++++---------------------------
 2 files changed, 26 insertions(+), 28 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index 3e7ab26d..8fe0cd3b 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -19,6 +19,7 @@ base64 = "0.13"
 bincode = "1.3"
 bs58 = "0.4"
 cached = "0.34"
+derive_builder = "0.11"
 futures = { default-features = false, version = "0.3", features = ["alloc", "std"] }
 hex = "0.4"
 hyper = "0.14" # Prometheus server
diff --git a/src/orbit.rs b/src/orbit.rs
index 9240620f..051db607 100644
--- a/src/orbit.rs
+++ b/src/orbit.rs
@@ -3,9 +3,10 @@ use crate::{
     config,
     kv::{behaviour::BehaviourProcess, Service as KVService, Store},
     manifest::Manifest,
-    storage::{BlockStores, ImmutableStore, StorageConfig},
+    storage::{BlockConfig, BlockStores, ImmutableStore, StorageConfig},
 };
 use anyhow::{anyhow, Result};
+use derive_builder::Builder;
 use kepler_lib::libipld::cid::{
     multihash::{Code, MultihashDigest},
     Cid,
@@ -19,9 +20,7 @@ use libp2p::{
 use rocket::tokio::task::JoinHandle;
 
 use cached::proc_macro::cached;
-use std::{convert::TryFrom, ops::Deref};
-
-use super::storage::StorageUtils;
+use std::{convert::TryFrom, error::Error as StdError, ops::Deref};
 
 #[derive(Debug)]
 pub struct AbortOnDrop<T>(JoinHandle<T>);
@@ -66,16 +65,17 @@ pub struct Orbit<B> {
     pub capabilities: CapService<B>,
 }
 
-#[derive(Clone, Debug)]
-pub struct PeerConfig {
-    kp: Ed25519Keypair,
+#[derive(Clone, Debug, Builder)]
+pub struct OrbitPeerConfig<B, I = config::IndexStorage> {
+    #[builder(setter(into))]
+    identity: Ed25519Keypair,
+    #[builder(setter(into))]
     manifest: Manifest,
+    #[builder(setter(into))]
     relay: Option<(PeerId, Multiaddr)>,
-}
-
-#[derive(Clone, Debug)]
-pub struct StoreConfigs<B, I = config::IndexStorage> {
+    #[builder(setter(into))]
     blocks: B,
+    #[builder(setter(into))]
     index: I,
 }
 
@@ -83,60 +83,57 @@ impl<B> Orbit<B>
 where
     B: ImmutableStore + Clone,
 {
-    async fn open<C>(stores: &StoreConfigs<C>, peer: PeerConfig) -> anyhow::Result<Option<Self>>
+    async fn open<C>(config: &OrbitPeerConfig<C>) -> anyhow::Result<Option<Self>>
     where
         C: StorageConfig<B>,
         C::Error: 'static + Sync + Send,
         B::Error: 'static,
     {
-        let id = peer.manifest.id().get_cid();
-        let local_peer_id = PeerId::from_public_key(&PublicKey::Ed25519(peer.kp.public()));
+        let id = config.manifest.id().get_cid();
+        let local_peer_id = PeerId::from_public_key(&PublicKey::Ed25519(config.identity.public()));
 
-        let blocks = match stores.blocks.open(peer.manifest.id()).await? {
+        let blocks = match config.blocks.open(config.manifest.id()).await? {
             Some(b) => b,
             None => return Ok(None),
         };
-        let service_store = Store::new(id, blocks.clone(), stores.index.clone()).await?;
+        let service_store = Store::new(id, blocks.clone(), config.index.clone()).await?;
         let service = KVService::start(service_store).await?;
 
         let cap_store =
-            CapStore::new(peer.manifest.id(), blocks.clone(), stores.index.clone()).await?;
+            CapStore::new(config.manifest.id(), blocks.clone(), config.index.clone()).await?;
         let capabilities = CapService::start(cap_store).await?;
 
         Ok(Some(Orbit {
             service,
-            manifest: peer.manifest,
+            manifest: config.manifest.clone(),
             capabilities,
         }))
     }
 
-    async fn create<C>(stores: &StoreConfigs<C>, peer: PeerConfig) -> anyhow::Result<Self>
+    async fn create<C>(config: &OrbitPeerConfig<C>) -> anyhow::Result<Self>
     where
         C: StorageConfig<B>,
         C::Error: 'static + Sync + Send,
         B::Error: 'static,
     {
-        let id = peer.manifest.id().get_cid();
-        let local_peer_id = PeerId::from_public_key(&PublicKey::Ed25519(peer.kp.public()));
+        let id = config.manifest.id().get_cid();
+        let local_peer_id = PeerId::from_public_key(&PublicKey::Ed25519(config.identity.public()));
 
-        let blocks = stores.blocks.create(peer.manifest.id()).await?;
-        let service_store = Store::new(id, blocks.clone(), stores.index.clone()).await?;
+        let blocks = config.blocks.create(config.manifest.id()).await?;
+        let service_store = Store::new(id, blocks.clone(), config.index.clone()).await?;
         let service = KVService::start(service_store).await?;
 
         let cap_store =
-            CapStore::new(peer.manifest.id(), blocks.clone(), stores.index.clone()).await?;
+            CapStore::new(config.manifest.id(), blocks.clone(), config.index.clone()).await?;
         let capabilities = CapService::start(cap_store).await?;
 
         Ok(Orbit {
             service,
-            manifest: peer.manifest,
+            manifest: config.manifest.clone(),
             capabilities,
         })
     }
 
-    // pub async fn connect(&self, node: MultiaddrWithPeerId) -> anyhow::Result<()> {
-    // self.service.store.ipfs.connect(node).await
-    // }
 }
 
 // Using Option to distinguish when the orbit already exists from a hard error

From e7b91258ad2220de5e9cc766b5b2c229412730bc Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Tue, 1 Nov 2022 17:48:13 +0100
Subject: [PATCH 30/73] add ProviderUtils and refactor orbit load/create

---
 src/orbit.rs               |   8 +++
 src/storage/either.rs      |  46 ++++++++++++++--
 src/storage/file_system.rs |  63 ++++++++++++++++++++--
 src/storage/mod.rs         |   4 +-
 src/storage/s3.rs          | 107 ++++++++++++++++++++++++++++++++++++-
 5 files changed, 217 insertions(+), 11 deletions(-)

diff --git a/src/orbit.rs b/src/orbit.rs
index 051db607..a6978ad3 100644
--- a/src/orbit.rs
+++ b/src/orbit.rs
@@ -133,7 +133,15 @@ where
             capabilities,
         })
     }
+}
 
+#[async_trait]
+pub trait ProviderUtils {
+    type Error: StdError;
+    async fn exists(&self, orbit: &OrbitId) -> Result<bool, Self::Error>;
+    async fn relay_key_pair(&self) -> Result<Ed25519Keypair, Self::Error>;
+    async fn key_pair(&self, orbit: &OrbitId) -> Result<Option<Ed25519Keypair>, Self::Error>;
+    async fn setup_orbit(&self, orbit: &OrbitId, key: &Ed25519Keypair) -> Result<(), Self::Error>;
 }
 
 // Using Option to distinguish when the orbit already exists from a hard error
diff --git a/src/storage/either.rs b/src/storage/either.rs
index 4a0c2d52..409d11f0 100644
--- a/src/storage/either.rs
+++ b/src/storage/either.rs
@@ -1,12 +1,19 @@
-use super::{ImmutableStore, StorageConfig};
+use crate::{
+    orbit::ProviderUtils,
+    storage::{ImmutableStore, StorageConfig},
+};
 use core::pin::Pin;
 use futures::{
     io::{AsyncRead, Error},
     task::{Context, Poll},
 };
-use kepler_lib::{libipld::cid::multihash::Multihash, resource::OrbitId};
+use kepler_lib::{
+    libipld::cid::{multihash::Multihash, Cid},
+    resource::OrbitId,
+};
+use libp2p::identity::ed25519::Keypair as Ed25519Keypair;
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Hash, PartialEq, Eq)]
 pub enum Either<A, B> {
     A(A),
     B(B),
@@ -138,3 +145,36 @@ where
         }
     }
 }
+
+#[async_trait]
+impl<A, B> ProviderUtils for Either<A, B>
+where
+    A: ProviderUtils + Sync,
+    B: ProviderUtils + Sync,
+{
+    type Error = EitherError<A::Error, B::Error>;
+    async fn exists(&self, orbit: &OrbitId) -> Result<bool, Self::Error> {
+        match self {
+            Self::A(a) => a.exists(orbit).await.map_err(Self::Error::A),
+            Self::B(b) => b.exists(orbit).await.map_err(Self::Error::B),
+        }
+    }
+    async fn relay_key_pair(&self) -> Result<Ed25519Keypair, Self::Error> {
+        match self {
+            Self::A(a) => a.relay_key_pair().await.map_err(Self::Error::A),
+            Self::B(b) => b.relay_key_pair().await.map_err(Self::Error::B),
+        }
+    }
+    async fn key_pair(&self, orbit: &OrbitId) -> Result<Option<Ed25519Keypair>, Self::Error> {
+        match self {
+            Self::A(a) => a.key_pair(orbit).await.map_err(Self::Error::A),
+            Self::B(b) => b.key_pair(orbit).await.map_err(Self::Error::B),
+        }
+    }
+    async fn setup_orbit(&self, orbit: &OrbitId, key: &Ed25519Keypair) -> Result<(), Self::Error> {
+        match self {
+            Self::A(a) => a.setup_orbit(orbit, key).await.map_err(Self::Error::A),
+            Self::B(b) => b.setup_orbit(orbit, key).await.map_err(Self::Error::B),
+        }
+    }
+}
diff --git a/src/storage/file_system.rs b/src/storage/file_system.rs
index b034341e..943c6eed 100644
--- a/src/storage/file_system.rs
+++ b/src/storage/file_system.rs
@@ -1,4 +1,7 @@
-use super::{ImmutableStore, StorageConfig};
+use crate::{
+    orbit::ProviderUtils,
+    storage::{ImmutableStore, StorageConfig},
+};
 use kepler_lib::{
     libipld::cid::{
         multibase::{encode, Base},
@@ -6,9 +9,13 @@ use kepler_lib::{
     },
     resource::OrbitId,
 };
+use libp2p::identity::{ed25519::Keypair as Ed25519Keypair, error::DecodingError};
 use serde::{Deserialize, Serialize};
-use std::{io::ErrorKind, path::PathBuf};
-use tokio::fs::{create_dir_all, remove_file, File};
+use std::{
+    io::{Error as IoError, ErrorKind},
+    path::PathBuf,
+};
+use tokio::fs::{create_dir_all, read, remove_file, write, File};
 use tokio_util::compat::{Compat, TokioAsyncReadCompatExt};
 
 #[derive(Debug, Clone)]
@@ -33,7 +40,7 @@ pub struct FileSystemConfig {
 
 #[async_trait]
 impl StorageConfig<FileSystemStore> for FileSystemConfig {
-    type Error = std::io::Error;
+    type Error = IoError;
     async fn open(&self, orbit: &OrbitId) -> Result<Option<FileSystemStore>, Self::Error> {
         let path = self.path.join(orbit.get_cid().to_string()).join("blocks");
         if path.is_dir() {
@@ -51,9 +58,55 @@ impl StorageConfig<FileSystemStore> for FileSystemConfig {
     }
 }
 
+#[derive(thiserror::Error, Debug)]
+pub enum ProviderError {
+    #[error(transparent)]
+    Io(#[from] IoError),
+    #[error(transparent)]
+    KeypairDecode(#[from] DecodingError),
+}
+
+#[async_trait]
+impl ProviderUtils for FileSystemConfig {
+    type Error = ProviderError;
+    async fn exists(&self, orbit: &OrbitId) -> Result<bool, Self::Error> {
+        Ok(self
+            .path
+            .join(orbit.get_cid().to_string())
+            .join("blocks")
+            .is_dir())
+    }
+    async fn relay_key_pair(&self) -> Result<Ed25519Keypair, Self::Error> {
+        let path = self.path.join("kp");
+        match read(path).await {
+            Ok(mut k) => Ok(Ed25519Keypair::decode(&mut k)?),
+            Err(e) if e.kind() == ErrorKind::NotFound => {
+                let k = Ed25519Keypair::generate();
+                write(path, k.encode()).await?;
+                Ok(k)
+            }
+            Err(e) => Err(e.into()),
+        }
+    }
+    async fn key_pair(&self, orbit: &OrbitId) -> Result<Option<Ed25519Keypair>, Self::Error> {
+        match read(self.path.join(orbit.get_cid().to_string()).join("kp")).await {
+            Ok(mut k) => Ok(Some(Ed25519Keypair::decode(&mut k)?)),
+            Err(e) if e.kind() == ErrorKind::NotFound => Ok(None),
+            Err(e) => Err(e.into()),
+        }
+    }
+    async fn setup_orbit(&self, orbit: &OrbitId, key: &Ed25519Keypair) -> Result<(), Self::Error> {
+        let dir = self.path.join(orbit.get_cid().to_string());
+        create_dir_all(&dir).await?;
+        write(dir.join("kp"), key.encode()).await?;
+        write(dir.join("id"), orbit.to_string()).await?;
+        Ok(())
+    }
+}
+
 #[async_trait]
 impl ImmutableStore for FileSystemStore {
-    type Error = std::io::Error;
+    type Error = IoError;
     type Readable = Compat<File>;
     async fn contains(&self, id: &Multihash) -> Result<bool, Self::Error> {
         Ok(self.get_path(id).exists())
diff --git a/src/storage/mod.rs b/src/storage/mod.rs
index 9a3a20fd..b7b69239 100644
--- a/src/storage/mod.rs
+++ b/src/storage/mod.rs
@@ -8,7 +8,7 @@ use kepler_lib::libipld::cid::{multibase::Base, multihash::Multihash, Cid};
 use kepler_lib::resource::OrbitId;
 use libp2p::identity::ed25519::Keypair as Ed25519Keypair;
 use rocket::tokio::fs;
-use std::{collections::HashMap, path::PathBuf, str::FromStr};
+use std::{collections::HashMap, error::Error as StdError, path::PathBuf, str::FromStr};
 use tracing::instrument;
 
 mod dynamodb;
@@ -26,7 +26,7 @@ pub struct StorageUtils {
     config: config::BlockStorage,
 }
 
-pub type BlockStores = either::Either<Box<s3::S3BlockStore>, Box<file_system::FileSystemStore>>;
+pub type BlockStores = either::Either<s3::S3BlockStore, file_system::FileSystemStore>;
 
 #[derive(Debug)]
 pub enum DataStores {
diff --git a/src/storage/s3.rs b/src/storage/s3.rs
index 80b0620b..efdb89ad 100644
--- a/src/storage/s3.rs
+++ b/src/storage/s3.rs
@@ -1,6 +1,6 @@
 use anyhow::Error;
 use aws_sdk_s3::{
-    error::{GetObjectError, GetObjectErrorKind},
+    error::{GetObjectError, GetObjectErrorKind, PutObjectError},
     types::{ByteStream, SdkError},
     Client, // Config,
     Error as S3Error,
@@ -15,6 +15,7 @@ use kepler_lib::{
     },
     resource::OrbitId,
 };
+use libp2p::identity::{ed25519::Keypair as Ed25519Keypair, error::DecodingError};
 use regex::Regex;
 use rocket::{async_trait, http::hyper::Uri};
 use serde::{Deserialize, Serialize};
@@ -23,6 +24,7 @@ use std::{io::Error as IoError, path::PathBuf, str::FromStr};
 
 use crate::{
     config,
+    orbit::ProviderUtils,
     storage::{dynamodb::DynamoPinStore, ImmutableStore, StorageConfig},
 };
 
@@ -69,6 +71,109 @@ impl StorageConfig<S3BlockStore> for S3BlockConfig {
     }
 }
 
+#[derive(thiserror::Error, Debug)]
+pub enum ProviderError {
+    #[error(transparent)]
+    S3(#[from] S3Error),
+    #[error(transparent)]
+    KeypairDecode(#[from] DecodingError),
+    #[error(transparent)]
+    ByteStream(#[from] ByteStreamError),
+}
+
+impl From<SdkError<GetObjectError>> for ProviderError {
+    fn from(e: SdkError<GetObjectError>) -> Self {
+        Self::S3(e.into())
+    }
+}
+
+impl From<SdkError<PutObjectError>> for ProviderError {
+    fn from(e: SdkError<PutObjectError>) -> Self {
+        Self::S3(e.into())
+    }
+}
+
+#[async_trait]
+impl ProviderUtils for S3BlockConfig {
+    type Error = ProviderError;
+    async fn exists(&self, orbit: &OrbitId) -> Result<bool, Self::Error> {
+        self.key_pair(orbit).await.map(|o| o.is_some())
+    }
+    async fn relay_key_pair(&self) -> Result<Ed25519Keypair, Self::Error> {
+        let client = new_client(&self);
+        match client
+            .get_object()
+            .bucket(self.bucket)
+            .key("kp")
+            .send()
+            .await
+        {
+            Ok(o) => Ok(Ed25519Keypair::decode(
+                &mut o.body.collect().await?.into_bytes().to_vec(),
+            )?),
+            Err(SdkError::ServiceError {
+                err:
+                    GetObjectError {
+                        kind: GetObjectErrorKind::NoSuchKey(_),
+                        ..
+                    },
+                ..
+            }) => {
+                let kp = Ed25519Keypair::generate();
+                client
+                    .put_object()
+                    .bucket(self.bucket)
+                    .key("kp")
+                    .body(ByteStream::new(SdkBody::from(kp.encode().to_vec())))
+                    .send()
+                    .await?;
+                Ok(kp)
+            }
+            Err(e) => Err(e.into()),
+        }
+    }
+    async fn key_pair(&self, orbit: &OrbitId) -> Result<Option<Ed25519Keypair>, Self::Error> {
+        match new_client(&self)
+            .get_object()
+            .bucket(self.bucket)
+            .key(format!("{}/keypair", orbit.get_cid()))
+            .send()
+            .await
+        {
+            Ok(o) => Ok(Some(Ed25519Keypair::decode(
+                &mut o.body.collect().await?.into_bytes().to_vec(),
+            )?)),
+            Err(SdkError::ServiceError {
+                err:
+                    GetObjectError {
+                        kind: GetObjectErrorKind::NoSuchKey(_),
+                        ..
+                    },
+                ..
+            }) => Ok(None),
+            Err(e) => Err(e.into()),
+        }
+    }
+    async fn setup_orbit(&self, orbit: &OrbitId, key: &Ed25519Keypair) -> Result<(), Self::Error> {
+        let client = new_client(&self);
+        client
+            .put_object()
+            .bucket(self.bucket)
+            .key(format!("{}/keypair", orbit.get_cid()))
+            .body(ByteStream::new(SdkBody::from(key.encode().to_vec())))
+            .send()
+            .await?;
+        client
+            .put_object()
+            .bucket(self.bucket)
+            .key(format!("{}/orbit_url", orbit.get_cid()))
+            .body(ByteStream::new(SdkBody::from(orbit.to_string())))
+            .send()
+            .await?;
+        Ok(())
+    }
+}
+
 pub fn new_client(config: &S3BlockConfig) -> Client {
     let general_config = super::utils::aws_config();
     let sdk_config = aws_sdk_s3::config::Builder::from(&general_config);

From cb90608f13cd1e4f927c6dc0e2d5ed995af12baf Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Wed, 2 Nov 2022 13:15:42 +0100
Subject: [PATCH 31/73] remove storageutils impl

---
 Cargo.lock         | 516 ++++++++++++++++-----------------------------
 src/storage/mod.rs | 198 -----------------
 2 files changed, 184 insertions(+), 530 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index f0c3a40d..3747068e 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -345,9 +345,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
 
 [[package]]
 name = "aws-config"
-version = "0.9.0"
+version = "0.49.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2da63196d2d0dd38667b404459a35d32562a8d83c1f46c5b789ab89ab176fd53"
+checksum = "b309b2154d224728d845a958c580834f24213037ed61b195da80c0b0fc7469fa"
 dependencies = [
  "aws-http",
  "aws-sdk-sso",
@@ -364,6 +364,7 @@ dependencies = [
  "http",
  "hyper",
  "ring",
+ "time 0.3.15",
  "tokio",
  "tower",
  "tracing",
@@ -372,11 +373,12 @@ dependencies = [
 
 [[package]]
 name = "aws-endpoint"
-version = "0.9.0"
+version = "0.49.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5279590d48e92b287f864e099c7e851af03a5e184a57cec0959872cee297c7a0"
+checksum = "76f35c8f5877ad60db4f0d9dcdfbcb2233a8cc539f9e568df39ee0581ec62e89"
 dependencies = [
  "aws-smithy-http",
+ "aws-smithy-types",
  "aws-types",
  "http",
  "regex",
@@ -385,24 +387,27 @@ dependencies = [
 
 [[package]]
 name = "aws-http"
-version = "0.9.0"
+version = "0.49.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e7046bdd807c70caf28d6dbc69b9d6d8dda1728577866d3ff3862de585b8b0eb"
+checksum = "2f5422c9632d887968ccb66e2871a6d190d6104e276034912bee72ef58a5d890"
 dependencies = [
  "aws-smithy-http",
  "aws-smithy-types",
  "aws-types",
+ "bytes",
  "http",
+ "http-body",
  "lazy_static",
  "percent-encoding",
+ "pin-project-lite",
  "tracing",
 ]
 
 [[package]]
 name = "aws-sdk-dynamodb"
-version = "0.9.0"
+version = "0.19.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "762a92e68a3a99d43593ff9b4c00dac630c1e5cfd04ed15e33f235c9c9125a09"
+checksum = "0d757928810bbc52fe8bc3005689fb2465d4d62c280054693ec6d0162ce33bbc"
 dependencies = [
  "aws-endpoint",
  "aws-http",
@@ -423,15 +428,16 @@ dependencies = [
 
 [[package]]
 name = "aws-sdk-s3"
-version = "0.9.0"
+version = "0.19.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c4fd99b22cbdb894925468005ad55defcfe0ce294fadcc5f7be9d9119646b0de"
+checksum = "a4d31765abb258c501d5572ebce43dee524b4b3b6256cb8b4c78534898dc205b"
 dependencies = [
  "aws-endpoint",
  "aws-http",
  "aws-sig-auth",
  "aws-sigv4",
  "aws-smithy-async",
+ "aws-smithy-checksums",
  "aws-smithy-client",
  "aws-smithy-eventstream",
  "aws-smithy-http",
@@ -440,17 +446,19 @@ dependencies = [
  "aws-smithy-xml",
  "aws-types",
  "bytes",
+ "bytes-utils",
  "http",
- "md5",
+ "http-body",
  "tokio-stream",
  "tower",
+ "tracing",
 ]
 
 [[package]]
 name = "aws-sdk-sso"
-version = "0.9.0"
+version = "0.19.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "96f9038b498944025a39e426ae38f64e3e8481a9d675469580e1de7397b46ed5"
+checksum = "e2cc8b50281e1350d0b5c7207c2ce53c6721186ad196472caff4f20fa4b42e96"
 dependencies = [
  "aws-endpoint",
  "aws-http",
@@ -470,9 +478,9 @@ dependencies = [
 
 [[package]]
 name = "aws-sdk-sts"
-version = "0.9.0"
+version = "0.19.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e717e67debcd7f9d87563d08e7d40e3c5c28634a8badc491650d5ad2305befd3"
+checksum = "d6179f13c9fbab3226860f377354dece860e34ff129b69c7c1b0fa828d1e9c76"
 dependencies = [
  "aws-endpoint",
  "aws-http",
@@ -492,24 +500,23 @@ dependencies = [
 
 [[package]]
 name = "aws-sig-auth"
-version = "0.9.0"
+version = "0.49.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a0e6e4ba09f502057ad6a4ebf3627f9dae8402e366cf7b36ca1c09cbff8b5834"
+checksum = "b16f4d70c9c865af392eb40cacfe2bec3fa18f651fbdf49919cfc1dda13b189e"
 dependencies = [
  "aws-sigv4",
  "aws-smithy-eventstream",
  "aws-smithy-http",
  "aws-types",
  "http",
- "thiserror",
  "tracing",
 ]
 
 [[package]]
 name = "aws-sigv4"
-version = "0.9.0"
+version = "0.49.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ea07a5a108ee538793d681d608057218df95c5575f6c0699a1973c27a09334b2"
+checksum = "8d33790cecae42b999d197074c8a19e9b96b9e346284a6f93989e7489c9fa0f5"
 dependencies = [
  "aws-smithy-eventstream",
  "aws-smithy-http",
@@ -527,9 +534,9 @@ dependencies = [
 
 [[package]]
 name = "aws-smithy-async"
-version = "0.39.0"
+version = "0.49.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "66ab5373d24e1651860240f122a8d956f7a2094d4553c78979617a7fac640030"
+checksum = "bc604f278bae64bbd15854baa9c46ed69a56dfb0669d04aab80974749f2d6599"
 dependencies = [
  "futures-util",
  "pin-project-lite",
@@ -537,11 +544,32 @@ dependencies = [
  "tokio-stream",
 ]
 
+[[package]]
+name = "aws-smithy-checksums"
+version = "0.49.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4b402da39bc5aae618b70a9b8d828acad21fe4a3a73b82c0205b89db55d71ce8"
+dependencies = [
+ "aws-smithy-http",
+ "aws-smithy-types",
+ "bytes",
+ "crc32c",
+ "crc32fast",
+ "hex",
+ "http",
+ "http-body",
+ "md-5 0.10.5",
+ "pin-project-lite",
+ "sha1",
+ "sha2 0.10.6",
+ "tracing",
+]
+
 [[package]]
 name = "aws-smithy-client"
-version = "0.39.0"
+version = "0.49.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "88e8a92747322eace67f666402a5949da27675f60a2b9098b84b63edef8e6980"
+checksum = "ec39585f8274fa543ad5c63cc09cbd435666be16b2cf99e4e07be5cf798bc050"
 dependencies = [
  "aws-smithy-async",
  "aws-smithy-http",
@@ -554,7 +582,6 @@ dependencies = [
  "hyper",
  "hyper-rustls",
  "lazy_static",
- "pin-project",
  "pin-project-lite",
  "tokio",
  "tower",
@@ -563,9 +590,9 @@ dependencies = [
 
 [[package]]
 name = "aws-smithy-eventstream"
-version = "0.39.0"
+version = "0.49.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "775b1de8d55fd1cda393c3d81cb5c3dc0e1cac38170883049e2d7a8e16cefad1"
+checksum = "98c2a7b9490fd2bc7af3a1c486ae921102d7234d1fa5e7d91039068e7af48a01"
 dependencies = [
  "aws-smithy-types",
  "bytes",
@@ -574,9 +601,9 @@ dependencies = [
 
 [[package]]
 name = "aws-smithy-http"
-version = "0.39.0"
+version = "0.49.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "579d0c2ae96c700499c5330f082c4170b0535835f01eb845056324aa0abd04b4"
+checksum = "014a0ef5c4508fc2f6a9d3925c214725af19f020ea388db48e20196cc4cc9d6d"
 dependencies = [
  "aws-smithy-eventstream",
  "aws-smithy-types",
@@ -588,51 +615,51 @@ dependencies = [
  "hyper",
  "once_cell",
  "percent-encoding",
- "pin-project",
+ "pin-project-lite",
  "tokio",
- "tokio-util 0.6.10",
+ "tokio-util",
  "tracing",
 ]
 
 [[package]]
 name = "aws-smithy-http-tower"
-version = "0.39.0"
+version = "0.49.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "101a2e213acebe624cfb9bfc944de5e33c849e0df0f09c3d3aa3b54368dbe7af"
+checksum = "deecb478dc3cc40203e0e97ac0fb92947e0719754bbafd0026bdc49318e2fd03"
 dependencies = [
  "aws-smithy-http",
  "bytes",
  "http",
  "http-body",
- "pin-project",
+ "pin-project-lite",
  "tower",
  "tracing",
 ]
 
 [[package]]
 name = "aws-smithy-json"
-version = "0.39.0"
+version = "0.49.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bd21f28535a2538b77274aa590abfb6d37aece3281dfc4c9411c1625d3b9239e"
+checksum = "6593456af93c4a39724f7dc9d239833102ab96c1d1e94c35ea79f0e55f9fd54c"
 dependencies = [
  "aws-smithy-types",
 ]
 
 [[package]]
 name = "aws-smithy-query"
-version = "0.39.0"
+version = "0.49.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eb5a2c90311b0d20cf23212a15961cad2b76480863b1f7ce0608d9ece8dacdfb"
+checksum = "b803460b71645dfa9f6be47c4f00f91632f01e5bb01f9dc43890cd6cba983f08"
 dependencies = [
  "aws-smithy-types",
- "urlencoding 1.3.3",
+ "urlencoding",
 ]
 
 [[package]]
 name = "aws-smithy-types"
-version = "0.39.0"
+version = "0.49.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "962f2da621cd29f272636eebce39ca321c91e02bbb7eb848c4587ac14933d339"
+checksum = "e93b0c93a3b963da946a0b8ef3853a7252298eb75cdbfb21dad60f5ed0ded861"
 dependencies = [
  "itoa",
  "num-integer",
@@ -642,23 +669,24 @@ dependencies = [
 
 [[package]]
 name = "aws-smithy-xml"
-version = "0.39.0"
+version = "0.49.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "829c7efd92b7a6d0536ceb48fd93a289ddf8763c67bffe875d82eae3f9886546"
+checksum = "36b9efb4855b4acb29961a776d45680f3cbdd7c4783cbbae078da54c342575dd"
 dependencies = [
- "thiserror",
  "xmlparser",
 ]
 
 [[package]]
 name = "aws-types"
-version = "0.9.0"
+version = "0.49.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68159725aa77553dbc6028f36d8378563cd45b18ef9cf03d1515ac469efacf13"
+checksum = "93f3f349b39781849261db1c727369923bb97007cf7bd0deb3a6e9e461c8d38f"
 dependencies = [
  "aws-smithy-async",
  "aws-smithy-client",
+ "aws-smithy-http",
  "aws-smithy-types",
+ "http",
  "rustc_version",
  "tracing",
  "zeroize",
@@ -938,10 +966,10 @@ dependencies = [
  "hex",
  "http",
  "iri-string 0.6.0",
- "libipld 0.14.0",
+ "libipld",
  "serde",
  "serde_with 2.0.1",
- "siwe 0.5.0",
+ "siwe",
  "thiserror",
  "time 0.3.15",
  "url",
@@ -1001,15 +1029,15 @@ checksum = "3a4f925191b4367301851c6d99b09890311d74b0d43f274c0b34c86d308a3663"
 
 [[package]]
 name = "capgrok"
-version = "0.3.1"
+version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c5fec5445658b25821dee5ab2788b0788620194c49cddf17c7aca0431b5ffae8"
+checksum = "17b1f49d14a68f6b30e935586d15bc1fae6444c8e42cd222d4b68cf16406eecb"
 dependencies = [
  "base64 0.12.3",
- "iri-string 0.4.1",
+ "iri-string 0.6.0",
  "serde",
  "serde_json",
- "siwe 0.4.2",
+ "siwe",
  "thiserror",
 ]
 
@@ -1244,6 +1272,15 @@ version = "0.1.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fd121741cf3eb82c08dd3023eb55bf2665e5f60ec20f89760cf836ae4562e6a0"
 
+[[package]]
+name = "crc32c"
+version = "0.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3dfea2db42e9927a3845fb268a10a72faed6d416065f77873f05e411457c363e"
+dependencies = [
+ "rustc_version",
+]
+
 [[package]]
 name = "crc32fast"
 version = "1.3.2"
@@ -1601,12 +1638,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a2658621297f2cf68762a6f7dc0bb7e1ff2cfd6583daef8ee0fed6f7ec468ec0"
 dependencies = [
  "darling 0.10.2",
- "derive_builder_core",
+ "derive_builder_core 0.9.0",
  "proc-macro2",
  "quote",
  "syn",
 ]
 
+[[package]]
+name = "derive_builder"
+version = "0.11.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d07adf7be193b71cc36b193d0f5fe60b918a3a9db4dad0449f57bcfd519704a3"
+dependencies = [
+ "derive_builder_macro",
+]
+
 [[package]]
 name = "derive_builder_core"
 version = "0.9.0"
@@ -1619,6 +1665,28 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "derive_builder_core"
+version = "0.11.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1f91d4cfa921f1c05904dc3c57b4a32c38aed3340cce209f3a6fd1478babafc4"
+dependencies = [
+ "darling 0.14.1",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "derive_builder_macro"
+version = "0.11.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f0314b72bed045f3a68671b3c86328386762c93f82d98c65c3cb5e5f573dd68"
+dependencies = [
+ "derive_builder_core 0.11.2",
+ "syn",
+]
+
 [[package]]
 name = "des"
 version = "0.7.0"
@@ -1666,7 +1734,7 @@ dependencies = [
 [[package]]
 name = "did-ethr"
 version = "0.1.1"
-source = "git+https://github.com/spruceid/ssi?branch=main#c4a2051c9ed8fdaa503bec39175b13ca24523812"
+source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#fce1a8b031d21ba8abbb559e791adf4c5f1b3a68"
 dependencies = [
  "async-trait",
  "chrono",
@@ -1682,7 +1750,7 @@ dependencies = [
 [[package]]
 name = "did-ion"
 version = "0.1.0"
-source = "git+https://github.com/spruceid/ssi?branch=main#c4a2051c9ed8fdaa503bec39175b13ca24523812"
+source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#fce1a8b031d21ba8abbb559e791adf4c5f1b3a68"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -1704,7 +1772,7 @@ dependencies = [
 [[package]]
 name = "did-method-key"
 version = "0.1.3"
-source = "git+https://github.com/spruceid/ssi?branch=main#c4a2051c9ed8fdaa503bec39175b13ca24523812"
+source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#fce1a8b031d21ba8abbb559e791adf4c5f1b3a68"
 dependencies = [
  "async-trait",
  "multibase 0.8.0",
@@ -1719,7 +1787,7 @@ dependencies = [
 [[package]]
 name = "did-onion"
 version = "0.1.2"
-source = "git+https://github.com/spruceid/ssi?branch=main#c4a2051c9ed8fdaa503bec39175b13ca24523812"
+source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#fce1a8b031d21ba8abbb559e791adf4c5f1b3a68"
 dependencies = [
  "async-trait",
  "http",
@@ -1732,7 +1800,7 @@ dependencies = [
 [[package]]
 name = "did-pkh"
 version = "0.1.1"
-source = "git+https://github.com/spruceid/ssi?branch=main#c4a2051c9ed8fdaa503bec39175b13ca24523812"
+source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#fce1a8b031d21ba8abbb559e791adf4c5f1b3a68"
 dependencies = [
  "async-trait",
  "bech32",
@@ -1749,7 +1817,7 @@ dependencies = [
 [[package]]
 name = "did-tz"
 version = "0.1.2"
-source = "git+https://github.com/spruceid/ssi?branch=main#c4a2051c9ed8fdaa503bec39175b13ca24523812"
+source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#fce1a8b031d21ba8abbb559e791adf4c5f1b3a68"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -1769,7 +1837,7 @@ dependencies = [
 [[package]]
 name = "did-web"
 version = "0.1.2"
-source = "git+https://github.com/spruceid/ssi?branch=main#c4a2051c9ed8fdaa503bec39175b13ca24523812"
+source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#fce1a8b031d21ba8abbb559e791adf4c5f1b3a68"
 dependencies = [
  "async-trait",
  "http",
@@ -1781,7 +1849,7 @@ dependencies = [
 [[package]]
 name = "did-webkey"
 version = "0.1.1"
-source = "git+https://github.com/spruceid/ssi?branch=main#c4a2051c9ed8fdaa503bec39175b13ca24523812"
+source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#fce1a8b031d21ba8abbb559e791adf4c5f1b3a68"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -1908,7 +1976,7 @@ version = "0.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c9720bba047d567ffc8a3cba48bf19126600e249ab7f128e9233e6376976a116"
 dependencies = [
- "heck 0.4.0",
+ "heck",
  "proc-macro2",
  "quote",
  "syn",
@@ -2297,7 +2365,7 @@ dependencies = [
  "indexmap",
  "slab",
  "tokio",
- "tokio-util 0.7.4",
+ "tokio-util",
  "tracing",
 ]
 
@@ -2341,15 +2409,6 @@ dependencies = [
  "http",
 ]
 
-[[package]]
-name = "heck"
-version = "0.3.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c"
-dependencies = [
- "unicode-segmentation",
-]
-
 [[package]]
 name = "heck"
 version = "0.4.0"
@@ -2642,15 +2701,6 @@ dependencies = [
  "smallvec",
 ]
 
-[[package]]
-name = "iri-string"
-version = "0.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f0f7638c1e223529f1bfdc48c8b133b9e0b434094d1d28473161ee48b235f78"
-dependencies = [
- "nom 7.1.1",
-]
-
 [[package]]
 name = "iri-string"
 version = "0.5.6"
@@ -2767,13 +2817,14 @@ dependencies = [
  "bincode",
  "bs58",
  "cached 0.34.1",
+ "derive_builder 0.11.2",
  "futures",
  "hex",
  "hyper",
  "iri-string 0.5.6",
  "kepler-lib",
  "lazy_static",
- "libipld 0.14.0",
+ "libipld",
  "libp2p",
  "opentelemetry",
  "opentelemetry-jaeger",
@@ -2791,12 +2842,12 @@ dependencies = [
  "time 0.3.15",
  "tokio",
  "tokio-stream",
- "tokio-util 0.6.10",
+ "tokio-util",
  "tracing",
  "tracing-log",
  "tracing-opentelemetry",
  "tracing-subscriber",
- "urlencoding 2.1.2",
+ "urlencoding",
  "uuid",
  "void",
 ]
@@ -2819,7 +2870,7 @@ dependencies = [
  "did-webkey",
  "iri-string 0.6.0",
  "lazy_static",
- "libipld 0.14.0",
+ "libipld",
  "serde",
  "serde_json",
  "serde_with 1.14.0",
@@ -2834,15 +2885,15 @@ name = "kepler-sdk"
 version = "0.1.0"
 dependencies = [
  "base64 0.13.1",
- "chrono",
  "hex",
  "http",
- "iri-string 0.4.1",
+ "iri-string 0.6.0",
  "kepler-lib",
  "serde",
  "serde_json",
  "serde_with 1.14.0",
  "thiserror",
+ "time 0.3.15",
  "tokio",
  "tracing",
 ]
@@ -2890,27 +2941,6 @@ version = "0.2.135"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "68783febc7782c6c5cb401fbda4de5a9898be1762314da0bb2c10ced61f18b0c"
 
-[[package]]
-name = "libipld"
-version = "0.13.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dfb2f6a99d838dcb1cc18361c79e827b76e91987227406e705d07cbef51ab7be"
-dependencies = [
- "async-trait",
- "cached 0.30.0",
- "fnv",
- "libipld-cbor 0.13.1",
- "libipld-cbor-derive 0.13.1",
- "libipld-core 0.13.1",
- "libipld-json 0.13.1",
- "libipld-macro 0.13.1",
- "libipld-pb 0.13.1",
- "log",
- "multihash",
- "parking_lot 0.12.1",
- "thiserror",
-]
-
 [[package]]
 name = "libipld"
 version = "0.14.0"
@@ -2920,29 +2950,18 @@ dependencies = [
  "async-trait",
  "cached 0.30.0",
  "fnv",
- "libipld-cbor 0.14.0",
- "libipld-cbor-derive 0.14.0",
- "libipld-core 0.14.0",
- "libipld-json 0.14.0",
- "libipld-macro 0.14.0",
- "libipld-pb 0.14.0",
+ "libipld-cbor",
+ "libipld-cbor-derive",
+ "libipld-core",
+ "libipld-json",
+ "libipld-macro",
+ "libipld-pb",
  "log",
  "multihash",
  "parking_lot 0.12.1",
  "thiserror",
 ]
 
-[[package]]
-name = "libipld-cbor"
-version = "0.13.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "46b3bbc4f35b8f25eb140d183f33d4610a1eb6531b312f01161fa06693f8c28a"
-dependencies = [
- "byteorder",
- "libipld-core 0.13.1",
- "thiserror",
-]
-
 [[package]]
 name = "libipld-cbor"
 version = "0.14.0"
@@ -2950,23 +2969,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8dd1ab68c9d26f20c7d0dfea6eecbae8c00359875210001b33ca27d4a02f3d09"
 dependencies = [
  "byteorder",
- "libipld-core 0.14.0",
+ "libipld-core",
  "thiserror",
 ]
 
-[[package]]
-name = "libipld-cbor-derive"
-version = "0.13.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b6ec17d094f4f62e33e201ded63d911fd0bff3b12121ddc50d611e967f7e6824"
-dependencies = [
- "proc-macro-crate",
- "proc-macro2",
- "quote",
- "syn",
- "synstructure",
-]
-
 [[package]]
 name = "libipld-cbor-derive"
 version = "0.14.0"
@@ -2980,21 +2986,6 @@ dependencies = [
  "synstructure",
 ]
 
-[[package]]
-name = "libipld-core"
-version = "0.13.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fbdd758764f9680a818af33c31db733eb7c45224715d8816b9dcf0548c75f7c5"
-dependencies = [
- "anyhow",
- "cid",
- "core2",
- "multibase 0.9.1",
- "multihash",
- "serde",
- "thiserror",
-]
-
 [[package]]
 name = "libipld-core"
 version = "0.14.0"
@@ -3006,19 +2997,8 @@ dependencies = [
  "core2",
  "multibase 0.9.1",
  "multihash",
- "thiserror",
-]
-
-[[package]]
-name = "libipld-json"
-version = "0.13.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "415427568962961e21009eb92fd6052de95174423cff9d3c583491858e060f4e"
-dependencies = [
- "libipld-core 0.13.1",
- "multihash",
  "serde",
- "serde_json",
+ "thiserror",
 ]
 
 [[package]]
@@ -3027,40 +3007,19 @@ version = "0.14.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "18aa481a87f084d98473dd9ece253a9569c762b75f6bbba8217d54e48c9d63b3"
 dependencies = [
- "libipld-core 0.14.0",
+ "libipld-core",
  "multihash",
  "serde",
  "serde_json",
 ]
 
-[[package]]
-name = "libipld-macro"
-version = "0.13.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c6c4cb1056262ef4056ad9e5fb41f252c45f55009888e21b7837ac051f38814a"
-dependencies = [
- "libipld-core 0.13.1",
-]
-
 [[package]]
 name = "libipld-macro"
 version = "0.14.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "852c011562ae5059b67c3a917f9f5945af5a68df8e39ede4444fff33274d25e2"
 dependencies = [
- "libipld-core 0.14.0",
-]
-
-[[package]]
-name = "libipld-pb"
-version = "0.13.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6b851a08aeff23e1e8df42081ef22052a838e8e39b769fcb7867d2fadd834d2d"
-dependencies = [
- "libipld-core 0.13.1",
- "prost 0.9.0",
- "prost-build 0.9.0",
- "thiserror",
+ "libipld-core",
 ]
 
 [[package]]
@@ -3069,7 +3028,7 @@ version = "0.14.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c003be513496578115256a1b4ac7b80d4ece2462c9869dfb736fd30d8bb1d1c0"
 dependencies = [
- "libipld-core 0.14.0",
+ "libipld-core",
  "prost 0.10.4",
  "prost-build 0.10.4",
  "thiserror",
@@ -3350,7 +3309,7 @@ version = "0.30.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a0eddc4497a8b5a506013c40e8189864f9c3a00db2b25671f428ae9007f3ba32"
 dependencies = [
- "heck 0.4.0",
+ "heck",
  "quote",
  "syn",
 ]
@@ -3492,10 +3451,13 @@ dependencies = [
 ]
 
 [[package]]
-name = "md5"
-version = "0.7.0"
+name = "md-5"
+version = "0.10.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771"
+checksum = "6365506850d44bff6e2fbcb5176cf63650e48bd45ef2fe2665ae1570e0f4b9ca"
+dependencies = [
+ "digest 0.10.5",
+]
 
 [[package]]
 name = "memchr"
@@ -3518,12 +3480,6 @@ version = "0.3.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d"
 
-[[package]]
-name = "minimal-lexical"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
-
 [[package]]
 name = "miniz_oxide"
 version = "0.5.4"
@@ -3567,7 +3523,7 @@ dependencies = [
  "mime",
  "spin 0.9.4",
  "tokio",
- "tokio-util 0.7.4",
+ "tokio-util",
  "version_check 0.9.4",
 ]
 
@@ -3775,16 +3731,6 @@ dependencies = [
  "version_check 0.1.5",
 ]
 
-[[package]]
-name = "nom"
-version = "7.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a8903e5a29a317527874d0402f867152a3d21c908bb0b933e416c65e301d4c36"
-dependencies = [
- "memchr",
- "minimal-lexical",
-]
-
 [[package]]
 name = "nu-ansi-term"
 version = "0.46.0"
@@ -4181,7 +4127,7 @@ dependencies = [
  "circular",
  "clear_on_drop",
  "crc24",
- "derive_builder",
+ "derive_builder 0.9.0",
  "des",
  "digest 0.9.0",
  "ed25519-dalek",
@@ -4190,8 +4136,8 @@ dependencies = [
  "hex",
  "lazy_static",
  "log",
- "md-5",
- "nom 4.2.3",
+ "md-5 0.9.1",
+ "nom",
  "num-bigint-dig",
  "num-derive",
  "num-traits",
@@ -4454,16 +4400,6 @@ dependencies = [
  "syn",
 ]
 
-[[package]]
-name = "prost"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "444879275cb4fd84958b1a1d5420d15e6fcf7c235fe47f053c9c2a80aceb6001"
-dependencies = [
- "bytes",
- "prost-derive 0.9.0",
-]
-
 [[package]]
 name = "prost"
 version = "0.10.4"
@@ -4484,26 +4420,6 @@ dependencies = [
  "prost-derive 0.11.0",
 ]
 
-[[package]]
-name = "prost-build"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62941722fb675d463659e49c4f3fe1fe792ff24fe5bbaa9c08cd3b98a1c354f5"
-dependencies = [
- "bytes",
- "heck 0.3.3",
- "itertools",
- "lazy_static",
- "log",
- "multimap",
- "petgraph",
- "prost 0.9.0",
- "prost-types 0.9.0",
- "regex",
- "tempfile",
- "which",
-]
-
 [[package]]
 name = "prost-build"
 version = "0.10.4"
@@ -4513,7 +4429,7 @@ dependencies = [
  "bytes",
  "cfg-if",
  "cmake",
- "heck 0.4.0",
+ "heck",
  "itertools",
  "lazy_static",
  "log",
@@ -4533,7 +4449,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7f835c582e6bd972ba8347313300219fed5bfa52caf175298d860b61ff6069bb"
 dependencies = [
  "bytes",
- "heck 0.4.0",
+ "heck",
  "itertools",
  "lazy_static",
  "log",
@@ -4559,19 +4475,6 @@ dependencies = [
  "unsigned-varint",
 ]
 
-[[package]]
-name = "prost-derive"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f9cc1a3263e07e0bf68e96268f37665207b49560d98739662cdfaae215c720fe"
-dependencies = [
- "anyhow",
- "itertools",
- "proc-macro2",
- "quote",
- "syn",
-]
-
 [[package]]
 name = "prost-derive"
 version = "0.10.1"
@@ -4598,16 +4501,6 @@ dependencies = [
  "syn",
 ]
 
-[[package]]
-name = "prost-types"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "534b7a0e836e3c482d2693070f982e39e7611da9695d4d1f5a4b186b51faef0a"
-dependencies = [
- "bytes",
- "prost 0.9.0",
-]
-
 [[package]]
 name = "prost-types"
 version = "0.10.1"
@@ -4945,7 +4838,7 @@ dependencies = [
  "time 0.3.15",
  "tokio",
  "tokio-stream",
- "tokio-util 0.7.4",
+ "tokio-util",
  "ubyte",
  "version_check 0.9.4",
  "yansi",
@@ -5439,22 +5332,6 @@ dependencies = [
  "thiserror",
 ]
 
-[[package]]
-name = "siwe"
-version = "0.4.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f24fe2b646c33a670e7d79a232bffb41821fed28b1870a8bd1a47e6ae686ace6"
-dependencies = [
- "hex",
- "http",
- "iri-string 0.4.1",
- "k256",
- "rand 0.8.5",
- "sha3 0.9.1",
- "thiserror",
- "time 0.3.15",
-]
-
 [[package]]
 name = "siwe"
 version = "0.5.0"
@@ -5481,7 +5358,7 @@ dependencies = [
  "iri-string 0.6.0",
  "serde",
  "serde_json",
- "siwe 0.5.0",
+ "siwe",
  "thiserror",
 ]
 
@@ -5589,7 +5466,7 @@ dependencies = [
 [[package]]
 name = "ssi"
 version = "0.4.0"
-source = "git+https://github.com/spruceid/ssi?branch=main#c4a2051c9ed8fdaa503bec39175b13ca24523812"
+source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#fce1a8b031d21ba8abbb559e791adf4c5f1b3a68"
 dependencies = [
  "ssi-caips",
  "ssi-core",
@@ -5610,7 +5487,7 @@ dependencies = [
 [[package]]
 name = "ssi-caips"
 version = "0.1.0"
-source = "git+https://github.com/spruceid/ssi?branch=main#c4a2051c9ed8fdaa503bec39175b13ca24523812"
+source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#fce1a8b031d21ba8abbb559e791adf4c5f1b3a68"
 dependencies = [
  "bs58",
  "ssi-jwk",
@@ -5620,12 +5497,12 @@ dependencies = [
 [[package]]
 name = "ssi-contexts"
 version = "0.1.3"
-source = "git+https://github.com/spruceid/ssi?branch=main#c4a2051c9ed8fdaa503bec39175b13ca24523812"
+source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#fce1a8b031d21ba8abbb559e791adf4c5f1b3a68"
 
 [[package]]
 name = "ssi-core"
 version = "0.1.0"
-source = "git+https://github.com/spruceid/ssi?branch=main#c4a2051c9ed8fdaa503bec39175b13ca24523812"
+source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#fce1a8b031d21ba8abbb559e791adf4c5f1b3a68"
 dependencies = [
  "async-trait",
  "serde",
@@ -5635,7 +5512,7 @@ dependencies = [
 [[package]]
 name = "ssi-crypto"
 version = "0.1.0"
-source = "git+https://github.com/spruceid/ssi?branch=main#c4a2051c9ed8fdaa503bec39175b13ca24523812"
+source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#fce1a8b031d21ba8abbb559e791adf4c5f1b3a68"
 dependencies = [
  "bs58",
  "digest 0.9.0",
@@ -5650,14 +5527,14 @@ dependencies = [
 [[package]]
 name = "ssi-dids"
 version = "0.1.0"
-source = "git+https://github.com/spruceid/ssi?branch=main#c4a2051c9ed8fdaa503bec39175b13ca24523812"
+source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#fce1a8b031d21ba8abbb559e791adf4c5f1b3a68"
 dependencies = [
  "anyhow",
  "async-std",
  "async-trait",
  "bs58",
  "chrono",
- "derive_builder",
+ "derive_builder 0.9.0",
  "hex",
  "multibase 0.8.0",
  "percent-encoding",
@@ -5675,7 +5552,7 @@ dependencies = [
 [[package]]
 name = "ssi-json-ld"
 version = "0.1.0"
-source = "git+https://github.com/spruceid/ssi?branch=main#c4a2051c9ed8fdaa503bec39175b13ca24523812"
+source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#fce1a8b031d21ba8abbb559e791adf4c5f1b3a68"
 dependencies = [
  "async-std",
  "chrono",
@@ -5695,7 +5572,7 @@ dependencies = [
 [[package]]
 name = "ssi-jwk"
 version = "0.1.0"
-source = "git+https://github.com/spruceid/ssi?branch=main#c4a2051c9ed8fdaa503bec39175b13ca24523812"
+source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#fce1a8b031d21ba8abbb559e791adf4c5f1b3a68"
 dependencies = [
  "base64 0.12.3",
  "blake2b_simd 0.5.11",
@@ -5719,7 +5596,7 @@ dependencies = [
 [[package]]
 name = "ssi-jws"
 version = "0.1.0"
-source = "git+https://github.com/spruceid/ssi?branch=main#c4a2051c9ed8fdaa503bec39175b13ca24523812"
+source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#fce1a8b031d21ba8abbb559e791adf4c5f1b3a68"
 dependencies = [
  "base64 0.12.3",
  "blake2",
@@ -5740,7 +5617,7 @@ dependencies = [
 [[package]]
 name = "ssi-jwt"
 version = "0.1.0"
-source = "git+https://github.com/spruceid/ssi?branch=main#c4a2051c9ed8fdaa503bec39175b13ca24523812"
+source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#fce1a8b031d21ba8abbb559e791adf4c5f1b3a68"
 dependencies = [
  "chrono",
  "serde",
@@ -5753,7 +5630,7 @@ dependencies = [
 [[package]]
 name = "ssi-ldp"
 version = "0.1.0"
-source = "git+https://github.com/spruceid/ssi?branch=main#c4a2051c9ed8fdaa503bec39175b13ca24523812"
+source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#fce1a8b031d21ba8abbb559e791adf4c5f1b3a68"
 dependencies = [
  "async-trait",
  "bs58",
@@ -5778,7 +5655,7 @@ dependencies = [
 [[package]]
 name = "ssi-ssh"
 version = "0.1.0"
-source = "git+https://github.com/spruceid/ssi?branch=main#c4a2051c9ed8fdaa503bec39175b13ca24523812"
+source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#fce1a8b031d21ba8abbb559e791adf4c5f1b3a68"
 dependencies = [
  "sshkeys",
  "ssi-jwk",
@@ -5788,7 +5665,7 @@ dependencies = [
 [[package]]
 name = "ssi-tzkey"
 version = "0.1.0"
-source = "git+https://github.com/spruceid/ssi?branch=main#c4a2051c9ed8fdaa503bec39175b13ca24523812"
+source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#fce1a8b031d21ba8abbb559e791adf4c5f1b3a68"
 dependencies = [
  "bs58",
  "ed25519-dalek",
@@ -5800,11 +5677,11 @@ dependencies = [
 [[package]]
 name = "ssi-ucan"
 version = "0.1.0"
-source = "git+https://github.com/spruceid/ssi?branch=main#c4a2051c9ed8fdaa503bec39175b13ca24523812"
+source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#fce1a8b031d21ba8abbb559e791adf4c5f1b3a68"
 dependencies = [
  "base64 0.12.3",
  "chrono",
- "libipld 0.13.1",
+ "libipld",
  "serde",
  "serde_json",
  "serde_with 1.14.0",
@@ -5820,7 +5697,7 @@ dependencies = [
 [[package]]
 name = "ssi-vc"
 version = "0.1.0"
-source = "git+https://github.com/spruceid/ssi?branch=main#c4a2051c9ed8fdaa503bec39175b13ca24523812"
+source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#fce1a8b031d21ba8abbb559e791adf4c5f1b3a68"
 dependencies = [
  "async-trait",
  "base64 0.12.3",
@@ -5828,7 +5705,7 @@ dependencies = [
  "cacaos",
  "chrono",
  "flate2",
- "libipld 0.14.0",
+ "libipld",
  "multihash",
  "reqwest",
  "serde",
@@ -5847,7 +5724,7 @@ dependencies = [
 [[package]]
 name = "ssi-zcap-ld"
 version = "0.1.0"
-source = "git+https://github.com/spruceid/ssi?branch=main#c4a2051c9ed8fdaa503bec39175b13ca24523812"
+source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#fce1a8b031d21ba8abbb559e791adf4c5f1b3a68"
 dependencies = [
  "async-trait",
  "serde",
@@ -6164,20 +6041,6 @@ dependencies = [
  "tokio",
 ]
 
-[[package]]
-name = "tokio-util"
-version = "0.6.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "36943ee01a6d67977dd3f84a5a1d2efeb4ada3a1ae771cadfaa535d9d9fc6507"
-dependencies = [
- "bytes",
- "futures-core",
- "futures-sink",
- "log",
- "pin-project-lite",
- "tokio",
-]
-
 [[package]]
 name = "tokio-util"
 version = "0.7.4"
@@ -6186,6 +6049,7 @@ checksum = "0bb2e075f03b3d66d8d8785356224ba688d2906a371015e225beeb65ca92c740"
 dependencies = [
  "bytes",
  "futures-core",
+ "futures-io",
  "futures-sink",
  "pin-project-lite",
  "tokio",
@@ -6448,12 +6312,6 @@ dependencies = [
  "tinyvec",
 ]
 
-[[package]]
-name = "unicode-segmentation"
-version = "1.10.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0fdbf052a0783de01e944a6ce7a8cb939e295b1e7be835a1112c3b9a7f047a5a"
-
 [[package]]
 name = "unicode-width"
 version = "0.1.10"
@@ -6513,12 +6371,6 @@ dependencies = [
  "percent-encoding",
 ]
 
-[[package]]
-name = "urlencoding"
-version = "1.3.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5a1f0175e03a0973cf4afd476bef05c26e228520400eb1fd473ad417b1c00ffb"
-
 [[package]]
 name = "urlencoding"
 version = "2.1.2"
diff --git a/src/storage/mod.rs b/src/storage/mod.rs
index b7b69239..4aa316e5 100644
--- a/src/storage/mod.rs
+++ b/src/storage/mod.rs
@@ -43,204 +43,6 @@ pub trait StorageConfig<S> {
     async fn create(&self, orbit: &OrbitId) -> Result<S, Self::Error>;
 }
 
-impl StorageUtils {
-    pub fn new(config: config::BlockStorage) -> Self {
-        Self { config }
-    }
-
-    pub async fn healthcheck(&self) -> Result<()> {
-        match &self.config {
-            config::BlockStorage::S3(r) => {
-                let client = s3::S3BlockStore::new_(r, Cid::default());
-                // client.init().await
-                Ok(())
-            }
-            config::BlockStorage::Local(r) => {
-                if !r.path.is_dir() {
-                    Err(anyhow!(
-                        "{} does not exist or is not a directory",
-                        r.path.to_str().unwrap()
-                    ))
-                } else {
-                    Ok(())
-                }
-            }
-        }
-    }
-
-    // TODO either remove this method and only rely on fetching the kp, or find a better way for S3
-    pub async fn exists(&self, orbit: Cid) -> Result<bool> {
-        match &self.config {
-            config::BlockStorage::S3(_) => Ok(self.key_pair(orbit).await?.is_some()),
-            config::BlockStorage::Local(r) => {
-                let dir = r.path.join(&orbit.to_string_of_base(Base::Base58Btc)?);
-                Ok(dir.exists())
-            }
-        }
-    }
-
-    pub async fn relay_key_pair(config: config::BlockStorage) -> Result<Ed25519Keypair> {
-        let kp = match config.clone() {
-            config::BlockStorage::S3(r) => {
-                let client = s3::new_client(r.clone());
-                let res = client
-                    .get_object()
-                    .bucket(r.bucket.clone())
-                    .key("kp")
-                    .send()
-                    .await;
-                match res {
-                    Ok(o) => Some(Ed25519Keypair::decode(
-                        &mut o.body.collect().await?.into_bytes().to_vec(),
-                    )?),
-                    Err(SdkError::ServiceError {
-                        err:
-                            GetObjectError {
-                                kind: GetObjectErrorKind::NoSuchKey(_),
-                                ..
-                            },
-                        ..
-                    }) => None,
-                    Err(e) => return Err(e.into()),
-                }
-            }
-            config::BlockStorage::Local(r) => {
-                if let Ok(mut bytes) = fs::read(r.path.join("kp")).await {
-                    Some(Ed25519Keypair::decode(&mut bytes)?)
-                } else {
-                    None
-                }
-            }
-        };
-        if let Some(k) = kp {
-            Ok(k)
-        } else {
-            let kp = Ed25519Keypair::generate();
-            match config {
-                config::BlockStorage::S3(r) => {
-                    let client = s3::new_client(r.clone());
-                    client
-                        .put_object()
-                        .bucket(r.bucket.clone())
-                        .key("kp")
-                        .body(ByteStream::new(SdkBody::from(kp.encode().to_vec())))
-                        .send()
-                        .await?;
-                }
-                config::BlockStorage::Local(r) => {
-                    fs::write(r.path.join("kp"), kp.encode()).await?;
-                }
-            };
-            Ok(kp)
-        }
-    }
-
-    pub async fn key_pair(&self, orbit: Cid) -> Result<Option<Ed25519Keypair>> {
-        match &self.config {
-            config::BlockStorage::S3(r) => {
-                let client = s3::S3DataStore::new_(r.clone(), orbit);
-                Ok(client
-                    .get_("keypair".to_string())
-                    .await?
-                    .map(|mut b| Ed25519Keypair::decode(&mut b))
-                    .transpose()?)
-            }
-            config::BlockStorage::Local(r) => {
-                let dir = r.path.join(&orbit.to_string_of_base(Base::Base58Btc)?);
-                match fs::read(dir.join("kp")).await {
-                    Ok(mut k) => Ok(Some(Ed25519Keypair::decode(&mut k)?)),
-                    Err(e) => match e.kind() {
-                        std::io::ErrorKind::NotFound => Ok(None),
-                        _ => Err(e.into()),
-                    },
-                }
-            }
-        }
-    }
-
-    pub async fn orbit_id(&self, orbit: Cid) -> Result<Option<OrbitId>> {
-        match &self.config {
-            config::BlockStorage::S3(r) => {
-                let client = s3::S3DataStore::new_(r.clone(), orbit);
-                Ok(client
-                    .get_("orbit_url".to_string())
-                    .await?
-                    .map(String::from_utf8)
-                    .transpose()?
-                    .map(|b| OrbitId::from_str(&b))
-                    .transpose()?)
-            }
-            config::BlockStorage::Local(r) => {
-                let dir = r.path.join(&orbit.to_string_of_base(Base::Base58Btc)?);
-                match fs::read(dir.join("id")).await {
-                    Ok(i) => Ok(Some(String::from_utf8(i)?.parse()?)),
-                    Err(e) => match e.kind() {
-                        std::io::ErrorKind::NotFound => Ok(None),
-                        _ => Err(e.into()),
-                    },
-                }
-            }
-        }
-    }
-
-    pub async fn setup_orbit(&self, orbit: OrbitId, kp: Ed25519Keypair, auth: &[u8]) -> Result<()> {
-        match &self.config {
-            config::BlockStorage::S3(r) => {
-                let client = s3::S3DataStore::new_(r.clone(), orbit.get_cid());
-                client
-                    .put_("keypair".to_string(), kp.encode().to_vec())
-                    .await?;
-                client
-                    .put_(
-                        "orbit_url".to_string(),
-                        orbit.to_string().as_bytes().to_vec(),
-                    )
-                    .await?;
-            }
-            config::BlockStorage::Local(r) => {
-                let dir = r
-                    .path
-                    .join(orbit.get_cid().to_string_of_base(Base::Base58Btc)?);
-                fs::create_dir_all(&dir)
-                    .await
-                    .map_err(|e| anyhow!("Couldn't create dir: {}", e))?;
-
-                fs::write(dir.join("access_log"), auth).await?;
-                fs::write(dir.join("kp"), kp.encode()).await?;
-                fs::write(dir.join("id"), orbit.to_string()).await?;
-            }
-        };
-        Ok(())
-    }
-
-    pub async fn ipfs_path(&self, orbit: Cid) -> Result<PathBuf> {
-        match &self.config {
-            config::BlockStorage::S3(r) => Ok(PathBuf::from(&format!(
-                "/s3bucket/{}/s3endpoint/{}/dynamotable/{}/dynamoendpoint/{}/orbitcid/{}",
-                r.bucket,
-                r.endpoint
-                    .as_ref()
-                    .map(|e| e.to_string())
-                    .unwrap_or_default(),
-                r.dynamodb.table,
-                r.dynamodb
-                    .endpoint
-                    .as_ref()
-                    .map(|e| e.to_string())
-                    .unwrap_or_default(),
-                orbit
-            ))),
-            config::BlockStorage::Local(r) => {
-                let path = r.path.join(orbit.to_string_of_base(Base::Base58Btc)?);
-                if !path.exists() {
-                    fs::create_dir_all(&path).await?;
-                }
-                Ok(path)
-            }
-        }
-    }
-}
-
 #[derive(thiserror::Error, Debug)]
 pub enum VecReadError<E> {
     #[error(transparent)]

From 7174431dd3a756201d828d0aa0164e3bc0599544 Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Wed, 2 Nov 2022 13:16:36 +0100
Subject: [PATCH 32/73] fix lifetime and usage errors

---
 src/capabilities/store.rs  |  1 +
 src/routes/mod.rs          |  4 ++--
 src/storage/file_system.rs |  4 ++--
 src/storage/s3.rs          | 14 +++++++-------
 4 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/src/capabilities/store.rs b/src/capabilities/store.rs
index 7e42e599..f0334c69 100644
--- a/src/capabilities/store.rs
+++ b/src/capabilities/store.rs
@@ -410,6 +410,7 @@ where
 impl<B> CapStore for Store<B>
 where
     B: ImmutableStore,
+    B::Error: 'static,
 {
     async fn get_cap(&self, c: &Cid) -> Result<Option<Delegation>> {
         // annoyingly ipfs will error if it cant find something, so we probably dont want to error here
diff --git a/src/routes/mod.rs b/src/routes/mod.rs
index baaa922d..72846a2d 100644
--- a/src/routes/mod.rs
+++ b/src/routes/mod.rs
@@ -66,7 +66,7 @@ impl<R> KVResponse<R> {
 
 impl<'r, R> Responder<'r, 'static> for KVResponse<R>
 where
-    R: AsyncRead + Send,
+    R: 'static + AsyncRead + Send,
 {
     fn respond_to(self, r: &'r Request<'_>) -> rocket::response::Result<'static> {
         Ok(Response::build_from(self.1.respond_to(r)?)
@@ -247,7 +247,7 @@ pub enum InvocationResponse<R> {
 
 impl<'r, R> Responder<'r, 'static> for InvocationResponse<R>
 where
-    R: AsyncRead + Send,
+    R: 'static + AsyncRead + Send,
 {
     fn respond_to(self, request: &'r Request<'_>) -> rocket::response::Result<'static> {
         match self {
diff --git a/src/storage/file_system.rs b/src/storage/file_system.rs
index 943c6eed..7dd5ed76 100644
--- a/src/storage/file_system.rs
+++ b/src/storage/file_system.rs
@@ -78,11 +78,11 @@ impl ProviderUtils for FileSystemConfig {
     }
     async fn relay_key_pair(&self) -> Result<Ed25519Keypair, Self::Error> {
         let path = self.path.join("kp");
-        match read(path).await {
+        match read(&path).await {
             Ok(mut k) => Ok(Ed25519Keypair::decode(&mut k)?),
             Err(e) if e.kind() == ErrorKind::NotFound => {
                 let k = Ed25519Keypair::generate();
-                write(path, k.encode()).await?;
+                write(&path, k.encode()).await?;
                 Ok(k)
             }
             Err(e) => Err(e.into()),
diff --git a/src/storage/s3.rs b/src/storage/s3.rs
index efdb89ad..be892851 100644
--- a/src/storage/s3.rs
+++ b/src/storage/s3.rs
@@ -103,7 +103,7 @@ impl ProviderUtils for S3BlockConfig {
         let client = new_client(&self);
         match client
             .get_object()
-            .bucket(self.bucket)
+            .bucket(&self.bucket)
             .key("kp")
             .send()
             .await
@@ -122,7 +122,7 @@ impl ProviderUtils for S3BlockConfig {
                 let kp = Ed25519Keypair::generate();
                 client
                     .put_object()
-                    .bucket(self.bucket)
+                    .bucket(&self.bucket)
                     .key("kp")
                     .body(ByteStream::new(SdkBody::from(kp.encode().to_vec())))
                     .send()
@@ -135,7 +135,7 @@ impl ProviderUtils for S3BlockConfig {
     async fn key_pair(&self, orbit: &OrbitId) -> Result<Option<Ed25519Keypair>, Self::Error> {
         match new_client(&self)
             .get_object()
-            .bucket(self.bucket)
+            .bucket(&self.bucket)
             .key(format!("{}/keypair", orbit.get_cid()))
             .send()
             .await
@@ -158,14 +158,14 @@ impl ProviderUtils for S3BlockConfig {
         let client = new_client(&self);
         client
             .put_object()
-            .bucket(self.bucket)
+            .bucket(&self.bucket)
             .key(format!("{}/keypair", orbit.get_cid()))
             .body(ByteStream::new(SdkBody::from(key.encode().to_vec())))
             .send()
             .await?;
         client
             .put_object()
-            .bucket(self.bucket)
+            .bucket(&self.bucket)
             .key(format!("{}/orbit_url", orbit.get_cid()))
             .body(ByteStream::new(SdkBody::from(orbit.to_string())))
             .send()
@@ -177,8 +177,8 @@ impl ProviderUtils for S3BlockConfig {
 pub fn new_client(config: &S3BlockConfig) -> Client {
     let general_config = super::utils::aws_config();
     let sdk_config = aws_sdk_s3::config::Builder::from(&general_config);
-    let sdk_config = match config.endpoint {
-        Some(e) => sdk_config.endpoint_resolver(Endpoint::immutable(e)),
+    let sdk_config = match &config.endpoint {
+        Some(e) => sdk_config.endpoint_resolver(Endpoint::immutable(e.clone())),
         None => sdk_config,
     };
     let sdk_config = sdk_config.build();

From fd4a32688942f1b18735706721b95ac04613ec58 Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Wed, 2 Nov 2022 13:18:16 +0100
Subject: [PATCH 33/73] fix create_orbit

---
 src/orbit.rs | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/src/orbit.rs b/src/orbit.rs
index a6978ad3..a4150d1f 100644
--- a/src/orbit.rs
+++ b/src/orbit.rs
@@ -3,7 +3,8 @@ use crate::{
     config,
     kv::{behaviour::BehaviourProcess, Service as KVService, Store},
     manifest::Manifest,
-    storage::{BlockConfig, BlockStores, ImmutableStore, StorageConfig},
+    storage::{ImmutableStore, StorageConfig},
+    BlockConfig, BlockStores,
 };
 use anyhow::{anyhow, Result};
 use derive_builder::Builder;
@@ -147,8 +148,8 @@ pub trait ProviderUtils {
 // Using Option to distinguish when the orbit already exists from a hard error
 pub async fn create_orbit(
     id: &OrbitId,
-    config: &config::Config,
-    auth: &[u8],
+    store_config: &BlockConfig,
+    index_config: &config::IndexStorage,
     relay: (PeerId, Multiaddr),
     kp: Ed25519Keypair,
 ) -> Result<Option<Orbit<BlockStores>>> {
@@ -158,30 +159,29 @@ pub async fn create_orbit(
     };
 
     // fails if DIR exists, this is Create, not Open
-    let storage_utils = StorageUtils::new(config.storage.blocks.clone());
-    if storage_utils.exists(id.get_cid()).await? {
+    if store_config.exists(&id).await? {
         return Ok(None);
     }
 
-    storage_utils.setup_orbit(id.clone(), kp, auth).await?;
+    store_config.setup_orbit(&id, &kp).await?;
 
     Ok(Some(
-        load_orbit(md.id().get_cid(), config, relay)
+        load_orbit(id.clone(), store_config, index_config, relay)
             .await
             .map(|o| o.ok_or_else(|| anyhow!("Couldn't find newly created orbit")))??,
     ))
 }
 
 pub async fn load_orbit(
-    id_cid: Cid,
-    config: &config::Config,
+    orbit: OrbitId,
+    store_config: &BlockConfig,
+    index_config: &config::IndexStorage,
     relay: (PeerId, Multiaddr),
 ) -> Result<Option<Orbit<BlockStores>>> {
-    let storage_utils = StorageUtils::new(config.storage.blocks.clone());
-    if !storage_utils.exists(id_cid).await? {
+    if !store_config.exists(&orbit).await? {
         return Ok(None);
     }
-    load_orbit_inner(id_cid, config.clone(), relay)
+    load_orbit_inner(orbit, store_config.clone(), index_config.clone(), relay)
         .await
         .map(Some)
 }

From 45382a91ed7394aca2bea43df7a3710fc7cbd242 Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Wed, 2 Nov 2022 13:19:49 +0100
Subject: [PATCH 34/73] integrate new StorageConfig structs with existing
 config stuff

---
 src/config.rs              | 41 +++++++++++---------------------------
 src/lib.rs                 | 40 ++++++++++++++++++++++++++++++++-----
 src/routes/mod.rs          |  3 ++-
 src/storage/file_system.rs |  8 ++++++++
 src/storage/mod.rs         | 27 +++++--------------------
 src/storage/s3.rs          | 13 ++++++------
 6 files changed, 68 insertions(+), 64 deletions(-)

diff --git a/src/config.rs b/src/config.rs
index 2e8d6863..bc88bb08 100644
--- a/src/config.rs
+++ b/src/config.rs
@@ -1,7 +1,11 @@
-use crate::allow_list::OrbitAllowListService;
+use crate::{
+    allow_list::OrbitAllowListService,
+    storage::{file_system::FileSystemConfig, s3::S3BlockConfig},
+    BlockConfig,
+};
 use rocket::http::hyper::Uri;
 use serde::{Deserialize, Serialize};
-use serde_with::{serde_as, DisplayFromStr};
+use serde_with::{serde_as, DisplayFromStr, FromInto};
 use std::path::PathBuf;
 
 #[derive(Serialize, Deserialize, Debug, Default, Clone, Hash, PartialEq, Eq)]
@@ -37,32 +41,19 @@ pub struct OrbitsConfig {
     pub allowlist: Option<OrbitAllowListService>,
 }
 
+#[serde_as]
 #[derive(Serialize, Deserialize, Debug, Default, Clone, Hash, PartialEq, Eq)]
 pub struct Storage {
-    pub blocks: BlockStorage,
+    #[serde_as(as = "FromInto<BlockStorage>")]
+    pub blocks: BlockConfig,
     pub indexes: IndexStorage,
 }
 
 #[derive(Serialize, Deserialize, Debug, Clone, Hash, PartialEq, Eq)]
 #[serde(tag = "type")]
 pub enum BlockStorage {
-    Local(LocalBlockStorage),
-    S3(S3BlockStorage),
-}
-
-#[derive(Serialize, Deserialize, Debug, Clone, Hash, PartialEq, Eq)]
-pub struct LocalBlockStorage {
-    pub path: PathBuf,
-}
-
-#[serde_as]
-#[derive(Serialize, Deserialize, Debug, Clone, Hash, PartialEq, Eq)]
-pub struct S3BlockStorage {
-    pub bucket: String,
-    #[serde_as(as = "Option<DisplayFromStr>")]
-    #[serde(default)]
-    pub endpoint: Option<Uri>,
-    pub dynamodb: DynamoStorage,
+    Local(FileSystemConfig),
+    S3(S3BlockConfig),
 }
 
 #[derive(Serialize, Deserialize, Debug, Clone, Hash, PartialEq, Eq)]
@@ -114,15 +105,7 @@ impl Default for LoggingFormat {
 
 impl Default for BlockStorage {
     fn default() -> BlockStorage {
-        BlockStorage::Local(LocalBlockStorage::default())
-    }
-}
-
-impl Default for LocalBlockStorage {
-    fn default() -> LocalBlockStorage {
-        LocalBlockStorage {
-            path: PathBuf::from(r"/tmp/kepler/blocks"),
-        }
+        BlockStorage::Local(FileSystemConfig::default())
     }
 }
 
diff --git a/src/lib.rs b/src/lib.rs
index f8c35056..3da0a31d 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -27,27 +27,57 @@ pub mod storage;
 mod tracing;
 pub mod transport;
 
+use config::{BlockStorage, Config};
 use libp2p::{
     identity::{ed25519::Keypair as Ed25519Keypair, Keypair},
     PeerId,
 };
+use orbit::ProviderUtils;
 use relay::RelayNode;
 use routes::{delegate, invoke, open_host_key, relay_addr, util_routes::*};
 use std::{collections::HashMap, sync::RwLock};
+use storage::{
+    either::Either,
+    file_system::{FileSystemConfig, FileSystemStore},
+    s3::{S3BlockConfig, S3BlockStore},
+};
 
 pub type Block = OBlock<DefaultParams>;
+pub type BlockStores = Either<S3BlockStore, FileSystemStore>;
+pub type BlockConfig = Either<S3BlockConfig, FileSystemConfig>;
+
+impl Default for BlockConfig {
+    fn default() -> Self {
+        Self::B(FileSystemConfig::default())
+    }
+}
+
+impl From<BlockStorage> for BlockConfig {
+    fn from(c: BlockStorage) -> Self {
+        match c {
+            BlockStorage::S3(s) => Self::A(s),
+            BlockStorage::Local(l) => Self::B(l),
+        }
+    }
+}
+
+impl From<BlockConfig> for BlockStorage {
+    fn from(c: BlockConfig) -> Self {
+        match c {
+            BlockConfig::A(a) => Self::S3(a),
+            BlockConfig::B(b) => Self::Local(b),
+        }
+    }
+}
 
 pub async fn app(config: &Figment) -> Result<Rocket<Build>> {
-    let kepler_config = config.extract::<config::Config>()?;
+    let kepler_config: Config = config.extract::<Config>()?.into();
 
     tracing::tracing_try_init(&kepler_config.log);
 
     storage::KV::healthcheck(kepler_config.storage.indexes.clone()).await?;
-    storage::StorageUtils::new(kepler_config.storage.blocks.clone())
-        .healthcheck()
-        .await?;
+    let kp = kepler_config.storage.blocks.relay_key_pair().await?;
 
-    let kp = storage::StorageUtils::relay_key_pair(kepler_config.storage.blocks).await?;
     let relay_node = RelayNode::new(kepler_config.relay.port, Keypair::Ed25519(kp)).await?;
 
     let routes = routes![
diff --git a/src/routes/mod.rs b/src/routes/mod.rs
index 72846a2d..e9c989f0 100644
--- a/src/routes/mod.rs
+++ b/src/routes/mod.rs
@@ -24,8 +24,9 @@ use crate::{
     auth_guards::{DelegateAuthWrapper, InvokeAuthWrapper, KVAction},
     kv::ObjectBuilder,
     relay::RelayNode,
-    storage::{BlockStores, ImmutableStore},
+    storage::ImmutableStore,
     tracing::TracingSpan,
+    BlockStores,
 };
 use tokio_util::compat::{FuturesAsyncReadCompatExt, TokioAsyncReadCompatExt};
 
diff --git a/src/storage/file_system.rs b/src/storage/file_system.rs
index 7dd5ed76..b878b639 100644
--- a/src/storage/file_system.rs
+++ b/src/storage/file_system.rs
@@ -58,6 +58,14 @@ impl StorageConfig<FileSystemStore> for FileSystemConfig {
     }
 }
 
+impl Default for FileSystemConfig {
+    fn default() -> Self {
+        Self {
+            path: PathBuf::from(r"/tmp/kepler/blocks"),
+        }
+    }
+}
+
 #[derive(thiserror::Error, Debug)]
 pub enum ProviderError {
     #[error(transparent)]
diff --git a/src/storage/mod.rs b/src/storage/mod.rs
index 4aa316e5..dc59480b 100644
--- a/src/storage/mod.rs
+++ b/src/storage/mod.rs
@@ -1,41 +1,24 @@
 use anyhow::{Error, Result};
-use aws_sdk_s3::{
-    error::{GetObjectError, GetObjectErrorKind},
-    types::{ByteStream, SdkError},
-};
-use aws_smithy_http::body::SdkBody;
-use kepler_lib::libipld::cid::{multibase::Base, multihash::Multihash, Cid};
+use kepler_lib::libipld::cid::{multihash::Multihash, Cid};
 use kepler_lib::resource::OrbitId;
-use libp2p::identity::ed25519::Keypair as Ed25519Keypair;
-use rocket::tokio::fs;
-use std::{collections::HashMap, error::Error as StdError, path::PathBuf, str::FromStr};
+use std::{collections::HashMap, error::Error as StdError};
 use tracing::instrument;
 
 mod dynamodb;
-mod either;
-mod file_system;
+pub mod either;
+pub mod file_system;
 mod indexes;
-mod s3;
+pub mod s3;
 mod utils;
 
 pub use indexes::KV;
 
-use crate::config;
-
-pub struct StorageUtils {
-    config: config::BlockStorage,
-}
-
-pub type BlockStores = either::Either<s3::S3BlockStore, file_system::FileSystemStore>;
-
 #[derive(Debug)]
 pub enum DataStores {
     S3(Box<s3::S3DataStore>),
     Local(Box<file_system::FileSystemStore>),
 }
 
-pub type BlockConfig = either::Either<s3::S3BlockConfig, file_system::FileSystemConfig>;
-
 #[async_trait]
 pub trait StorageConfig<S> {
     type Error: StdError;
diff --git a/src/storage/s3.rs b/src/storage/s3.rs
index be892851..9719a374 100644
--- a/src/storage/s3.rs
+++ b/src/storage/s3.rs
@@ -23,7 +23,6 @@ use serde_with::{serde_as, DisplayFromStr};
 use std::{io::Error as IoError, path::PathBuf, str::FromStr};
 
 use crate::{
-    config,
     orbit::ProviderUtils,
     storage::{dynamodb::DynamoPinStore, ImmutableStore, StorageConfig},
 };
@@ -308,7 +307,7 @@ impl ImmutableStore for S3BlockStore {
     }
 }
 
-fn path_to_config(path: PathBuf) -> (config::S3BlockStorage, Cid) {
+fn path_to_config(path: PathBuf) -> (S3BlockConfig, Cid) {
     let re =
             Regex::new(r"^/s3bucket/(?P<bucket>.*)/s3endpoint/(?P<s3endpoint>.*)/dynamotable/(?P<table>.*)/dynamoendpoint/(?P<dynamoendpoint>.*)/orbitcid/(?P<orbit>.*)/(blockstore|datastore)$")
                 .unwrap();
@@ -323,13 +322,13 @@ fn path_to_config(path: PathBuf) -> (config::S3BlockStorage, Cid) {
         .map(|e| Uri::from_str(e).unwrap());
     let orbit = Cid::from_str(fields.name("orbit").unwrap().as_str()).unwrap();
 
-    let config = config::S3BlockStorage {
+    let config = S3BlockConfig {
         bucket: s3_bucket,
         endpoint: s3_endpoint,
-        dynamodb: config::DynamoStorage {
-            table: dynamo_table,
-            endpoint: dynamo_endpoint,
-        },
+        // dynamodb: config::DynamoStorage {
+        //     table: dynamo_table,
+        //     endpoint: dynamo_endpoint,
+        // },
     };
     (config, orbit)
 }

From d6aa092cf2741d9641ddda1050cc3ecad3b0a203 Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Wed, 2 Nov 2022 13:20:33 +0100
Subject: [PATCH 35/73] fix orbit loading/creation in routes

---
 src/auth_guards.rs | 38 +++++++++++++++++++++++++++++---------
 src/orbit.rs       | 43 ++++++++++++++++++++++++-------------------
 2 files changed, 53 insertions(+), 28 deletions(-)

diff --git a/src/auth_guards.rs b/src/auth_guards.rs
index 411f6189..06525287 100644
--- a/src/auth_guards.rs
+++ b/src/auth_guards.rs
@@ -4,7 +4,7 @@ use crate::config;
 use crate::orbit::{create_orbit, load_orbit, Orbit};
 use crate::relay::RelayNode;
 use crate::routes::Metadata;
-use crate::storage::BlockStores;
+use crate::{BlockConfig, BlockStores};
 use anyhow::Result;
 use kepler_lib::{
     libipld::Cid,
@@ -143,7 +143,13 @@ impl<'l> FromRequest<'l> for DelegateAuthWrapper {
                 .map(|((orbit_id, peer), (config, relay, token))| async move {
                     match (
                         peer,
-                        load_orbit(orbit_id.get_cid(), &config, relay.clone()).await,
+                        load_orbit(
+                            (*orbit_id).clone(),
+                            &config.storage.blocks,
+                            &config.storage.indexes,
+                            relay.clone(),
+                        )
+                        .await,
                     ) {
                         (Some(p), Ok(None)) => {
                             let keys = match req
@@ -182,7 +188,15 @@ impl<'l> FromRequest<'l> for DelegateAuthWrapper {
                                 }
                             };
 
-                            match create_orbit(orbit_id, &config, &[], relay.clone(), kp).await {
+                            match create_orbit(
+                                orbit_id,
+                                &config.storage.blocks,
+                                &config.storage.indexes,
+                                relay.clone(),
+                                kp,
+                            )
+                            .await
+                            {
                                 Ok(Some(orbit)) => Ok(orbit),
                                 Ok(None) => Err(conflict(anyhow!("Orbit already exists"))),
                                 Err(e) => Err(internal_server_error(e)),
@@ -301,12 +315,18 @@ impl<'l> FromRequest<'l> for InvokeAuthWrapper<BlockStores> {
             let res = match target.resource.service() {
                 None => bad_request(anyhow!("missing service in invocation target")),
                 Some("kv") => {
-                    let orbit =
-                        match load_orbit(target.resource.orbit().get_cid(), &config, relay).await {
-                            Ok(Some(o)) => o,
-                            Ok(None) => return not_found(anyhow!("No Orbit found")),
-                            Err(e) => return internal_server_error(e),
-                        };
+                    let orbit = match load_orbit(
+                        target.resource.orbit().clone(),
+                        &config.storage.blocks,
+                        &config.storage.indexes,
+                        relay,
+                    )
+                    .await
+                    {
+                        Ok(Some(o)) => o,
+                        Ok(None) => return not_found(anyhow!("No Orbit found")),
+                        Err(e) => return internal_server_error(e),
+                    };
                     let auth_ref = match orbit.capabilities.invoke([token.clone()]).await {
                         Ok(c) => c,
                         Err(InvokeError::Unauthorized(e)) => return unauthorized(e),
diff --git a/src/orbit.rs b/src/orbit.rs
index a4150d1f..8bbcb645 100644
--- a/src/orbit.rs
+++ b/src/orbit.rs
@@ -190,27 +190,32 @@ pub async fn load_orbit(
 // 100 orbits => 600 FDs
 #[cached(size = 100, result = true, sync_writes = true)]
 async fn load_orbit_inner(
-    orbit: Cid,
-    config: config::Config,
+    orbit: OrbitId,
+    store_config: BlockConfig,
+    index_config: config::IndexStorage,
     relay: (PeerId, Multiaddr),
 ) -> Result<Orbit<BlockStores>> {
-    let storage_utils = StorageUtils::new(config.storage.blocks.clone());
-    let id = storage_utils
-        .orbit_id(orbit)
-        .await?
-        .ok_or_else(|| anyhow!("Orbit `{}` doesn't have its orbit URL stored.", orbit))?;
-
-    let md = Manifest::resolve_dyn(&id, None)
-        .await?
-        .ok_or_else(|| anyhow!("Orbit DID Document not resolvable"))?;
-
-    // let kp = Ed25519Keypair::decode(&mut fs::read(dir.join("kp")).await?)?;
-    let kp = storage_utils.key_pair(orbit).await?.unwrap();
-
-    debug!("loading orbit {}", &id);
-
-    let orbit = Orbit::new(&config, kp, md, Some(relay)).await?;
-    Ok(orbit)
+    debug!("loading orbit {}", &orbit);
+    Orbit::open(
+        &OrbitPeerConfigBuilder::<BlockConfig, config::IndexStorage>::default()
+            .manifest(
+                Manifest::resolve_dyn(&orbit, None)
+                    .await?
+                    .ok_or_else(|| anyhow!("Orbit DID Document not resolvable"))?,
+            )
+            .identity(
+                store_config
+                    .key_pair(&orbit)
+                    .await?
+                    .ok_or_else(|| anyhow!("Peer Identity key could not be found"))?,
+            )
+            .blocks(store_config)
+            .index(index_config)
+            .relay(relay)
+            .build()?,
+    )
+    .await?
+    .ok_or_else(|| anyhow!("Orbit could not be opened: not found"))
 }
 
 pub fn hash_same<B: AsRef<[u8]>>(c: &Cid, b: B) -> Result<Cid> {

From b75fddb88f749a978323cd809f55e976520d3d09 Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Wed, 2 Nov 2022 14:02:53 +0100
Subject: [PATCH 36/73] fix orbit creation

---
 src/orbit.rs | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/src/orbit.rs b/src/orbit.rs
index 8bbcb645..854bc3f0 100644
--- a/src/orbit.rs
+++ b/src/orbit.rs
@@ -165,6 +165,26 @@ pub async fn create_orbit(
 
     store_config.setup_orbit(&id, &kp).await?;
 
+    Orbit::create(
+        &OrbitPeerConfigBuilder::<BlockConfig, config::IndexStorage>::default()
+            .manifest(
+                Manifest::resolve_dyn(&id, None)
+                    .await?
+                    .ok_or_else(|| anyhow!("Orbit DID Document not resolvable"))?,
+            )
+            .identity(
+                store_config
+                    .key_pair(&id)
+                    .await?
+                    .ok_or_else(|| anyhow!("Peer Identity key could not be found"))?,
+            )
+            .blocks(store_config.clone())
+            .index(index_config.clone())
+            .relay(relay.clone())
+            .build()?,
+    )
+    .await?;
+
     Ok(Some(
         load_orbit(id.clone(), store_config, index_config, relay)
             .await

From 91077a93c5b3b34d7762126a75d6e608f34096e0 Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Wed, 2 Nov 2022 15:34:53 +0100
Subject: [PATCH 37/73] remove s3datastore and PinStore impl for dynamo

---
 src/storage/dynamodb.rs     | 301 ------------------------------------
 src/storage/mod.rs          |   6 -
 src/storage/s3.rs           |  65 +-------
 test/load/signer/Cargo.toml |   1 -
 4 files changed, 1 insertion(+), 372 deletions(-)

diff --git a/src/storage/dynamodb.rs b/src/storage/dynamodb.rs
index 0fc329b2..2f256e21 100644
--- a/src/storage/dynamodb.rs
+++ b/src/storage/dynamodb.rs
@@ -59,304 +59,3 @@ impl DynamoPinStore {
         Ok(())
     }
 }
-
-// TODO implement mutex
-
-// TODO make that public in rust-ipfs
-pub type References<'a> = futures::stream::BoxStream<'a, Result<Cid, IpldRefsError>>;
-
-#[async_trait]
-impl PinStore for DynamoPinStore {
-    async fn is_pinned(&self, cid: &Cid) -> Result<bool, Error> {
-        match self
-            .client
-            .lock()
-            .await
-            .get_item()
-            .table_name(self.table.clone())
-            .key(
-                CID_ATTRIBUTE,
-                AttributeValue::S(format!(
-                    "{}/{}",
-                    self.orbit.to_string_of_base(Base::Base58Btc)?,
-                    cid
-                )),
-            )
-            .send()
-            .await
-        {
-            Ok(_) => Ok(true),
-            Err(SdkError::ServiceError {
-                err:
-                    GetItemError {
-                        kind: GetItemErrorKind::ResourceNotFoundException(_),
-                        ..
-                    },
-                ..
-            }) => Ok(false),
-            Err(e) => Err(anyhow!("Error checking on item: {}", e)),
-        }
-    }
-
-    /// Only using recursive pins
-    async fn insert_direct_pin(&self, _target: &Cid) -> Result<(), Error> {
-        Ok(())
-        // Assuming a direct pin can't be already recursively/indirectly pinned already (i.e. already exist);
-    }
-
-    async fn insert_recursive_pin(
-        &self,
-        target: &Cid,
-        referenced: References<'_>,
-    ) -> Result<(), Error> {
-        let client = self.client.lock().await;
-        // TODO either insert or increment
-        client
-            .update_item()
-            .table_name(self.table.clone())
-            .key(
-                CID_ATTRIBUTE,
-                AttributeValue::S(format!(
-                    "{}/{}",
-                    self.orbit.to_string_of_base(Base::Base58Btc)?,
-                    target
-                )),
-            )
-            .update_expression(format!(
-                "SET {} = :pin, {p} = {p} + :increment",
-                ROOT_ATTRIBUTE,
-                p = PARENTS_ATTRIBUTE
-            ))
-            .expression_attribute_values(":pin", AttributeValue::Bool(true))
-            .expression_attribute_values(":increment", AttributeValue::N(1.to_string()))
-            .send()
-            .await?;
-
-        let set = referenced.try_collect::<BTreeSet<_>>().await?;
-        for cid in set.iter() {
-            client
-                .update_item()
-                .table_name(self.table.clone())
-                .key(
-                    CID_ATTRIBUTE,
-                    AttributeValue::S(format!(
-                        "{}/{}",
-                        self.orbit.to_string_of_base(Base::Base58Btc)?,
-                        cid
-                    )),
-                )
-                .update_expression(format!("SET {p} = {p} + :increment", p = PARENTS_ATTRIBUTE))
-                .expression_attribute_values(":increment", AttributeValue::N(1.to_string()))
-                .send()
-                .await?;
-        }
-        Ok(())
-    }
-
-    /// Only using recursive pins
-    async fn remove_direct_pin(&self, _target: &Cid) -> Result<(), Error> {
-        Ok(())
-    }
-
-    async fn remove_recursive_pin(
-        &self,
-        target: &Cid,
-        referenced: References<'_>,
-    ) -> Result<(), Error> {
-        let client = self.client.lock().await;
-        let res = client
-            .update_item()
-            .table_name(self.table.clone())
-            .key(
-                CID_ATTRIBUTE,
-                AttributeValue::S(format!(
-                    "{}/{}",
-                    self.orbit.to_string_of_base(Base::Base58Btc)?,
-                    target
-                )),
-            )
-            .update_expression(format!(
-                "SET {} = :pin, {p} = {p} - :increment",
-                ROOT_ATTRIBUTE,
-                p = PARENTS_ATTRIBUTE
-            ))
-            .expression_attribute_values(":pin", AttributeValue::Bool(false))
-            .expression_attribute_values(":increment", AttributeValue::N(1.to_string()))
-            .return_values(ReturnValue::UpdatedNew)
-            .send()
-            .await?;
-
-        // TODO use a conditional delete
-        match res
-            .attributes
-            .and_then(|m| m.get(PARENTS_ATTRIBUTE).cloned())
-        {
-            Some(AttributeValue::N(parents)) => {
-                if *parents == 0.to_string() {
-                    client
-                        .delete_item()
-                        .table_name(self.table.clone())
-                        .key(
-                            CID_ATTRIBUTE,
-                            AttributeValue::S(format!(
-                                "{}/{}",
-                                self.orbit.to_string_of_base(Base::Base58Btc)?,
-                                target
-                            )),
-                        )
-                        .send()
-                        .await?;
-                }
-            }
-            _ => error!("No attribute returned."),
-        };
-
-        let set = referenced.try_collect::<BTreeSet<_>>().await?;
-        for cid in set.iter() {
-            let res = client
-                .update_item()
-                .table_name(self.table.clone())
-                .key(
-                    CID_ATTRIBUTE,
-                    AttributeValue::S(format!(
-                        "{}/{}",
-                        self.orbit.to_string_of_base(Base::Base58Btc)?,
-                        cid
-                    )),
-                )
-                .update_expression(format!("SET {p} = {p} - :increment", p = PARENTS_ATTRIBUTE))
-                .expression_attribute_values(":increment", AttributeValue::N(1.to_string()))
-                .return_values(ReturnValue::UpdatedNew)
-                .send()
-                .await?;
-
-            match res.attributes.map(|m| m.get(PARENTS_ATTRIBUTE).cloned()) {
-                Some(Some(AttributeValue::N(parents))) => {
-                    if *parents == 0.to_string() {
-                        client
-                            .delete_item()
-                            .table_name(self.table.clone())
-                            .key(
-                                CID_ATTRIBUTE,
-                                AttributeValue::S(format!(
-                                    "{}/{}",
-                                    self.orbit.to_string_of_base(Base::Base58Btc)?,
-                                    target
-                                )),
-                            )
-                            .send()
-                            .await?;
-                    }
-                }
-                _ => error!("No attribute returned."),
-            };
-        }
-        Ok(())
-    }
-
-    async fn list(
-        &self,
-        requirement: Option<PinMode>,
-    ) -> futures::stream::BoxStream<'static, Result<(Cid, PinMode), Error>> {
-        let query = self
-            .client
-            .lock()
-            .await
-            .scan()
-            .table_name(self.table.clone());
-        let query = match requirement {
-            Some(PinMode::Recursive | PinMode::Direct) => query
-                .filter_expression(format!("{} = :r", ROOT_ATTRIBUTE))
-                .expression_attribute_values(":r", AttributeValue::Bool(true)),
-            None | Some(PinMode::Indirect) => query,
-        };
-        // TODO handle pagination
-        match query.send().await {
-            Ok(ScanOutput {
-                items: Some(items), ..
-            }) => stream::iter(items)
-                .map(|map| {
-                    let cid = match map.get(CID_ATTRIBUTE) {
-                        Some(AttributeValue::S(c)) => c,
-                        _ => return Err(anyhow!("Row with no string Cid key.")),
-                    };
-                    let pin_mode = match map.get(ROOT_ATTRIBUTE) {
-                        Some(AttributeValue::Bool(true)) => PinMode::Recursive,
-                        Some(AttributeValue::Bool(false)) => PinMode::Indirect,
-                        Some(_) | None => {
-                            error!("Cid `{}` with no boolean Root attribute.", cid);
-                            return Err(anyhow!("Cid `{}` with no boolean Root attribute.", cid));
-                        }
-                    };
-                    Cid::from_str(cid)
-                        .map_err(|e| anyhow!("Couldn't convert cid key to Cid: {}", e))
-                        .map(|kk| (kk, pin_mode))
-                })
-                .boxed(),
-            Ok(ScanOutput { items: None, .. }) => stream::iter(vec![]).boxed(),
-            Err(e) => stream::iter(vec![Err(anyhow!("Error checking on item: {}", e))]).boxed(),
-        }
-    }
-
-    async fn query(
-        &self,
-        ids: Vec<Cid>,
-        requirement: Option<PinMode>,
-    ) -> Result<Vec<(Cid, PinKind<Cid>)>, Error> {
-        // TODO impl PinKind::IndirectFrom
-        let query = self
-            .client
-            .lock()
-            .await
-            .scan()
-            .table_name(self.table.clone());
-        let (query, expression) = match requirement {
-            Some(PinMode::Recursive | PinMode::Direct) => (
-                query.expression_attribute_values(":r", AttributeValue::Bool(true)),
-                format!("{} == :r and", ROOT_ATTRIBUTE),
-            ),
-            None | Some(PinMode::Indirect) => (query, String::new()),
-        };
-        let query = query
-            .filter_expression(format!("{} contains (:ids, {})", expression, CID_ATTRIBUTE))
-            .expression_attribute_values(
-                ":ids",
-                AttributeValue::Ns(
-                    ids.iter()
-                        .map(|id| {
-                            self.orbit
-                                .to_string_of_base(Base::Base58Btc)
-                                .map(|c| format!("{}/{}", c, id))
-                        })
-                        .collect::<Result<Vec<_>, _>>()?,
-                ),
-            );
-        // TODO handle pagination
-        match query.send().await {
-            Ok(ScanOutput {
-                items: Some(items), ..
-            }) => items
-                .iter()
-                .map(|map| {
-                    let cid = match map.get(CID_ATTRIBUTE) {
-                        Some(AttributeValue::S(c)) => c,
-                        _ => return Err(anyhow!("Row with no string Cid key.")),
-                    };
-                    let pin_mode = match map.get(ROOT_ATTRIBUTE) {
-                        Some(AttributeValue::Bool(true)) => PinKind::Recursive(0),
-                        Some(AttributeValue::Bool(false)) => PinKind::IndirectFrom(Cid::default()),
-                        Some(_) | None => {
-                            error!("Cid `{}` with no boolean Root attribute.", cid);
-                            return Err(anyhow!("Cid `{}` with no boolean Root attribute.", cid));
-                        }
-                    };
-                    Cid::from_str(cid)
-                        .map_err(|e| anyhow!("Couldn't convert cid key to Cid: {}", e))
-                        .map(|kk| (kk, pin_mode))
-                })
-                .collect(),
-            Ok(ScanOutput { items: None, .. }) => Ok(vec![]),
-            Err(e) => Err(anyhow!("Error checking on item: {}", e)),
-        }
-    }
-}
diff --git a/src/storage/mod.rs b/src/storage/mod.rs
index dc59480b..e32465d7 100644
--- a/src/storage/mod.rs
+++ b/src/storage/mod.rs
@@ -13,12 +13,6 @@ mod utils;
 
 pub use indexes::KV;
 
-#[derive(Debug)]
-pub enum DataStores {
-    S3(Box<s3::S3DataStore>),
-    Local(Box<file_system::FileSystemStore>),
-}
-
 #[async_trait]
 pub trait StorageConfig<S> {
     type Error: StdError;
diff --git a/src/storage/s3.rs b/src/storage/s3.rs
index 9719a374..6a171c89 100644
--- a/src/storage/s3.rs
+++ b/src/storage/s3.rs
@@ -24,19 +24,9 @@ use std::{io::Error as IoError, path::PathBuf, str::FromStr};
 
 use crate::{
     orbit::ProviderUtils,
-    storage::{dynamodb::DynamoPinStore, ImmutableStore, StorageConfig},
+    storage::{ImmutableStore, StorageConfig},
 };
 
-#[derive(Debug)]
-pub struct S3DataStore {
-    // TODO Remove is unused (orbit::delete is never called).
-    // When that changes we will need to use a mutex, either local or in Dynamo
-    pub client: Client,
-    pub bucket: String,
-    pub dynamodb: DynamoPinStore,
-    pub orbit: Cid,
-}
-
 // TODO we could use the same struct for both the block store and the data
 // (pin) store, but we need to remember that for now it will be two different
 // objects in rust-ipfs
@@ -56,7 +46,6 @@ pub struct S3BlockConfig {
     #[serde_as(as = "Option<DisplayFromStr>")]
     #[serde(default)]
     pub endpoint: Option<Uri>,
-    // pub dynamodb: DynamoStorage,
 }
 
 #[async_trait]
@@ -184,58 +173,6 @@ pub fn new_client(config: &S3BlockConfig) -> Client {
     Client::from_conf(sdk_config)
 }
 
-impl S3DataStore {
-    pub fn new_(config: S3BlockConfig, orbit: Cid) -> Self {
-        S3DataStore {
-            client: new_client(&config),
-            bucket: config.bucket,
-            dynamodb: todo!(),
-            orbit,
-        }
-    }
-
-    pub async fn get_(&self, key: String) -> Result<Option<Vec<u8>>, Error> {
-        let res = self
-            .client
-            .get_object()
-            .bucket(self.bucket.clone())
-            .key(format!(
-                "{}/{}",
-                self.orbit.to_string_of_base(Base::Base58Btc)?,
-                key
-            ))
-            .send()
-            .await;
-        match res {
-            Ok(o) => Ok(Some(o.body.collect().await?.into_bytes().to_vec())),
-            Err(SdkError::ServiceError {
-                err:
-                    GetObjectError {
-                        kind: GetObjectErrorKind::NoSuchKey(_),
-                        ..
-                    },
-                ..
-            }) => Ok(None),
-            Err(e) => Err(e.into()),
-        }
-    }
-
-    pub async fn put_(&self, key: String, body: Vec<u8>) -> Result<(), Error> {
-        self.client
-            .put_object()
-            .bucket(self.bucket.clone())
-            .key(format!(
-                "{}/{}",
-                self.orbit.to_string_of_base(Base::Base58Btc)?,
-                key
-            ))
-            .body(ByteStream::new(SdkBody::from(body)))
-            .send()
-            .await?;
-        Ok(())
-    }
-}
-
 impl S3BlockStore {
     pub fn new_(config: &S3BlockConfig, orbit: Cid) -> Self {
         S3BlockStore {
diff --git a/test/load/signer/Cargo.toml b/test/load/signer/Cargo.toml
index 7869f1e2..18f75d33 100644
--- a/test/load/signer/Cargo.toml
+++ b/test/load/signer/Cargo.toml
@@ -7,7 +7,6 @@ edition = "2021"
 
 [dependencies]
 axum = "0.5.6"
-chrono = "0.4.19"
 ethers = "0.6.2"
 iri-string = "0.4"
 kepler-lib = { path = "../../../lib" }

From 66de638135ac02542333606aaff988d8a8e5a586 Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Wed, 2 Nov 2022 16:35:45 +0100
Subject: [PATCH 38/73] use pin_project, remove unsafe in AsyncReadEither

---
 Cargo.toml            |  1 +
 src/storage/either.rs | 29 +++++++++++------------------
 2 files changed, 12 insertions(+), 18 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index 8fe0cd3b..619eb102 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -29,6 +29,7 @@ libipld = "0.14"
 libp2p = { default-features = false, features = ["floodsub", "identify", "kad", "tcp-tokio", "mplex", "noise", "ping", "yamux", "dns-tokio", "relay"], version = "0.49.0" }
 opentelemetry = { version = "0.17.0", features = ["rt-tokio"] }
 opentelemetry-jaeger = { version = "0.16.0", features = ["rt-tokio", "reqwest_collector_client"] }
+pin-project = "1"
 prometheus = { version = "0.13.0", features = ["process"] }
 regex = "1.5"
 reqwest = { version = "0.11", features = ["json"] }
diff --git a/src/storage/either.rs b/src/storage/either.rs
index 409d11f0..43a77e5d 100644
--- a/src/storage/either.rs
+++ b/src/storage/either.rs
@@ -7,11 +7,9 @@ use futures::{
     io::{AsyncRead, Error},
     task::{Context, Poll},
 };
-use kepler_lib::{
-    libipld::cid::{multihash::Multihash, Cid},
-    resource::OrbitId,
-};
+use kepler_lib::{libipld::cid::multihash::Multihash, resource::OrbitId};
 use libp2p::identity::ed25519::Keypair as Ed25519Keypair;
+use pin_project::pin_project;
 
 #[derive(Debug, Clone, Hash, PartialEq, Eq)]
 pub enum Either<A, B> {
@@ -19,14 +17,15 @@ pub enum Either<A, B> {
     B(B),
 }
 
+#[pin_project(project = AsyncReadEitherProjection)]
 #[derive(Debug, Clone)]
 pub enum AsyncReadEither<A, B>
 where
     A: ImmutableStore,
     B: ImmutableStore,
 {
-    A(A::Readable),
-    B(B::Readable),
+    A(#[pin] A::Readable),
+    B(#[pin] B::Readable),
 }
 
 impl<A, B> AsyncRead for AsyncReadEither<A, B>
@@ -40,13 +39,9 @@ where
         cx: &mut Context<'_>,
         buf: &mut [u8],
     ) -> Poll<Result<usize, Error>> {
-        // it actually seems like this is only possible with unsafe :(
-        // TODO use pin-project crate
-        unsafe {
-            match self.get_unchecked_mut() {
-                Self::A(l) => AsyncRead::poll_read(Pin::new_unchecked(l), cx, buf),
-                Self::B(r) => AsyncRead::poll_read(Pin::new_unchecked(r), cx, buf),
-            }
+        match self.project() {
+            AsyncReadEitherProjection::A(a) => a.poll_read(cx, buf),
+            AsyncReadEitherProjection::B(b) => b.poll_read(cx, buf),
         }
     }
     #[inline]
@@ -55,11 +50,9 @@ where
         cx: &mut Context<'_>,
         bufs: &mut [std::io::IoSliceMut<'_>],
     ) -> Poll<Result<usize, Error>> {
-        unsafe {
-            match self.get_unchecked_mut() {
-                Self::A(l) => AsyncRead::poll_read_vectored(Pin::new_unchecked(l), cx, bufs),
-                Self::B(r) => AsyncRead::poll_read_vectored(Pin::new_unchecked(r), cx, bufs),
-            }
+        match self.project() {
+            AsyncReadEitherProjection::A(a) => a.poll_read_vectored(cx, bufs),
+            AsyncReadEitherProjection::B(b) => b.poll_read_vectored(cx, bufs),
         }
     }
 }

From 6afac134bd71d0c92118cdb61a021a7dd69df1b3 Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Thu, 3 Nov 2022 10:25:06 +0100
Subject: [PATCH 39/73] small cleanup, warnings

---
 src/auth_guards.rs |  2 +-
 src/orbit.rs       | 49 +++++++++++++++++-----------------------------
 src/relay.rs       |  7 +------
 3 files changed, 20 insertions(+), 38 deletions(-)

diff --git a/src/auth_guards.rs b/src/auth_guards.rs
index 06525287..cf6fb05d 100644
--- a/src/auth_guards.rs
+++ b/src/auth_guards.rs
@@ -4,7 +4,7 @@ use crate::config;
 use crate::orbit::{create_orbit, load_orbit, Orbit};
 use crate::relay::RelayNode;
 use crate::routes::Metadata;
-use crate::{BlockConfig, BlockStores};
+use crate::BlockStores;
 use anyhow::Result;
 use kepler_lib::{
     libipld::Cid,
diff --git a/src/orbit.rs b/src/orbit.rs
index 854bc3f0..74e72b3d 100644
--- a/src/orbit.rs
+++ b/src/orbit.rs
@@ -1,7 +1,7 @@
 use crate::{
     capabilities::{store::Store as CapStore, Service as CapService},
     config,
-    kv::{behaviour::BehaviourProcess, Service as KVService, Store},
+    kv::{Service as KVService, Store},
     manifest::Manifest,
     storage::{ImmutableStore, StorageConfig},
     BlockConfig, BlockStores,
@@ -46,19 +46,6 @@ impl<T> Deref for AbortOnDrop<T> {
     }
 }
 
-#[derive(Clone, Debug)]
-struct OrbitTasks {
-    _behaviour_process: BehaviourProcess,
-}
-
-impl OrbitTasks {
-    fn new(behaviour_process: BehaviourProcess) -> Self {
-        Self {
-            _behaviour_process: behaviour_process,
-        }
-    }
-}
-
 #[derive(Clone)]
 pub struct Orbit<B> {
     pub service: KVService<B>,
@@ -254,25 +241,25 @@ mod tests {
         jwk::JWK,
     };
     use std::convert::TryInto;
-    use tempdir::TempDir;
 
     async fn op(md: Manifest) -> anyhow::Result<Orbit<BlockStores>> {
-        let dir = TempDir::new(&md.id().get_cid().to_string())
-            .unwrap()
-            .path()
-            .to_path_buf();
-        let config = config::Config {
-            storage: config::Storage {
-                blocks: config::BlockStorage::Local(config::LocalBlockStorage {
-                    path: dir.clone(),
-                }),
-                indexes: config::IndexStorage::Local(config::LocalIndexStorage {
-                    path: dir.clone(),
-                }),
-            },
-            ..Default::default()
-        };
-        Orbit::new(&config, Ed25519Keypair::generate(), md, None).await
+        // let dir = Tempfile::new(&md.id().get_cid().to_string())
+        //     .unwrap()
+        //     .path()
+        //     .to_path_buf();
+        // let config = config::Config {
+        //     storage: config::Storage {
+        //         blocks: config::BlockStorage::Local(config::LocalBlockStorage {
+        //             path: dir.clone(),
+        //         }),
+        //         indexes: config::IndexStorage::Local(config::LocalIndexStorage {
+        //             path: dir.clone(),
+        //         }),
+        //     },
+        //     ..Default::default()
+        // };
+        // Orbit::new(&config, Ed25519Keypair::generate(), md, None).await
+        todo!()
     }
 
     #[test]
diff --git a/src/relay.rs b/src/relay.rs
index df6aa39b..9de2967c 100644
--- a/src/relay.rs
+++ b/src/relay.rs
@@ -1,10 +1,5 @@
 use anyhow::Result;
-use libp2p::core::{
-    identity::Keypair, multiaddr::multiaddr, transport::MemoryTransport, Multiaddr, PeerId,
-};
-use rocket::tokio::spawn;
-
-use crate::orbit::AbortOnDrop;
+use libp2p::core::{identity::Keypair, multiaddr::multiaddr, Multiaddr, PeerId};
 
 pub struct RelayNode {
     pub port: u16,

From f1d7310900a9df57d2c6d9501b824ec2a01dc40f Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Thu, 3 Nov 2022 10:25:44 +0100
Subject: [PATCH 40/73] fully impl stores using HashBuffer

---
 Cargo.lock                 |  56 +------------------
 Cargo.toml                 |   4 +-
 src/capabilities/store.rs  |   2 +-
 src/kv/entries.rs          |   2 +-
 src/kv/mod.rs              |   2 +-
 src/storage/file_system.rs |  47 +++++++++++-----
 src/storage/s3.rs          | 110 ++++++++++++++++++++++++++++---------
 src/storage/utils.rs       |  57 ++++++++++++++++++-
 8 files changed, 178 insertions(+), 102 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 3747068e..37442bfa 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2113,12 +2113,6 @@ dependencies = [
  "winapi",
 ]
 
-[[package]]
-name = "fuchsia-cprng"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba"
-
 [[package]]
 name = "funty"
 version = "1.1.0"
@@ -2829,6 +2823,7 @@ dependencies = [
  "opentelemetry",
  "opentelemetry-jaeger",
  "percent-encoding",
+ "pin-project",
  "prometheus",
  "regex",
  "reqwest",
@@ -2837,7 +2832,7 @@ dependencies = [
  "serde_json",
  "serde_with 1.14.0",
  "sled",
- "tempdir",
+ "tempfile",
  "thiserror",
  "time 0.3.15",
  "tokio",
@@ -4548,19 +4543,6 @@ version = "0.6.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "643f8f41a8ebc4c5dc4515c82bb8abd397b527fc20fd681b7c011c2aee5d44fb"
 
-[[package]]
-name = "rand"
-version = "0.4.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293"
-dependencies = [
- "fuchsia-cprng",
- "libc",
- "rand_core 0.3.1",
- "rdrand",
- "winapi",
-]
-
 [[package]]
 name = "rand"
 version = "0.7.3"
@@ -4605,21 +4587,6 @@ dependencies = [
  "rand_core 0.6.4",
 ]
 
-[[package]]
-name = "rand_core"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b"
-dependencies = [
- "rand_core 0.4.2",
-]
-
-[[package]]
-name = "rand_core"
-version = "0.4.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc"
-
 [[package]]
 name = "rand_core"
 version = "0.5.1"
@@ -4647,15 +4614,6 @@ dependencies = [
  "rand_core 0.5.1",
 ]
 
-[[package]]
-name = "rdrand"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2"
-dependencies = [
- "rand_core 0.3.1",
-]
-
 [[package]]
 name = "redox_syscall"
 version = "0.2.16"
@@ -5829,16 +5787,6 @@ version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369"
 
-[[package]]
-name = "tempdir"
-version = "0.3.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8"
-dependencies = [
- "rand 0.4.6",
- "remove_dir_all",
-]
-
 [[package]]
 name = "tempfile"
 version = "3.3.0"
diff --git a/Cargo.toml b/Cargo.toml
index 619eb102..481b7d28 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -39,6 +39,7 @@ serde_json = "1"
 serde_with = { version = "1", features = ["hex"] }
 sled = "0.34"
 thiserror = "1"
+tempfile = "3"
 time = "0.3"
 tokio = { version = "1", features = ["rt", "macros", "rt-multi-thread"] }
 tokio-stream = { version = "0.1", features = ["fs"] }
@@ -55,9 +56,6 @@ percent-encoding = "2.1"
 [dependencies.kepler-lib]
 path = "lib/"
 
-[dev-dependencies]
-tempdir = "0.3.7"
-
 [workspace]
 
 members = [
diff --git a/src/capabilities/store.rs b/src/capabilities/store.rs
index f0334c69..3e9ace7a 100644
--- a/src/capabilities/store.rs
+++ b/src/capabilities/store.rs
@@ -622,7 +622,7 @@ mod test {
     // use crate::ipfs::create_ipfs;
     // use ipfs::Keypair;
     // async fn get_store(id: &OrbitId) -> Store {
-    //     let tmp = tempdir::TempDir::new("test_streams").unwrap();
+    //     let tmp = tempfile::TempDir::new("test_streams").unwrap();
     //     let kp = Keypair::generate_ed25519();
     //     let (ipfs, ipfs_task, receiver) = create_ipfs(id.to_string(), &tmp.path(), kp, [])
     //         .await
diff --git a/src/kv/entries.rs b/src/kv/entries.rs
index a0860366..6b8583eb 100644
--- a/src/kv/entries.rs
+++ b/src/kv/entries.rs
@@ -65,7 +65,7 @@ mod test {
     // #[tokio::test(flavor = "multi_thread")]
     // async fn write() -> Result<(), anyhow::Error> {
     //     tracing_try_init(&config::Logging::default());
-    //     let tmp = tempdir::TempDir::new("test_streams")?;
+    //     let tmp = tempfile::TempDir::new("test_streams")?;
     //     let data = vec![3u8; 1000000000 * 3];
 
     //     let mut config = IpfsOptions::inmemory_with_generated_keys();
diff --git a/src/kv/mod.rs b/src/kv/mod.rs
index d9337f54..9e72f630 100644
--- a/src/kv/mod.rs
+++ b/src/kv/mod.rs
@@ -159,7 +159,7 @@ mod test {
     //     let relay_peer_id = relay.id.clone();
     //     let relay_internal = relay.internal();
 
-    //     let tmp = tempdir::TempDir::new("test_streams")?;
+    //     let tmp = tempfile::TempDir::new("test_streams")?;
     //     let id =
     //         Cid::from_str("bafkreieq5jui4j25lacwomsqgjeswwl3y5zcdrresptwgmfylxo2depppq").unwrap();
 
diff --git a/src/storage/file_system.rs b/src/storage/file_system.rs
index b878b639..c4d57ddd 100644
--- a/src/storage/file_system.rs
+++ b/src/storage/file_system.rs
@@ -1,11 +1,14 @@
 use crate::{
     orbit::ProviderUtils,
-    storage::{ImmutableStore, StorageConfig},
+    storage::{utils::HashBuffer, ImmutableStore, StorageConfig},
 };
+use futures::io::{copy, AllowStdIo};
 use kepler_lib::{
     libipld::cid::{
         multibase::{encode, Base},
-        multihash::Multihash,
+        multihash::{
+            Blake3Hasher, Code, Error as MultihashError, Hasher, Multihash, MultihashDigest,
+        },
     },
     resource::OrbitId,
 };
@@ -15,6 +18,7 @@ use std::{
     io::{Error as IoError, ErrorKind},
     path::PathBuf,
 };
+use tempfile::{NamedTempFile, PersistError};
 use tokio::fs::{create_dir_all, read, remove_file, write, File};
 use tokio_util::compat::{Compat, TokioAsyncReadCompatExt};
 
@@ -112,9 +116,19 @@ impl ProviderUtils for FileSystemConfig {
     }
 }
 
+#[derive(thiserror::Error, Debug)]
+pub enum FileSystemStoreError {
+    #[error(transparent)]
+    Io(#[from] IoError),
+    #[error(transparent)]
+    Multihash(#[from] MultihashError),
+    #[error(transparent)]
+    Persist(#[from] PersistError),
+}
+
 #[async_trait]
 impl ImmutableStore for FileSystemStore {
-    type Error = IoError;
+    type Error = FileSystemStoreError;
     type Readable = Compat<File>;
     async fn contains(&self, id: &Multihash) -> Result<bool, Self::Error> {
         Ok(self.get_path(id).exists())
@@ -123,29 +137,32 @@ impl ImmutableStore for FileSystemStore {
         &self,
         data: impl futures::io::AsyncRead + Send,
     ) -> Result<Multihash, Self::Error> {
-        // TODO lock file to prevent overlapping writes
-        // only open to write if not existing AND not being written to right now
-        todo!();
-        // write into tmp then rename, to name after the hash
-        // need to stream data through a hasher into the file and return hash
-        // match File::open(path.join(cid.to_string())),await {
-        //     Ok(f) => copy(data, file).await
-        //     Err(e) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),
-        //     Err(e) => Err(e),
-        // }
+        let mut hb = HashBuffer::<Blake3Hasher<32>, AllowStdIo<NamedTempFile>>::new(
+            AllowStdIo::new(NamedTempFile::new_in(&self.path)?),
+        );
+
+        copy(data, &mut hb).await?;
+
+        let (mut hasher, file) = hb.into_inner();
+
+        let multihash = Code::Blake3_256.wrap(hasher.finalize())?;
+
+        let file = file.into_inner();
+        file.persist(self.get_path(&multihash))?;
+        Ok(multihash)
     }
     async fn remove(&self, id: &Multihash) -> Result<Option<()>, Self::Error> {
         match remove_file(self.get_path(id)).await {
             Ok(()) => Ok(Some(())),
             Err(e) if e.kind() == ErrorKind::NotFound => Ok(None),
-            Err(e) => Err(e),
+            Err(e) => Err(e.into()),
         }
     }
     async fn read(&self, id: &Multihash) -> Result<Option<Self::Readable>, Self::Error> {
         match File::open(self.get_path(id)).await {
             Ok(f) => Ok(Some(f.compat())),
             Err(e) if e.kind() == ErrorKind::NotFound => Ok(None),
-            Err(e) => Err(e),
+            Err(e) => Err(e.into()),
         }
     }
 }
diff --git a/src/storage/s3.rs b/src/storage/s3.rs
index 6a171c89..073e2bc0 100644
--- a/src/storage/s3.rs
+++ b/src/storage/s3.rs
@@ -1,16 +1,22 @@
-use anyhow::Error;
 use aws_sdk_s3::{
-    error::{GetObjectError, GetObjectErrorKind, PutObjectError},
+    error::{
+        GetObjectError, GetObjectErrorKind, HeadObjectError, HeadObjectErrorKind, PutObjectError,
+    },
     types::{ByteStream, SdkError},
     Client, // Config,
     Error as S3Error,
 };
 use aws_smithy_http::{body::SdkBody, byte_stream::Error as ByteStreamError, endpoint::Endpoint};
-use futures::stream::{IntoAsyncRead, MapErr, TryStreamExt};
+use futures::{
+    io::copy,
+    stream::{IntoAsyncRead, MapErr, TryStreamExt},
+};
 use kepler_lib::{
     libipld::cid::{
         multibase::{encode, Base},
-        multihash::Multihash,
+        multihash::{
+            Blake3Hasher, Code, Error as MultihashError, Hasher, Multihash, MultihashDigest,
+        },
         Cid,
     },
     resource::OrbitId,
@@ -21,10 +27,13 @@ use rocket::{async_trait, http::hyper::Uri};
 use serde::{Deserialize, Serialize};
 use serde_with::{serde_as, DisplayFromStr};
 use std::{io::Error as IoError, path::PathBuf, str::FromStr};
+use tempfile::tempfile;
+use tokio::fs::File;
+use tokio_util::compat::{Compat, TokioAsyncReadCompatExt};
 
 use crate::{
     orbit::ProviderUtils,
-    storage::{ImmutableStore, StorageConfig},
+    storage::{utils::HashBuffer, ImmutableStore, StorageConfig},
 };
 
 // TODO we could use the same struct for both the block store and the data
@@ -181,48 +190,97 @@ impl S3BlockStore {
             orbit: orbit.to_string(),
         }
     }
+
+    fn key(&self, id: &Multihash) -> String {
+        format!("{}/{}", self.orbit, encode(Base::Base64Url, &id.to_bytes()))
+    }
 }
 
 pub fn convert(e: ByteStreamError) -> IoError {
     e.into()
 }
 
+#[derive(thiserror::Error, Debug)]
+pub enum S3StoreError {
+    #[error(transparent)]
+    S3(#[from] S3Error),
+    #[error(transparent)]
+    Io(#[from] IoError),
+    #[error(transparent)]
+    Bytestream(#[from] ByteStreamError),
+    #[error(transparent)]
+    Multihash(#[from] MultihashError),
+}
+
 #[async_trait]
 impl ImmutableStore for S3BlockStore {
-    type Error = S3Error;
+    type Error = S3StoreError;
     type Readable = IntoAsyncRead<MapErr<ByteStream, fn(ByteStreamError) -> IoError>>;
     async fn contains(&self, id: &Multihash) -> Result<bool, Self::Error> {
-        todo!()
+        match self
+            .client
+            .head_object()
+            .bucket(&self.bucket)
+            .key(self.key(id))
+            .send()
+            .await
+        {
+            Ok(_) => Ok(true),
+            Err(SdkError::ServiceError {
+                err:
+                    HeadObjectError {
+                        kind: HeadObjectErrorKind::NotFound(_),
+                        ..
+                    },
+                ..
+            }) => Ok(false),
+            Err(e) => Err(S3Error::from(e).into()),
+        }
     }
     async fn write(
         &self,
         data: impl futures::io::AsyncRead + Send,
     ) -> Result<Multihash, Self::Error> {
-        // write to a dummy ID (in fs or in s3)
-        // get the hash
-        // write or copy to correct ID (hash)
-        todo!();
-        // write into tmp then rename, to name after the hash
-        // need to stream data through a hasher into the file and return hash
-        // match File::open(path.join(cid.to_string())),await {
-        //     Ok(f) => copy(data, file).await
-        //     Err(e) if error.kind() == IoErrorKind::NotFound => Ok(None),
-        //     Err(e) => Err(e),
-        // }
+        // TODO find a way to do this without filesystem access
+        let mut hb =
+            HashBuffer::<Blake3Hasher<32>, Compat<File>>::new(File::from(tempfile()?).compat());
+
+        copy(data, &mut hb).await?;
+
+        let (mut hasher, file) = hb.into_inner();
+        let multihash = Code::Blake3_256.wrap(hasher.finalize())?;
+
+        self.client
+            .put_object()
+            .bucket(&self.bucket)
+            .key(self.key(&multihash))
+            // TODO deprecated but also doesnt require a path
+            .body(ByteStream::from_file(file.into_inner()).await?)
+            .send()
+            .await
+            .map_err(S3Error::from)?;
+        Ok(multihash)
     }
     async fn remove(&self, id: &Multihash) -> Result<Option<()>, Self::Error> {
-        todo!()
+        match self
+            .client
+            .delete_object()
+            .bucket(&self.bucket)
+            .key(self.key(id))
+            .send()
+            .await
+        {
+            Ok(_) => Ok(Some(())),
+            // TODO does this distinguish between object missing and object present?
+            Err(e) => Err(S3Error::from(e).into()),
+        }
     }
     async fn read(&self, id: &Multihash) -> Result<Option<Self::Readable>, Self::Error> {
         let res = self
             .client
             .get_object()
-            .bucket(self.bucket.clone())
-            .key(format!(
-                "{}/{}",
-                self.orbit,
-                encode(Base::Base64Url, &id.to_bytes())
-            ))
+            .bucket(&self.bucket)
+            .key(self.key(id))
             .send()
             .await;
         match res {
@@ -239,7 +297,7 @@ impl ImmutableStore for S3BlockStore {
                     },
                 ..
             }) => Ok(None),
-            Err(e) => Err(e.into()),
+            Err(e) => Err(S3Error::from(e).into()),
         }
     }
 }
diff --git a/src/storage/utils.rs b/src/storage/utils.rs
index 410dc384..582d5628 100644
--- a/src/storage/utils.rs
+++ b/src/storage/utils.rs
@@ -1,6 +1,61 @@
 use aws_types::sdk_config::SdkConfig;
-use futures::executor::block_on;
+use core::pin::Pin;
+use futures::{
+    executor::block_on,
+    io::{AllowStdIo, AsyncWrite, Error},
+    task::{Context, Poll},
+};
+use libipld::cid::multihash::Hasher;
+use pin_project::pin_project;
 
 pub fn aws_config() -> SdkConfig {
     block_on(async { aws_config::from_env().load().await })
 }
+
+#[pin_project]
+#[derive(Debug, Clone)]
+pub struct HashBuffer<H, B> {
+    #[pin]
+    buffer: B,
+    hasher: H,
+}
+
+impl<H, B> HashBuffer<H, B> {
+    pub fn into_inner(self) -> (H, B) {
+        (self.hasher, self.buffer)
+    }
+}
+
+impl<H, B> HashBuffer<H, B>
+where
+    H: Default,
+{
+    pub fn new(buffer: B) -> Self {
+        Self {
+            buffer,
+            hasher: H::default(),
+        }
+    }
+}
+
+impl<H, B> AsyncWrite for HashBuffer<H, B>
+where
+    H: Hasher,
+    B: AsyncWrite,
+{
+    fn poll_write(
+        self: Pin<&mut Self>,
+        cx: &mut Context<'_>,
+        buf: &[u8],
+    ) -> Poll<Result<usize, Error>> {
+        let p = self.project();
+        p.hasher.update(buf);
+        p.buffer.poll_write(cx, buf)
+    }
+    fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), Error>> {
+        self.project().buffer.poll_flush(cx)
+    }
+    fn poll_close(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), Error>> {
+        self.project().buffer.poll_close(cx)
+    }
+}

From 11b6218d56149addcea32652a3ceb591c173147e Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Thu, 3 Nov 2022 12:46:46 +0100
Subject: [PATCH 41/73] install protoc in workflows

---
 .github/workflows/rust.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index 004cc533..42247102 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -23,6 +23,9 @@ jobs:
       run: |
         rustup target add wasm32-unknown-unknown
 
+    - name: Install Protoc
+      uses: arduino/setup-protoc@v1
+
     - name: Build
       working-directory: ${{ matrix.pkg }}
       run: cargo build

From 478d2cac3faa3668305e2109a9f65791dcd44544 Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Thu, 3 Nov 2022 12:52:13 +0100
Subject: [PATCH 42/73] removing unused code and busted tests

---
 src/kv/behaviour.rs     | 118 ------------------
 src/kv/mod.rs           |   1 -
 src/relay.rs            | 188 ++++++++++++++--------------
 src/storage/dynamodb.rs |  61 ----------
 src/storage/mod.rs      |   2 -
 src/transport.rs        | 264 ++++++++++++++++++++--------------------
 6 files changed, 226 insertions(+), 408 deletions(-)
 delete mode 100644 src/kv/behaviour.rs
 delete mode 100644 src/storage/dynamodb.rs

diff --git a/src/kv/behaviour.rs b/src/kv/behaviour.rs
deleted file mode 100644
index 111b5182..00000000
--- a/src/kv/behaviour.rs
+++ /dev/null
@@ -1,118 +0,0 @@
-use std::{
-    sync::{
-        mpsc::{Receiver, SyncSender},
-        Arc,
-    },
-    task::{Context, Poll},
-};
-
-use libp2p::{
-    core::{connection::ConnectionId, ConnectedPoint, Multiaddr, PeerId},
-    swarm::{
-        dummy::ConnectionHandler as DummyConnectionHandler, IntoConnectionHandler,
-        NetworkBehaviour, NetworkBehaviourAction, PollParameters,
-    },
-};
-use void::Void;
-
-use crate::orbit::AbortOnDrop;
-
-use super::Store;
-
-#[derive(Clone, Debug)]
-pub struct Behaviour {
-    sender: SyncSender<Event>,
-}
-
-impl Behaviour {
-    pub fn new(sender: SyncSender<Event>) -> Self {
-        Self { sender }
-    }
-}
-
-impl NetworkBehaviour for Behaviour {
-    type ConnectionHandler = DummyConnectionHandler;
-
-    type OutEvent = ();
-
-    fn new_handler(&mut self) -> Self::ConnectionHandler {
-        DummyConnectionHandler
-    }
-
-    fn inject_event(&mut self, _peer_id: PeerId, _connection: ConnectionId, _event: Void) {}
-
-    fn poll(
-        &mut self,
-        _cx: &mut Context<'_>,
-        _params: &mut impl PollParameters,
-    ) -> Poll<NetworkBehaviourAction<(), Self::ConnectionHandler, Void>> {
-        Poll::Pending
-    }
-
-    fn inject_connection_established(
-        &mut self,
-        peer_id: &PeerId,
-        _connection: &ConnectionId,
-        _endpoint: &ConnectedPoint,
-        _failed_addresses: Option<&Vec<Multiaddr>>,
-        _other_established: usize,
-    ) {
-        if let Err(_e) = self.sender.send(Event::ConnectionEstablished(*peer_id)) {
-            tracing::error!("Behaviour process has shutdown.")
-        }
-    }
-
-    fn inject_connection_closed(
-        &mut self,
-        peer_id: &PeerId,
-        _connection: &ConnectionId,
-        _endpoint: &ConnectedPoint,
-        _handler: <Self::ConnectionHandler as IntoConnectionHandler>::Handler,
-        _remaining_established: usize,
-    ) {
-        if let Err(_e) = self.sender.send(Event::ConnectionTerminated(*peer_id)) {
-            tracing::error!("Behaviour process has shutdown.")
-        }
-    }
-}
-
-#[derive(Clone, Debug)]
-pub struct BehaviourProcess(Arc<AbortOnDrop<()>>);
-
-impl BehaviourProcess {
-    pub fn new<B>(store: Store<B>, mut receiver: Receiver<Event>) -> Self {
-        Self(Arc::new(AbortOnDrop::new(tokio::spawn(async move {
-            while let Ok(Ok((event, returned_receiver))) =
-                tokio::task::spawn_blocking(move || receiver.recv().map(|ev| (ev, receiver))).await
-            {
-                receiver = returned_receiver;
-                match event {
-                    Event::ConnectionEstablished(peer_id) => {
-                        todo!("synchronise state with peer");
-                        // if let Err(e) = store.ipfs.pubsub_add_peer(peer_id).await {
-                        //     tracing::error!("failed to add new peer to allowed pubsub peers: {}", e)
-                        // }
-                        // if let Err(e) = store.request_heads().await {
-                        //     tracing::error!("failed to request heads from peers: {}", e)
-                        // }
-                    }
-                    Event::ConnectionTerminated(peer_id) => {
-                        todo!("remove peer from set/deny connection");
-                        // if let Err(e) = store.ipfs.pubsub_remove_peer(peer_id).await {
-                        //     tracing::error!(
-                        //         "failed to remove disconnected peer from allowed pubsub peers: {}",
-                        //         e
-                        //     )
-                        // }
-                    }
-                }
-            }
-        }))))
-    }
-}
-
-#[derive(Clone, Debug)]
-pub enum Event {
-    ConnectionEstablished(PeerId),
-    ConnectionTerminated(PeerId),
-}
diff --git a/src/kv/mod.rs b/src/kv/mod.rs
index 9e72f630..26af4965 100644
--- a/src/kv/mod.rs
+++ b/src/kv/mod.rs
@@ -6,7 +6,6 @@ use libp2p::core::PeerId;
 use rocket::futures::{Stream, StreamExt};
 use serde::{Deserialize, Serialize};
 
-pub mod behaviour;
 mod entries;
 mod store;
 
diff --git a/src/relay.rs b/src/relay.rs
index 9de2967c..c3ead904 100644
--- a/src/relay.rs
+++ b/src/relay.rs
@@ -67,98 +67,98 @@ impl RelayNode {
 
 #[cfg(test)]
 pub mod test {
-    use super::*;
-    use ipfs::{
-        p2p::transport::TransportBuilder, IpfsOptions, MultiaddrWithoutPeerId, Types,
-        UninitializedIpfs,
-    };
-    use libp2p::core::multiaddr::{multiaddr, Protocol};
-    use std::{
-        convert::TryFrom,
-        sync::atomic::{AtomicU16, Ordering},
-        time::Duration,
-    };
-
-    static PORT: AtomicU16 = AtomicU16::new(10000);
-
-    pub async fn test_relay() -> Result<RelayNode> {
-        RelayNode::new(
-            PORT.fetch_add(1, Ordering::SeqCst),
-            Keypair::generate_ed25519(),
-        )
-        .await
-    }
-
-    #[tokio::test(flavor = "multi_thread")]
-    async fn relay() -> Result<()> {
-        let relay = test_relay().await?;
-
-        let dir = tempdir::TempDir::new("relay")?;
-        let alice_path = dir.path().join("alice");
-        std::fs::create_dir(&alice_path)?;
-        let bob_path = dir.path().join("bob");
-        std::fs::create_dir(&bob_path)?;
-
-        // Isn't actually in-memory, just provides useful defaults.
-        let mut alice_opts = IpfsOptions::inmemory_with_generated_keys();
-        alice_opts.ipfs_path = alice_path;
-        alice_opts.listening_addrs = vec![multiaddr!(P2pCircuit)];
-        let mut bob_opts = IpfsOptions::inmemory_with_generated_keys();
-        bob_opts.ipfs_path = bob_path;
-
-        let alice_peer_id = alice_opts.keypair.public().to_peer_id();
-        let bob_peer_id = bob_opts.keypair.public().to_peer_id();
-
-        let (alice_builder, alice_relay) = TransportBuilder::new(alice_opts.keypair.clone())?
-            .or(MemoryTransport::default())
-            .relay();
-        let alice_transport = alice_builder
-            .map_auth()
-            .map(crate::transport::auth_mapper([bob_peer_id.clone()]))
-            .build();
-        let (alice, task) =
-            UninitializedIpfs::<Types>::new(alice_opts, alice_transport, Some(alice_relay))
-                .start()
-                .await?;
-        tokio::spawn(task);
-
-        let (bob_builder, bob_relay) = TransportBuilder::new(bob_opts.keypair.clone())?
-            .or(MemoryTransport::default())
-            .relay();
-        let (bob, task) =
-            UninitializedIpfs::<Types>::new(bob_opts, bob_builder.build(), Some(bob_relay))
-                .start()
-                .await?;
-        tokio::spawn(task);
-
-        alice
-            .connect(MultiaddrWithoutPeerId::try_from(relay.internal())?.with(relay.id.clone()))
-            .await
-            .expect("alice failed to connect to relay");
-
-        bob.connect(
-            MultiaddrWithoutPeerId::try_from(
-                relay
-                    .external()
-                    .with(Protocol::P2p(relay.id.clone().into()))
-                    .with(Protocol::P2pCircuit),
-            )?
-            .with(alice_peer_id.clone()),
-        )
-        .await
-        .expect("bob failed to connect to alice");
-
-        tokio::time::sleep(Duration::from_millis(1000)).await;
-
-        let alice_peers = alice.peers().await?;
-        let bob_peers = bob.peers().await?;
-        assert!(alice_peers
-            .iter()
-            .any(|conn| conn.addr.peer_id == bob_peer_id));
-        assert!(bob_peers
-            .iter()
-            .any(|conn| conn.addr.peer_id == alice_peer_id));
-
-        Ok(())
-    }
+    // use super::*;
+    // use ipfs::{
+    //     p2p::transport::TransportBuilder, IpfsOptions, MultiaddrWithoutPeerId, Types,
+    //     UninitializedIpfs,
+    // };
+    // use libp2p::core::multiaddr::{multiaddr, Protocol};
+    // use std::{
+    //     convert::TryFrom,
+    //     sync::atomic::{AtomicU16, Ordering},
+    //     time::Duration,
+    // };
+
+    // static PORT: AtomicU16 = AtomicU16::new(10000);
+
+    // pub async fn test_relay() -> Result<RelayNode> {
+    //     RelayNode::new(
+    //         PORT.fetch_add(1, Ordering::SeqCst),
+    //         Keypair::generate_ed25519(),
+    //     )
+    //     .await
+    // }
+
+    // #[tokio::test(flavor = "multi_thread")]
+    // async fn relay() -> Result<()> {
+    //     let relay = test_relay().await?;
+
+    //     let dir = tempfile::TempDir::new("relay")?;
+    //     let alice_path = dir.path().join("alice");
+    //     std::fs::create_dir(&alice_path)?;
+    //     let bob_path = dir.path().join("bob");
+    //     std::fs::create_dir(&bob_path)?;
+
+    //     // Isn't actually in-memory, just provides useful defaults.
+    //     let mut alice_opts = IpfsOptions::inmemory_with_generated_keys();
+    //     alice_opts.ipfs_path = alice_path;
+    //     alice_opts.listening_addrs = vec![multiaddr!(P2pCircuit)];
+    //     let mut bob_opts = IpfsOptions::inmemory_with_generated_keys();
+    //     bob_opts.ipfs_path = bob_path;
+
+    //     let alice_peer_id = alice_opts.keypair.public().to_peer_id();
+    //     let bob_peer_id = bob_opts.keypair.public().to_peer_id();
+
+    //     let (alice_builder, alice_relay) = TransportBuilder::new(alice_opts.keypair.clone())?
+    //         .or(MemoryTransport::default())
+    //         .relay();
+    //     let alice_transport = alice_builder
+    //         .map_auth()
+    //         .map(crate::transport::auth_mapper([bob_peer_id.clone()]))
+    //         .build();
+    //     let (alice, task) =
+    //         UninitializedIpfs::<Types>::new(alice_opts, alice_transport, Some(alice_relay))
+    //             .start()
+    //             .await?;
+    //     tokio::spawn(task);
+
+    //     let (bob_builder, bob_relay) = TransportBuilder::new(bob_opts.keypair.clone())?
+    //         .or(MemoryTransport::default())
+    //         .relay();
+    //     let (bob, task) =
+    //         UninitializedIpfs::<Types>::new(bob_opts, bob_builder.build(), Some(bob_relay))
+    //             .start()
+    //             .await?;
+    //     tokio::spawn(task);
+
+    //     alice
+    //         .connect(MultiaddrWithoutPeerId::try_from(relay.internal())?.with(relay.id.clone()))
+    //         .await
+    //         .expect("alice failed to connect to relay");
+
+    //     bob.connect(
+    //         MultiaddrWithoutPeerId::try_from(
+    //             relay
+    //                 .external()
+    //                 .with(Protocol::P2p(relay.id.clone().into()))
+    //                 .with(Protocol::P2pCircuit),
+    //         )?
+    //         .with(alice_peer_id.clone()),
+    //     )
+    //     .await
+    //     .expect("bob failed to connect to alice");
+
+    //     tokio::time::sleep(Duration::from_millis(1000)).await;
+
+    //     let alice_peers = alice.peers().await?;
+    //     let bob_peers = bob.peers().await?;
+    //     assert!(alice_peers
+    //         .iter()
+    //         .any(|conn| conn.addr.peer_id == bob_peer_id));
+    //     assert!(bob_peers
+    //         .iter()
+    //         .any(|conn| conn.addr.peer_id == alice_peer_id));
+
+    //     Ok(())
+    // }
 }
diff --git a/src/storage/dynamodb.rs b/src/storage/dynamodb.rs
deleted file mode 100644
index 2f256e21..00000000
--- a/src/storage/dynamodb.rs
+++ /dev/null
@@ -1,61 +0,0 @@
-use anyhow::{Context, Error};
-use aws_sdk_dynamodb::{
-    error::{GetItemError, GetItemErrorKind},
-    model::{AttributeValue, ReturnValue},
-    output::ScanOutput,
-    types::SdkError,
-    Client,
-};
-use aws_smithy_http::endpoint::Endpoint;
-use futures::{
-    lock::Mutex,
-    stream::{self, StreamExt, TryStreamExt},
-};
-use kepler_lib::libipld::cid::{multibase::Base, Cid};
-use rocket::async_trait;
-use std::{collections::BTreeSet, str::FromStr};
-
-use crate::config;
-
-const CID_ATTRIBUTE: &str = "Cid";
-const ROOT_ATTRIBUTE: &str = "Root";
-const PARENTS_ATTRIBUTE: &str = "Parents";
-
-#[derive(Debug)]
-pub struct DynamoPinStore {
-    // TODO no need for Mutex???
-    client: Mutex<Client>,
-    table: String,
-    orbit: Cid,
-}
-
-impl DynamoPinStore {
-    pub fn new(config: config::DynamoStorage, orbit: Cid) -> Self {
-        let general_config = super::utils::aws_config();
-        let sdk_config = aws_sdk_dynamodb::config::Builder::from(&general_config);
-        let sdk_config = match config.endpoint {
-            Some(e) => sdk_config.endpoint_resolver(Endpoint::immutable(e)),
-            None => sdk_config,
-        };
-        let sdk_config = sdk_config.build();
-        let client = Mutex::new(Client::from_conf(sdk_config));
-        Self {
-            client,
-            table: config.table,
-            orbit,
-        }
-    }
-
-    pub async fn healthcheck(&self) -> Result<(), Error> {
-        // TODO ideally that would be in the builder
-        self.client
-            .lock()
-            .await
-            .describe_table()
-            .table_name(self.table.clone().clone())
-            .send()
-            .await
-            .context(anyhow!("Failed healthchec for table `{}`", self.table))?;
-        Ok(())
-    }
-}
diff --git a/src/storage/mod.rs b/src/storage/mod.rs
index e32465d7..ce57619c 100644
--- a/src/storage/mod.rs
+++ b/src/storage/mod.rs
@@ -2,9 +2,7 @@ use anyhow::{Error, Result};
 use kepler_lib::libipld::cid::{multihash::Multihash, Cid};
 use kepler_lib::resource::OrbitId;
 use std::{collections::HashMap, error::Error as StdError};
-use tracing::instrument;
 
-mod dynamodb;
 pub mod either;
 pub mod file_system;
 mod indexes;
diff --git a/src/transport.rs b/src/transport.rs
index 005839b3..28e3e299 100644
--- a/src/transport.rs
+++ b/src/transport.rs
@@ -41,136 +41,136 @@ pub enum AuthError {
 
 #[cfg(test)]
 mod test {
-    use std::convert::TryFrom;
-
-    use crate::{config, ipfs::create_ipfs, relay::test::test_relay};
-    use ipfs::multiaddr::Protocol;
-    use kepler_lib::libipld::Cid;
-    use libp2p::identity::Keypair;
-    use std::str::FromStr;
-    use tempdir::TempDir;
-
-    #[tokio::test]
-    async fn authorised() -> anyhow::Result<()> {
-        let id =
-            Cid::from_str("bafkreieq5jui4j25lacwomsqgjeswwl3y5zcdrresptwgmfylxo2depppq").unwrap();
-        let temp_dir = TempDir::new(&id.to_string())?;
-        let relay = test_relay().await?;
-
-        let alice_keypair = Keypair::generate_ed25519();
-        let alice_peer_id = alice_keypair.public().to_peer_id();
-        let alice_path = temp_dir.path().join("alice");
-        let bob_keypair = Keypair::generate_ed25519();
-        let bob_peer_id = bob_keypair.public().to_peer_id();
-        let bob_path = temp_dir.path().join("bob");
-        let alice_config = config::Config {
-            storage: config::Storage {
-                blocks: config::BlockStorage::Local(config::LocalBlockStorage {
-                    path: alice_path.clone(),
-                }),
-                indexes: config::IndexStorage::Local(config::LocalIndexStorage {
-                    path: alice_path.clone(),
-                }),
-            },
-            ..Default::default()
-        };
-        let bob_config = config::Config {
-            storage: config::Storage {
-                blocks: config::BlockStorage::Local(config::LocalBlockStorage {
-                    path: bob_path.clone(),
-                }),
-                indexes: config::IndexStorage::Local(config::LocalIndexStorage {
-                    path: bob_path.clone(),
-                }),
-            },
-            ..Default::default()
-        };
-
-        let (alice, ipfs_task, _alice_behaviour_process) =
-            create_ipfs(id, &alice_config, alice_keypair, vec![bob_peer_id]).await?;
-        tokio::task::spawn(ipfs_task);
-        let (bob, ipfs_task, _bob_behaviour_process) =
-            create_ipfs(id, &bob_config, bob_keypair, vec![]).await?;
-        tokio::task::spawn(ipfs_task);
-
-        alice
-            .connect(MultiaddrWithoutPeerId::try_from(relay.internal())?.with(relay.id))
-            .await?;
-
-        bob.connect(
-            MultiaddrWithoutPeerId::try_from(
-                relay
-                    .external()
-                    .with(Protocol::P2p(relay.id.into()))
-                    .with(Protocol::P2pCircuit),
-            )?
-            .with(alice_peer_id),
-        )
-        .await
-        .expect("authorised peer (bob) could not connect to alice");
-
-        Ok(())
-    }
-
-    #[tokio::test]
-    async fn unauthorised() -> anyhow::Result<()> {
-        let id =
-            Cid::from_str("bafkreieq5jui4j25lacwomsqgjeswwl3y5zcdrresptwgmfylxo2depppq").unwrap();
-        let temp_dir = TempDir::new(&id.to_string())?;
-        let relay = test_relay().await?;
-
-        let alice_keypair = Keypair::generate_ed25519();
-        let alice_peer_id = alice_keypair.public().to_peer_id();
-        let alice_path = temp_dir.path().join("alice");
-        let bob_keypair = Keypair::generate_ed25519();
-        let _bob_peer_id = bob_keypair.public().to_peer_id();
-        let bob_path = temp_dir.path().join("bob");
-        let alice_config = config::Config {
-            storage: config::Storage {
-                blocks: config::BlockStorage::Local(config::LocalBlockStorage {
-                    path: alice_path.clone(),
-                }),
-                indexes: config::IndexStorage::Local(config::LocalIndexStorage {
-                    path: alice_path.clone(),
-                }),
-            },
-            ..Default::default()
-        };
-        let bob_config = config::Config {
-            storage: config::Storage {
-                blocks: config::BlockStorage::Local(config::LocalBlockStorage {
-                    path: bob_path.clone(),
-                }),
-                indexes: config::IndexStorage::Local(config::LocalIndexStorage {
-                    path: bob_path.clone(),
-                }),
-            },
-            ..Default::default()
-        };
-
-        let (alice, ipfs_task, _alice_behaviour_process) =
-            create_ipfs(id, &alice_config, alice_keypair, vec![]).await?;
-        tokio::task::spawn(ipfs_task);
-        let (bob, ipfs_task, _bob_behaviour_process) =
-            create_ipfs(id, &bob_config, bob_keypair, vec![]).await?;
-        tokio::task::spawn(ipfs_task);
-
-        alice
-            .connect(MultiaddrWithoutPeerId::try_from(relay.internal())?.with(relay.id))
-            .await?;
-
-        bob.connect(
-            MultiaddrWithoutPeerId::try_from(
-                relay
-                    .external()
-                    .with(Protocol::P2p(relay.id.into()))
-                    .with(Protocol::P2pCircuit),
-            )?
-            .with(alice_peer_id),
-        )
-        .await
-        .expect_err("unauthorised peer (bob) connected to alice");
-
-        Ok(())
-    }
+    // use std::convert::TryFrom;
+
+    // use crate::{config, ipfs::create_ipfs, relay::test::test_relay};
+    // use ipfs::multiaddr::Protocol;
+    // use kepler_lib::libipld::Cid;
+    // use libp2p::identity::Keypair;
+    // use std::str::FromStr;
+    // use tempfile::TempDir;
+
+    // #[tokio::test]
+    // async fn authorised() -> anyhow::Result<()> {
+    //     let id =
+    //         Cid::from_str("bafkreieq5jui4j25lacwomsqgjeswwl3y5zcdrresptwgmfylxo2depppq").unwrap();
+    //     let temp_dir = tempfile::new(&id.to_string())?;
+    //     let relay = test_relay().await?;
+
+    //     let alice_keypair = Keypair::generate_ed25519();
+    //     let alice_peer_id = alice_keypair.public().to_peer_id();
+    //     let alice_path = temp_dir.path().join("alice");
+    //     let bob_keypair = Keypair::generate_ed25519();
+    //     let bob_peer_id = bob_keypair.public().to_peer_id();
+    //     let bob_path = temp_dir.path().join("bob");
+    //     let alice_config = config::Config {
+    //         storage: config::Storage {
+    //             blocks: config::BlockStorage::Local(config::LocalBlockStorage {
+    //                 path: alice_path.clone(),
+    //             }),
+    //             indexes: config::IndexStorage::Local(config::LocalIndexStorage {
+    //                 path: alice_path.clone(),
+    //             }),
+    //         },
+    //         ..Default::default()
+    //     };
+    //     let bob_config = config::Config {
+    //         storage: config::Storage {
+    //             blocks: config::BlockStorage::Local(config::LocalBlockStorage {
+    //                 path: bob_path.clone(),
+    //             }),
+    //             indexes: config::IndexStorage::Local(config::LocalIndexStorage {
+    //                 path: bob_path.clone(),
+    //             }),
+    //         },
+    //         ..Default::default()
+    //     };
+
+    //     let (alice, ipfs_task, _alice_behaviour_process) =
+    //         create_ipfs(id, &alice_config, alice_keypair, vec![bob_peer_id]).await?;
+    //     tokio::task::spawn(ipfs_task);
+    //     let (bob, ipfs_task, _bob_behaviour_process) =
+    //         create_ipfs(id, &bob_config, bob_keypair, vec![]).await?;
+    //     tokio::task::spawn(ipfs_task);
+
+    //     alice
+    //         .connect(MultiaddrWithoutPeerId::try_from(relay.internal())?.with(relay.id))
+    //         .await?;
+
+    //     bob.connect(
+    //         MultiaddrWithoutPeerId::try_from(
+    //             relay
+    //                 .external()
+    //                 .with(Protocol::P2p(relay.id.into()))
+    //                 .with(Protocol::P2pCircuit),
+    //         )?
+    //         .with(alice_peer_id),
+    //     )
+    //     .await
+    //     .expect("authorised peer (bob) could not connect to alice");
+
+    //     Ok(())
+    // }
+
+    // #[tokio::test]
+    // async fn unauthorised() -> anyhow::Result<()> {
+    //     let id =
+    //         Cid::from_str("bafkreieq5jui4j25lacwomsqgjeswwl3y5zcdrresptwgmfylxo2depppq").unwrap();
+    //     let temp_dir = tempfile::new(&id.to_string())?;
+    //     let relay = test_relay().await?;
+
+    //     let alice_keypair = Keypair::generate_ed25519();
+    //     let alice_peer_id = alice_keypair.public().to_peer_id();
+    //     let alice_path = temp_dir.path().join("alice");
+    //     let bob_keypair = Keypair::generate_ed25519();
+    //     let _bob_peer_id = bob_keypair.public().to_peer_id();
+    //     let bob_path = temp_dir.path().join("bob");
+    //     let alice_config = config::Config {
+    //         storage: config::Storage {
+    //             blocks: config::BlockStorage::Local(config::LocalBlockStorage {
+    //                 path: alice_path.clone(),
+    //             }),
+    //             indexes: config::IndexStorage::Local(config::LocalIndexStorage {
+    //                 path: alice_path.clone(),
+    //             }),
+    //         },
+    //         ..Default::default()
+    //     };
+    //     let bob_config = config::Config {
+    //         storage: config::Storage {
+    //             blocks: config::BlockStorage::Local(config::LocalBlockStorage {
+    //                 path: bob_path.clone(),
+    //             }),
+    //             indexes: config::IndexStorage::Local(config::LocalIndexStorage {
+    //                 path: bob_path.clone(),
+    //             }),
+    //         },
+    //         ..Default::default()
+    //     };
+
+    //     let (alice, ipfs_task, _alice_behaviour_process) =
+    //         create_ipfs(id, &alice_config, alice_keypair, vec![]).await?;
+    //     tokio::task::spawn(ipfs_task);
+    //     let (bob, ipfs_task, _bob_behaviour_process) =
+    //         create_ipfs(id, &bob_config, bob_keypair, vec![]).await?;
+    //     tokio::task::spawn(ipfs_task);
+
+    //     alice
+    //         .connect(MultiaddrWithoutPeerId::try_from(relay.internal())?.with(relay.id))
+    //         .await?;
+
+    //     bob.connect(
+    //         MultiaddrWithoutPeerId::try_from(
+    //             relay
+    //                 .external()
+    //                 .with(Protocol::P2p(relay.id.into()))
+    //                 .with(Protocol::P2pCircuit),
+    //         )?
+    //         .with(alice_peer_id),
+    //     )
+    //     .await
+    //     .expect_err("unauthorised peer (bob) connected to alice");
+
+    //     Ok(())
+    // }
 }

From 867251b6fd4740cc035a403bd7f934887ee94b60 Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Thu, 3 Nov 2022 13:14:42 +0100
Subject: [PATCH 43/73] cippy warnings, delete lots of unused code

---
 src/capabilities/mod.rs   |  52 +--------
 src/capabilities/store.rs | 139 ----------------------
 src/kv/mod.rs             | 234 +-------------------------------------
 src/kv/store.rs           |  10 +-
 src/lib.rs                |   2 +-
 src/orbit.rs              |  20 ++--
 src/routes/mod.rs         |   8 +-
 src/storage/s3.rs         |  35 +-----
 src/storage/utils.rs      |   2 +-
 9 files changed, 30 insertions(+), 472 deletions(-)

diff --git a/src/capabilities/mod.rs b/src/capabilities/mod.rs
index b091e2fc..f3636b3f 100644
--- a/src/capabilities/mod.rs
+++ b/src/capabilities/mod.rs
@@ -2,9 +2,7 @@ pub mod store;
 
 use anyhow::Result;
 use kepler_lib::libipld::Cid;
-use libp2p::core::PeerId;
-use rocket::futures::{Stream, StreamExt};
-use store::{CapsMessage, Store};
+use store::Store;
 
 #[rocket::async_trait]
 pub trait Invoke<T> {
@@ -31,51 +29,3 @@ impl<B> Service<B> {
         Ok(Service::new(store))
     }
 }
-
-async fn caps_task<B>(
-    events: impl Stream<Item = Result<(PeerId, CapsMessage)>> + Send,
-    store: Store<B>,
-    peer_id: PeerId,
-) {
-    debug!("starting caps task");
-    events
-        .for_each_concurrent(None, |ev| async {
-            match ev {
-                Ok((p, ev)) if p == peer_id => {
-                    debug!("{} filtered out this event from self: {:?}", p, ev)
-                }
-                Ok((_, CapsMessage::Invocation(cid))) => {
-                    debug!("recieved invocation");
-                    // if let Err(e) = store.try_merge_invocations([cid].into_iter()).await {
-                    //     debug!("failed to apply recieved invocation {}", e);
-                    // }
-                }
-                Ok((_, CapsMessage::StateReq)) => {
-                    // if let Err(e) = store.broadcast_heads().await {
-                    //     debug!(
-                    //         "failed to broadcast updates in response to state request {}",
-                    //         e
-                    //     );
-                    // }
-                }
-                Ok((
-                    _,
-                    CapsMessage::Heads {
-                        updates,
-                        invocations,
-                    },
-                )) => {
-                    // if let Err(e) = store
-                    //     .try_merge_heads(updates.into_iter(), invocations.into_iter())
-                    //     .await
-                    // {
-                    //     debug!("failed to merge heads {}", e);
-                    // }
-                }
-                Err(e) => {
-                    debug!("cap service task error {}", e);
-                }
-            }
-        })
-        .await;
-}
diff --git a/src/capabilities/store.rs b/src/capabilities/store.rs
index 3e9ace7a..ac8a48f6 100644
--- a/src/capabilities/store.rs
+++ b/src/capabilities/store.rs
@@ -6,8 +6,6 @@ use crate::{
     Block,
 };
 use anyhow::Result;
-use async_recursion::async_recursion;
-use futures::stream::{self, TryStreamExt};
 use kepler_lib::libipld::{cbor::DagCborCodec, multihash::Code, Cid, DagCbor};
 use kepler_lib::{
     authorization::{KeplerDelegation, KeplerInvocation, KeplerRevocation},
@@ -192,31 +190,6 @@ where
         // self.broadcast_heads().await?;
         Ok(cid)
     }
-
-    async fn get_invocation(&self, i: &Cid) -> Result<Invocations> {
-        let update: InvocationsBlock = self
-            .get_obj(i)
-            .await?
-            .ok_or_else(|| anyhow!("Incomplete Event"))?
-            .base;
-
-        Ok(Invocations {
-            prev: update.prev,
-            invoke: try_join_all(update.invoke.iter().map(|c| async {
-                let link: WithBlock<LinkedUpdate> = self
-                    .get_obj(c)
-                    .await?
-                    .ok_or_else(|| anyhow!("Incomplete Event"))?;
-                let inv: WithBlock<Invocation> = self
-                    .get_obj(&link.base.update)
-                    .await?
-                    .ok_or_else(|| anyhow!("Incomplete Event"))?;
-                Result::<(WithBlock<LinkedUpdate>, WithBlock<Invocation>)>::Ok((link, inv))
-            }))
-            .await?,
-        })
-    }
-
     pub(crate) async fn apply_invocations(&self, event: Invocations) -> Result<Cid> {
         try_join_all(
             event
@@ -282,41 +255,6 @@ where
         })
     }
 
-    async fn get_event(&self, e: &Cid) -> Result<Option<Event>> {
-        let update: EventBlock = match self.get_obj(e).await? {
-            Some(e) => e.base,
-            None => return Ok(None),
-        };
-
-        Ok(Some(Event {
-            prev: update.prev,
-            delegate: try_join_all(update.delegate.iter().map(|c| async {
-                let link: WithBlock<LinkedUpdate> = self
-                    .get_obj(c)
-                    .await?
-                    .ok_or_else(|| anyhow!("Incomplete Event"))?;
-                let del: WithBlock<Delegation> = self
-                    .get_obj(&link.base.update)
-                    .await?
-                    .ok_or_else(|| anyhow!("Incomplete Event"))?;
-                Result::<(WithBlock<LinkedUpdate>, WithBlock<Delegation>)>::Ok((link, del))
-            }))
-            .await?,
-            revoke: try_join_all(update.revoke.iter().map(|c| async {
-                let link: WithBlock<LinkedUpdate> = self
-                    .get_obj(c)
-                    .await?
-                    .ok_or_else(|| anyhow!("Incomplete Event"))?;
-                let rev: WithBlock<Revocation> = self
-                    .get_obj(&link.base.update)
-                    .await?
-                    .ok_or_else(|| anyhow!("Incomplete Event"))?;
-                Result::<(WithBlock<LinkedUpdate>, WithBlock<Revocation>)>::Ok((link, rev))
-            }))
-            .await?,
-        }))
-    }
-
     async fn get_obj<T>(&self, c: &Cid) -> Result<Option<WithBlock<T>>>
     where
         T: FromBlock,
@@ -327,83 +265,6 @@ where
             .map(|v| Block::new(*c, v).and_then(WithBlock::try_from))
             .transpose()
     }
-
-    pub(crate) async fn try_merge_heads(
-        &self,
-        updates: impl Iterator<Item = Cid> + Send,
-        invocations: impl Iterator<Item = Cid> + Send,
-    ) -> Result<()> {
-        self.try_merge_updates(updates).await?;
-        self.try_merge_invocations(invocations).await?;
-        Ok(())
-    }
-
-    #[async_recursion]
-    async fn try_merge_updates(
-        &self,
-        updates: impl Iterator<Item = Cid> + Send + 'async_recursion,
-    ) -> Result<()> {
-        try_join_all(updates.map(|head| async move {
-            if self.delegation_heads.get_height(&head).await?.is_some() {
-                return Ok(());
-            };
-            let update = self
-                .get_event(&head)
-                .await?
-                .ok_or_else(|| anyhow!("Missing Event"))?;
-
-            self.try_merge_updates(
-                stream::iter(update.prev.iter().map(Ok).collect::<Vec<Result<_>>>())
-                    .try_filter_map(|d| async move {
-                        self.delegation_heads.get_height(d).await.map(|o| match o {
-                            Some(_) => None,
-                            None => Some(*d),
-                        })
-                    })
-                    .try_collect::<Vec<Cid>>()
-                    .await?
-                    .into_iter(),
-            )
-            .await?;
-
-            self.apply(update).await
-        }))
-        .await?;
-        Ok(())
-    }
-
-    #[async_recursion]
-    pub(crate) async fn try_merge_invocations(
-        &self,
-        invocations: impl Iterator<Item = Cid> + Send + 'async_recursion,
-    ) -> Result<()> {
-        try_join_all(invocations.map(|head| async move {
-            if self.invocation_heads.get_height(&head).await?.is_some() {
-                return Result::<()>::Ok(());
-            };
-
-            let invs: Invocations = self.get_invocation(&head).await?;
-
-            self.try_merge_invocations(
-                stream::iter(invs.prev.iter().map(Ok).collect::<Vec<Result<_>>>())
-                    .try_filter_map(|i| async move {
-                        self.invocation_heads.get_height(i).await.map(|o| match o {
-                            Some(_) => None,
-                            None => Some(*i),
-                        })
-                    })
-                    .try_collect::<Vec<Cid>>()
-                    .await?
-                    .into_iter(),
-            )
-            .await?;
-
-            self.apply_invocations(invs).await?;
-            Ok(())
-        }))
-        .await?;
-        Ok(())
-    }
 }
 
 #[rocket::async_trait]
diff --git a/src/kv/mod.rs b/src/kv/mod.rs
index 26af4965..2fd87001 100644
--- a/src/kv/mod.rs
+++ b/src/kv/mod.rs
@@ -2,19 +2,15 @@ use anyhow::Result;
 use kepler_lib::libipld::{
     cbor::DagCborCodec, cid::Cid, codec::Encode, multihash::Code, raw::RawCodec,
 };
-use libp2p::core::PeerId;
-use rocket::futures::{Stream, StreamExt};
 use serde::{Deserialize, Serialize};
 
 mod entries;
 mod store;
 
-use super::{orbit::AbortOnDrop, Block};
+use super::Block;
 
 pub use entries::{Object, ObjectBuilder};
-pub use store::Store;
-
-type TaskHandle = AbortOnDrop<()>;
+pub use store::{ReadResponse, Store};
 
 #[derive(Clone)]
 pub struct Service<B> {
@@ -77,229 +73,3 @@ enum KVMessage {
     Heads(#[serde(with = "vec_cid_bin")] Vec<Cid>),
     StateReq,
 }
-
-async fn kv_task<B>(
-    events: impl Stream<Item = Result<(PeerId, KVMessage)>> + Send,
-    store: Store<B>,
-    peer_id: PeerId,
-) {
-    debug!("starting KV task");
-    events
-        .for_each_concurrent(None, |ev| async {
-            match ev {
-                Ok((p, ev)) if p == peer_id => {
-                    debug!("{} filtered out this event from self: {:?}", p, ev)
-                }
-                Ok((p, KVMessage::Heads(heads))) => {
-                    debug!("{} received new heads from {}", peer_id, p);
-                    // sync heads
-                    // if let Err(e) = store.try_merge_heads(heads.into_iter()).await {
-                    //     error!("failed to merge heads {}", e);
-                    // };
-                }
-                Ok((p, KVMessage::StateReq)) => {
-                    // debug!("{} requests state", p);
-                    // send heads
-                    // if let Err(e) = store.broadcast_heads().await {
-                    //     error!("failed to broadcast heads {}", e);
-                    // };
-                }
-                Err(e) => {
-                    error!("{}", e);
-                }
-            }
-        })
-        .await;
-}
-
-#[cfg(test)]
-mod test {
-    // use libp2p::{identity::Keypair, multiaddr::Protocol};
-
-    // use super::*;
-    // use crate::{config, relay::test::test_relay};
-    // use std::{
-    //     collections::BTreeMap, convert::TryFrom, path::PathBuf, str::FromStr, time::Duration,
-    // };
-
-    // async fn create_store<I>(
-    //     id: &Cid,
-    //     path: PathBuf,
-    //     keypair: Keypair,
-    //     allowed_peers: I,
-    // ) -> Result<(Store, behaviour::BehaviourProcess), anyhow::Error>
-    // where
-    //     I: IntoIterator<Item = PeerId> + 'static,
-    // {
-    //     std::fs::create_dir(path.clone())?;
-    //     let config = config::Config {
-    //         storage: config::Storage {
-    //             blocks: config::BlockStorage::Local(config::LocalBlockStorage {
-    //                 path: path.clone(),
-    //             }),
-    //             indexes: config::IndexStorage::Local(config::LocalIndexStorage {
-    //                 path: path.clone(),
-    //             }),
-    //         },
-    //         ..Default::default()
-    //     };
-    //     let (ipfs, ipfs_task, receiver) = create_ipfs(*id, &config, keypair, allowed_peers).await?;
-    //     tokio::spawn(ipfs_task);
-    //     let store = Store::new(*id, ipfs, config.storage.indexes).await?;
-    //     Ok((
-    //         store.clone(),
-    //         behaviour::BehaviourProcess::new(store, receiver),
-    //     ))
-    // }
-
-    // #[tokio::test(flavor = "multi_thread")]
-    // async fn test() -> Result<(), anyhow::Error> {
-    //     let relay = test_relay().await?;
-    //     let relay_peer_id = relay.id.clone();
-    //     let relay_internal = relay.internal();
-
-    //     let tmp = tempfile::TempDir::new("test_streams")?;
-    //     let id =
-    //         Cid::from_str("bafkreieq5jui4j25lacwomsqgjeswwl3y5zcdrresptwgmfylxo2depppq").unwrap();
-
-    //     let alice_keypair = Keypair::generate_ed25519();
-    //     let alice_peer_id = alice_keypair.public().to_peer_id();
-    //     let bob_keypair = Keypair::generate_ed25519();
-    //     let bob_peer_id = bob_keypair.public().to_peer_id();
-
-    //     let (alice_store, _alice_behaviour_process) =
-    //         create_store(&id, tmp.path().join("alice"), alice_keypair, [bob_peer_id]).await?;
-    //     let (bob_store, _bob_behaviour_process) =
-    //         create_store(&id, tmp.path().join("bob"), bob_keypair, [bob_peer_id]).await?;
-
-    //     let alice_service = alice_store.start_service().await?;
-    //     let bob_service = bob_store.start_service().await?;
-
-    //     // Connect the peers to the relay.
-    //     alice_service
-    //         .ipfs
-    //         .connect(
-    //             MultiaddrWithoutPeerId::try_from(relay_internal.clone())?
-    //                 .with(relay_peer_id.clone()),
-    //         )
-    //         .await
-    //         .expect("alice failed to connect to relay");
-    //     bob_service
-    //         .ipfs
-    //         .connect(
-    //             MultiaddrWithoutPeerId::try_from(relay_internal.clone())?
-    //                 .with(relay_peer_id.clone()),
-    //         )
-    //         .await
-    //         .expect("bob failed to connect to relay");
-
-    //     // Connect the peers to eachother.
-    //     bob_service
-    //         .ipfs
-    //         .connect(
-    //             MultiaddrWithoutPeerId::try_from(
-    //                 relay
-    //                     .external()
-    //                     .with(Protocol::P2p(relay_peer_id.into()))
-    //                     .with(Protocol::P2pCircuit),
-    //             )?
-    //             .with(alice_peer_id.clone()),
-    //         )
-    //         .await
-    //         .expect("bob failed to connect to alice");
-
-    //     // TODO: Work out why there is a race condition, and fix it so we don't need this sleep between connecting and writing.
-    //     tokio::time::sleep(Duration::from_millis(50)).await;
-
-    //     let json = r#"{"hello":"there"}"#;
-    //     let key1 = "my_json.json";
-    //     let key2 = "my_dup_json.json";
-    //     let md: BTreeMap<String, String> =
-    //         [("content-type".to_string(), "application/json".to_string())]
-    //             .to_vec()
-    //             .into_iter()
-    //             .collect();
-    //     let dab = to_block(&id).unwrap();
-    //     let dummy_auth = *dab.cid();
-    //     alice_service.ipfs.put_block(dab).await?;
-
-    //     let kv_obj_1 = ObjectBuilder::new(key1.as_bytes().to_vec(), md.clone(), dummy_auth.clone());
-    //     let kv_obj_2 = ObjectBuilder::new(key2.as_bytes().to_vec(), md.clone(), dummy_auth.clone());
-
-    //     type RmItem = (Vec<u8>, Option<(u64, Cid)>, Cid);
-    //     let rm: Vec<RmItem> = vec![];
-    //     alice_service
-    //         .write(vec![(kv_obj_1, json.as_bytes())], rm.clone())
-    //         .await?;
-    //     bob_service
-    //         .write(vec![(kv_obj_2, json.as_bytes())], rm)
-    //         .await?;
-
-    //     {
-    //         // ensure only alice has kv_obj_1
-    //         let o = alice_service
-    //             .get(key1)
-    //             .await?
-    //             .expect("object 1 not found for alice");
-    //         assert_eq!(&o.key, key1.as_bytes());
-    //         assert_eq!(&o.metadata, &md);
-    //         assert_eq!(bob_service.get(key1).await?, None, "object 1 found for bob");
-    //     };
-    //     {
-    //         // ensure only bob has kv_obj_2
-    //         let o = bob_service
-    //             .get(key2)
-    //             .await?
-    //             .expect("object 2 not found for bob");
-    //         assert_eq!(&o.key, key2.as_bytes());
-    //         assert_eq!(&o.metadata, &md);
-    //         assert_eq!(
-    //             alice_service.get(key2).await?,
-    //             None,
-    //             "object 2 found for alice"
-    //         );
-    //     };
-
-    //     tokio::time::sleep(Duration::from_millis(500)).await;
-
-    //     assert_eq!(
-    //         bob_service
-    //             .get(key1)
-    //             .await?
-    //             .expect("object 1 not found for bob"),
-    //         alice_service
-    //             .get(key1)
-    //             .await?
-    //             .expect("object 1 not found for alice")
-    //     );
-
-    //     assert_eq!(
-    //         bob_service
-    //             .get(key2)
-    //             .await?
-    //             .expect("object 2 not found for bob"),
-    //         alice_service
-    //             .get(key2)
-    //             .await?
-    //             .expect("object 2 not found for alice")
-    //     );
-
-    //     // remove key1
-    //     let add: Vec<(&[u8], Cid)> = vec![];
-    //     alice_service
-    //         .index(add, vec![(key1.as_bytes().to_vec(), None, dummy_auth)])
-    //         .await?;
-
-    //     assert_eq!(
-    //         alice_service.get(key1).await?,
-    //         None,
-    //         "alice still has object 1"
-    //     );
-
-    //     std::thread::sleep(Duration::from_millis(500));
-
-    //     assert_eq!(bob_service.get(key1).await?, None, "bob still has object 1");
-
-    //     Ok(())
-    // }
-}
diff --git a/src/kv/store.rs b/src/kv/store.rs
index 96f76fe9..9608c026 100644
--- a/src/kv/store.rs
+++ b/src/kv/store.rs
@@ -149,6 +149,8 @@ impl<B> Store<B> {
     }
 }
 
+pub struct ReadResponse<R>(pub BTreeMap<String, String>, pub R);
+
 impl<B> Store<B>
 where
     B: ImmutableStore + 'static,
@@ -168,7 +170,7 @@ where
                     .blocks
                     .read_to_vec(cid.hash())
                     .await?
-                    .map(|v| Ok(Block::new(cid, v)?.decode()?))
+                    .map(|v| Block::new(cid, v)?.decode())
                     .transpose(),
                 _ => Ok(None),
             },
@@ -177,7 +179,7 @@ where
     }
 
     #[instrument(name = "kv::read", skip_all)]
-    pub async fn read<N>(&self, key: N) -> Result<Option<(BTreeMap<String, String>, B::Readable)>>
+    pub async fn read<N>(&self, key: N) -> Result<Option<ReadResponse<B::Readable>>>
     where
         N: AsRef<[u8]>,
     {
@@ -185,8 +187,8 @@ where
             Ok(Some(content)) => content,
             _ => return Ok(None),
         };
-        match self.blocks.read(&kv_obj.value.hash()).await? {
-            Some(r) => Ok(Some((kv_obj.metadata, r))),
+        match self.blocks.read(kv_obj.value.hash()).await? {
+            Some(r) => Ok(Some(ReadResponse(kv_obj.metadata, r))),
             None => Err(anyhow!("Indexed contents missing from block store")),
         }
     }
diff --git a/src/lib.rs b/src/lib.rs
index 3da0a31d..b099ca39 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -71,7 +71,7 @@ impl From<BlockConfig> for BlockStorage {
 }
 
 pub async fn app(config: &Figment) -> Result<Rocket<Build>> {
-    let kepler_config: Config = config.extract::<Config>()?.into();
+    let kepler_config: Config = config.extract::<Config>()?;
 
     tracing::tracing_try_init(&kepler_config.log);
 
diff --git a/src/orbit.rs b/src/orbit.rs
index 74e72b3d..d2323de6 100644
--- a/src/orbit.rs
+++ b/src/orbit.rs
@@ -78,7 +78,8 @@ where
         B::Error: 'static,
     {
         let id = config.manifest.id().get_cid();
-        let local_peer_id = PeerId::from_public_key(&PublicKey::Ed25519(config.identity.public()));
+        let _local_peer_id = PeerId::from_public_key(&PublicKey::Ed25519(config.identity.public()));
+        let _relay = &config.relay;
 
         let blocks = match config.blocks.open(config.manifest.id()).await? {
             Some(b) => b,
@@ -105,7 +106,8 @@ where
         B::Error: 'static,
     {
         let id = config.manifest.id().get_cid();
-        let local_peer_id = PeerId::from_public_key(&PublicKey::Ed25519(config.identity.public()));
+        let _local_peer_id = PeerId::from_public_key(&PublicKey::Ed25519(config.identity.public()));
+        let _relay = &config.relay;
 
         let blocks = config.blocks.create(config.manifest.id()).await?;
         let service_store = Store::new(id, blocks.clone(), config.index.clone()).await?;
@@ -140,28 +142,28 @@ pub async fn create_orbit(
     relay: (PeerId, Multiaddr),
     kp: Ed25519Keypair,
 ) -> Result<Option<Orbit<BlockStores>>> {
-    let md = match Manifest::resolve_dyn(id, None).await? {
-        Some(m) => m,
+    match Manifest::resolve_dyn(id, None).await? {
+        Some(_) => {}
         _ => return Ok(None),
     };
 
     // fails if DIR exists, this is Create, not Open
-    if store_config.exists(&id).await? {
+    if store_config.exists(id).await? {
         return Ok(None);
     }
 
-    store_config.setup_orbit(&id, &kp).await?;
+    store_config.setup_orbit(id, &kp).await?;
 
     Orbit::create(
         &OrbitPeerConfigBuilder::<BlockConfig, config::IndexStorage>::default()
             .manifest(
-                Manifest::resolve_dyn(&id, None)
+                Manifest::resolve_dyn(id, None)
                     .await?
                     .ok_or_else(|| anyhow!("Orbit DID Document not resolvable"))?,
             )
             .identity(
                 store_config
-                    .key_pair(&id)
+                    .key_pair(id)
                     .await?
                     .ok_or_else(|| anyhow!("Peer Identity key could not be found"))?,
             )
@@ -242,7 +244,7 @@ mod tests {
     };
     use std::convert::TryInto;
 
-    async fn op(md: Manifest) -> anyhow::Result<Orbit<BlockStores>> {
+    async fn op(_md: Manifest) -> anyhow::Result<Orbit<BlockStores>> {
         // let dir = Tempfile::new(&md.id().get_cid().to_string())
         //     .unwrap()
         //     .path()
diff --git a/src/routes/mod.rs b/src/routes/mod.rs
index e9c989f0..958fe778 100644
--- a/src/routes/mod.rs
+++ b/src/routes/mod.rs
@@ -22,7 +22,7 @@ use tracing::{info_span, Instrument};
 
 use crate::{
     auth_guards::{DelegateAuthWrapper, InvokeAuthWrapper, KVAction},
-    kv::ObjectBuilder,
+    kv::{ObjectBuilder, ReadResponse},
     relay::RelayNode,
     storage::ImmutableStore,
     tracing::TracingSpan,
@@ -148,9 +148,9 @@ pub async fn invoke(
     .await
 }
 
-pub async fn handle_kv_action<'a, B>(
+pub async fn handle_kv_action<B>(
     action: KVAction<B>,
-    data: Data<'a>,
+    data: Data<'_>,
 ) -> Result<InvocationResponse<B::Readable>, (Status, String)>
 where
     B: 'static + ImmutableStore,
@@ -175,7 +175,7 @@ where
             Ok(InvocationResponse::EmptySuccess)
         }
         KVAction::Get { orbit, key } => match orbit.service.read(key).await {
-            Ok(Some((md, r))) => Ok(InvocationResponse::KVResponse(KVResponse::new(
+            Ok(Some(ReadResponse(md, r))) => Ok(InvocationResponse::KVResponse(KVResponse::new(
                 Metadata(md),
                 r,
             ))),
diff --git a/src/storage/s3.rs b/src/storage/s3.rs
index 073e2bc0..ae00cbd2 100644
--- a/src/storage/s3.rs
+++ b/src/storage/s3.rs
@@ -22,11 +22,10 @@ use kepler_lib::{
     resource::OrbitId,
 };
 use libp2p::identity::{ed25519::Keypair as Ed25519Keypair, error::DecodingError};
-use regex::Regex;
 use rocket::{async_trait, http::hyper::Uri};
 use serde::{Deserialize, Serialize};
 use serde_with::{serde_as, DisplayFromStr};
-use std::{io::Error as IoError, path::PathBuf, str::FromStr};
+use std::io::Error as IoError;
 use tempfile::tempfile;
 use tokio::fs::File;
 use tokio_util::compat::{Compat, TokioAsyncReadCompatExt};
@@ -97,7 +96,7 @@ impl ProviderUtils for S3BlockConfig {
         self.key_pair(orbit).await.map(|o| o.is_some())
     }
     async fn relay_key_pair(&self) -> Result<Ed25519Keypair, Self::Error> {
-        let client = new_client(&self);
+        let client = new_client(self);
         match client
             .get_object()
             .bucket(&self.bucket)
@@ -130,7 +129,7 @@ impl ProviderUtils for S3BlockConfig {
         }
     }
     async fn key_pair(&self, orbit: &OrbitId) -> Result<Option<Ed25519Keypair>, Self::Error> {
-        match new_client(&self)
+        match new_client(self)
             .get_object()
             .bucket(&self.bucket)
             .key(format!("{}/keypair", orbit.get_cid()))
@@ -152,7 +151,7 @@ impl ProviderUtils for S3BlockConfig {
         }
     }
     async fn setup_orbit(&self, orbit: &OrbitId, key: &Ed25519Keypair) -> Result<(), Self::Error> {
-        let client = new_client(&self);
+        let client = new_client(self);
         client
             .put_object()
             .bucket(&self.bucket)
@@ -301,29 +300,3 @@ impl ImmutableStore for S3BlockStore {
         }
     }
 }
-
-fn path_to_config(path: PathBuf) -> (S3BlockConfig, Cid) {
-    let re =
-            Regex::new(r"^/s3bucket/(?P<bucket>.*)/s3endpoint/(?P<s3endpoint>.*)/dynamotable/(?P<table>.*)/dynamoendpoint/(?P<dynamoendpoint>.*)/orbitcid/(?P<orbit>.*)/(blockstore|datastore)$")
-                .unwrap();
-    let fields = re.captures(path.to_str().unwrap()).unwrap();
-    let s3_bucket = fields.name("bucket").unwrap().as_str().to_string();
-    let s3_endpoint = Some(fields.name("s3endpoint").unwrap().as_str())
-        .filter(|s| !s.is_empty())
-        .map(|e| Uri::from_str(e).unwrap());
-    let dynamo_table = fields.name("table").unwrap().as_str().to_string();
-    let dynamo_endpoint = Some(fields.name("dynamoendpoint").unwrap().as_str())
-        .filter(|s| !s.is_empty())
-        .map(|e| Uri::from_str(e).unwrap());
-    let orbit = Cid::from_str(fields.name("orbit").unwrap().as_str()).unwrap();
-
-    let config = S3BlockConfig {
-        bucket: s3_bucket,
-        endpoint: s3_endpoint,
-        // dynamodb: config::DynamoStorage {
-        //     table: dynamo_table,
-        //     endpoint: dynamo_endpoint,
-        // },
-    };
-    (config, orbit)
-}
diff --git a/src/storage/utils.rs b/src/storage/utils.rs
index 582d5628..5f04dcd6 100644
--- a/src/storage/utils.rs
+++ b/src/storage/utils.rs
@@ -2,7 +2,7 @@ use aws_types::sdk_config::SdkConfig;
 use core::pin::Pin;
 use futures::{
     executor::block_on,
-    io::{AllowStdIo, AsyncWrite, Error},
+    io::{AsyncWrite, Error},
     task::{Context, Poll},
 };
 use libipld::cid::multihash::Hasher;

From 8fee540a94c6bcddc670ad198a1d9b4b9689be9c Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Thu, 3 Nov 2022 13:26:38 +0100
Subject: [PATCH 44/73] remove deprecated function, ensure writers are flushed

---
 src/storage/file_system.rs |  3 ++-
 src/storage/s3.rs          | 16 +++++++++-------
 2 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/src/storage/file_system.rs b/src/storage/file_system.rs
index c4d57ddd..64ccb27d 100644
--- a/src/storage/file_system.rs
+++ b/src/storage/file_system.rs
@@ -2,7 +2,7 @@ use crate::{
     orbit::ProviderUtils,
     storage::{utils::HashBuffer, ImmutableStore, StorageConfig},
 };
-use futures::io::{copy, AllowStdIo};
+use futures::io::{copy, AllowStdIo, AsyncWriteExt};
 use kepler_lib::{
     libipld::cid::{
         multibase::{encode, Base},
@@ -142,6 +142,7 @@ impl ImmutableStore for FileSystemStore {
         );
 
         copy(data, &mut hb).await?;
+        hb.flush().await?;
 
         let (mut hasher, file) = hb.into_inner();
 
diff --git a/src/storage/s3.rs b/src/storage/s3.rs
index ae00cbd2..3da24262 100644
--- a/src/storage/s3.rs
+++ b/src/storage/s3.rs
@@ -8,7 +8,7 @@ use aws_sdk_s3::{
 };
 use aws_smithy_http::{body::SdkBody, byte_stream::Error as ByteStreamError, endpoint::Endpoint};
 use futures::{
-    io::copy,
+    io::{copy, AllowStdIo, AsyncWriteExt},
     stream::{IntoAsyncRead, MapErr, TryStreamExt},
 };
 use kepler_lib::{
@@ -26,9 +26,7 @@ use rocket::{async_trait, http::hyper::Uri};
 use serde::{Deserialize, Serialize};
 use serde_with::{serde_as, DisplayFromStr};
 use std::io::Error as IoError;
-use tempfile::tempfile;
-use tokio::fs::File;
-use tokio_util::compat::{Compat, TokioAsyncReadCompatExt};
+use tempfile::NamedTempFile;
 
 use crate::{
     orbit::ProviderUtils,
@@ -241,20 +239,24 @@ impl ImmutableStore for S3BlockStore {
         data: impl futures::io::AsyncRead + Send,
     ) -> Result<Multihash, Self::Error> {
         // TODO find a way to do this without filesystem access
-        let mut hb =
-            HashBuffer::<Blake3Hasher<32>, Compat<File>>::new(File::from(tempfile()?).compat());
+        let mut hb = HashBuffer::<Blake3Hasher<32>, AllowStdIo<NamedTempFile>>::new(
+            AllowStdIo::new(NamedTempFile::new()?),
+        );
 
         copy(data, &mut hb).await?;
+        hb.flush().await?;
 
         let (mut hasher, file) = hb.into_inner();
         let multihash = Code::Blake3_256.wrap(hasher.finalize())?;
 
+        let (_, path) = file.into_inner().into_parts();
+
         self.client
             .put_object()
             .bucket(&self.bucket)
             .key(self.key(&multihash))
             // TODO deprecated but also doesnt require a path
-            .body(ByteStream::from_file(file.into_inner()).await?)
+            .body(ByteStream::from_path(&path).await?)
             .send()
             .await
             .map_err(S3Error::from)?;

From 52fe2cb8e10315991e43fa900b2c44371d282e7f Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Thu, 3 Nov 2022 13:52:32 +0100
Subject: [PATCH 45/73] fix protoc in clippy workflow

---
 .github/workflows/rust.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index 42247102..7fb54ff9 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -49,6 +49,9 @@ jobs:
       run: |
         rustup target add wasm32-unknown-unknown
 
+    - name: Install Protoc
+      uses: arduino/setup-protoc@v1
+
     - name: Clippy
       working-directory: ${{ matrix.pkg }}
       run: RUSTFLAGS="-Dwarnings" cargo clippy

From 4c10cbba2262ffe37415ce398f873ab1b9d1e8ac Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Thu, 3 Nov 2022 15:02:39 +0100
Subject: [PATCH 46/73] fix tests

---
 Cargo.lock                 | 335 +++++++++++++++++++++++--------------
 lib/Cargo.toml             |   4 +-
 src/orbit.rs               |  43 ++---
 src/storage/file_system.rs |  13 +-
 4 files changed, 250 insertions(+), 145 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 37442bfa..77095d89 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -41,9 +41,9 @@ dependencies = [
 
 [[package]]
 name = "aes"
-version = "0.8.1"
+version = "0.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bfe0133578c0986e1fe3dfcd4af1cc5b2dd6c3dbf534d69916ce16a2701d40ba"
+checksum = "433cfd6710c9986c576a25ca913c39d66a6474107b406f34f91d4a8923395241"
 dependencies = [
  "cfg-if",
  "cipher 0.4.3",
@@ -71,7 +71,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "82e1366e0c69c9f927b1fa5ce2c7bf9eafc8f9268c0b9800729e8b267612447c"
 dependencies = [
  "aead 0.5.1",
- "aes 0.8.1",
+ "aes 0.8.2",
  "cipher 0.4.3",
  "ctr 0.9.2",
  "ghash 0.5.0",
@@ -174,9 +174,9 @@ dependencies = [
 
 [[package]]
 name = "async-global-executor"
-version = "2.3.0"
+version = "2.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0da5b41ee986eed3f524c380e6d64965aea573882a8907682ad100f7859305ca"
+checksum = "f1b6f5d7df27bd294849f8eec66ecfc63d11814df7a4f5d74168a2394467b776"
 dependencies = [
  "async-channel",
  "async-executor",
@@ -189,16 +189,16 @@ dependencies = [
 
 [[package]]
 name = "async-io"
-version = "1.9.0"
+version = "1.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "83e21f3a490c72b3b0cf44962180e60045de2925d8dff97918f7ee43c8f637c7"
+checksum = "e8121296a9f05be7f34aa4196b1747243b3b62e048bb7906f644f3fbfc490cf7"
 dependencies = [
+ "async-lock",
  "autocfg",
  "concurrent-queue",
  "futures-lite",
  "libc",
  "log",
- "once_cell",
  "parking",
  "polling",
  "slab",
@@ -209,11 +209,12 @@ dependencies = [
 
 [[package]]
 name = "async-lock"
-version = "2.5.0"
+version = "2.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e97a171d191782fba31bb902b14ad94e24a68145032b7eedf871ab0bc0d077b6"
+checksum = "c8101efe8695a6c17e02911402145357e718ac92d3ff88ae8419e84b1707b685"
 dependencies = [
  "event-listener",
+ "futures-lite",
 ]
 
 [[package]]
@@ -364,7 +365,7 @@ dependencies = [
  "http",
  "hyper",
  "ring",
- "time 0.3.15",
+ "time 0.3.16",
  "tokio",
  "tower",
  "tracing",
@@ -528,7 +529,7 @@ dependencies = [
  "percent-encoding",
  "regex",
  "ring",
- "time 0.3.15",
+ "time 0.3.16",
  "tracing",
 ]
 
@@ -580,7 +581,7 @@ dependencies = [
  "http",
  "http-body",
  "hyper",
- "hyper-rustls",
+ "hyper-rustls 0.22.1",
  "lazy_static",
  "pin-project-lite",
  "tokio",
@@ -664,7 +665,7 @@ dependencies = [
  "itoa",
  "num-integer",
  "ryu",
- "time 0.3.15",
+ "time 0.3.16",
 ]
 
 [[package]]
@@ -971,7 +972,7 @@ dependencies = [
  "serde_with 2.0.1",
  "siwe",
  "thiserror",
- "time 0.3.15",
+ "time 0.3.16",
  "url",
 ]
 
@@ -1054,9 +1055,9 @@ dependencies = [
 
 [[package]]
 name = "cc"
-version = "1.0.73"
+version = "1.0.74"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11"
+checksum = "581f5dba903aac52ea3feb5ec4810848460ee833876f1f9b0fdeab1f19091574"
 
 [[package]]
 name = "cfb-mode"
@@ -1164,9 +1165,9 @@ dependencies = [
 
 [[package]]
 name = "cmake"
-version = "0.1.48"
+version = "0.1.49"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e8ad8cef104ac57b68b89df3208164d228503abbdce70f6880ffa3d970e7443a"
+checksum = "db34956e100b30725f2eb215f90d4871051239535632f84fea3bc92722c66b7c"
 dependencies = [
  "cc",
 ]
@@ -1228,7 +1229,7 @@ dependencies = [
  "rand 0.8.5",
  "sha2 0.10.6",
  "subtle",
- "time 0.3.15",
+ "time 0.3.16",
  "version_check 0.9.4",
 ]
 
@@ -1367,7 +1368,7 @@ version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c1a816186fa68d9e426e3cb4ae4dff1fcd8e4a2c34b781bf7a822574a0d0aac8"
 dependencies = [
- "sct",
+ "sct 0.6.1",
 ]
 
 [[package]]
@@ -1437,9 +1438,9 @@ dependencies = [
 
 [[package]]
 name = "cxx"
-version = "1.0.79"
+version = "1.0.80"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3f83d0ebf42c6eafb8d7c52f7e5f2d3003b89c7aa4fd2b79229209459a849af8"
+checksum = "6b7d4e43b25d3c994662706a1d4fcfc32aaa6afd287502c111b237093bb23f3a"
 dependencies = [
  "cc",
  "cxxbridge-flags",
@@ -1449,9 +1450,9 @@ dependencies = [
 
 [[package]]
 name = "cxx-build"
-version = "1.0.79"
+version = "1.0.80"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "07d050484b55975889284352b0ffc2ecbda25c0c55978017c132b29ba0818a86"
+checksum = "84f8829ddc213e2c1368e51a2564c552b65a8cb6a28f31e576270ac81d5e5827"
 dependencies = [
  "cc",
  "codespan-reporting",
@@ -1464,15 +1465,15 @@ dependencies = [
 
 [[package]]
 name = "cxxbridge-flags"
-version = "1.0.79"
+version = "1.0.80"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "99d2199b00553eda8012dfec8d3b1c75fce747cf27c169a270b3b99e3448ab78"
+checksum = "e72537424b474af1460806647c41d4b6d35d09ef7fe031c5c2fa5766047cc56a"
 
 [[package]]
 name = "cxxbridge-macro"
-version = "1.0.79"
+version = "1.0.80"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dcb67a6de1f602736dd7eaead0080cf3435df806c61b24b13328db128c58868f"
+checksum = "309e4fb93eed90e1e14bea0da16b209f81813ba9fc7830c20ed151dd7bc0a4d7"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1501,12 +1502,12 @@ dependencies = [
 
 [[package]]
 name = "darling"
-version = "0.14.1"
+version = "0.14.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4529658bdda7fd6769b8614be250cdcfc3aeb0ee72fe66f9e41e5e5eb73eac02"
+checksum = "b0dd3cd20dc6b5a876612a6e5accfe7f3dd883db6d07acfbf14c128f61550dfa"
 dependencies = [
- "darling_core 0.14.1",
- "darling_macro 0.14.1",
+ "darling_core 0.14.2",
+ "darling_macro 0.14.2",
 ]
 
 [[package]]
@@ -1539,9 +1540,9 @@ dependencies = [
 
 [[package]]
 name = "darling_core"
-version = "0.14.1"
+version = "0.14.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "649c91bc01e8b1eac09fb91e8dbc7d517684ca6be8ebc75bb9cafc894f9fdb6f"
+checksum = "a784d2ccaf7c98501746bf0be29b2022ba41fd62a2e622af997a03e9f972859f"
 dependencies = [
  "fnv",
  "ident_case",
@@ -1575,11 +1576,11 @@ dependencies = [
 
 [[package]]
 name = "darling_macro"
-version = "0.14.1"
+version = "0.14.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ddfc69c5bfcbd2fc09a0f38451d2daf0e372e367986a83906d1b0dbc88134fb5"
+checksum = "7618812407e9402654622dd402b0a89dff9ba93badd6540781526117b92aab7e"
 dependencies = [
- "darling_core 0.14.1",
+ "darling_core 0.14.2",
  "quote",
  "syn",
 ]
@@ -1671,7 +1672,7 @@ version = "0.11.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1f91d4cfa921f1c05904dc3c57b4a32c38aed3340cce209f3a6fd1478babafc4"
 dependencies = [
- "darling 0.14.1",
+ "darling 0.14.2",
  "proc-macro2",
  "quote",
  "syn",
@@ -1734,7 +1735,7 @@ dependencies = [
 [[package]]
 name = "did-ethr"
 version = "0.1.1"
-source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#fce1a8b031d21ba8abbb559e791adf4c5f1b3a68"
+source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#e5cf96b7efd899a080b8f2cc763fc39a434ce536"
 dependencies = [
  "async-trait",
  "chrono",
@@ -1750,7 +1751,7 @@ dependencies = [
 [[package]]
 name = "did-ion"
 version = "0.1.0"
-source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#fce1a8b031d21ba8abbb559e791adf4c5f1b3a68"
+source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#e5cf96b7efd899a080b8f2cc763fc39a434ce536"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -1772,7 +1773,7 @@ dependencies = [
 [[package]]
 name = "did-method-key"
 version = "0.1.3"
-source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#fce1a8b031d21ba8abbb559e791adf4c5f1b3a68"
+source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#e5cf96b7efd899a080b8f2cc763fc39a434ce536"
 dependencies = [
  "async-trait",
  "multibase 0.8.0",
@@ -1787,7 +1788,7 @@ dependencies = [
 [[package]]
 name = "did-onion"
 version = "0.1.2"
-source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#fce1a8b031d21ba8abbb559e791adf4c5f1b3a68"
+source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#e5cf96b7efd899a080b8f2cc763fc39a434ce536"
 dependencies = [
  "async-trait",
  "http",
@@ -1800,7 +1801,7 @@ dependencies = [
 [[package]]
 name = "did-pkh"
 version = "0.1.1"
-source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#fce1a8b031d21ba8abbb559e791adf4c5f1b3a68"
+source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#e5cf96b7efd899a080b8f2cc763fc39a434ce536"
 dependencies = [
  "async-trait",
  "bech32",
@@ -1817,7 +1818,7 @@ dependencies = [
 [[package]]
 name = "did-tz"
 version = "0.1.2"
-source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#fce1a8b031d21ba8abbb559e791adf4c5f1b3a68"
+source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#e5cf96b7efd899a080b8f2cc763fc39a434ce536"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -1837,7 +1838,7 @@ dependencies = [
 [[package]]
 name = "did-web"
 version = "0.1.2"
-source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#fce1a8b031d21ba8abbb559e791adf4c5f1b3a68"
+source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#e5cf96b7efd899a080b8f2cc763fc39a434ce536"
 dependencies = [
  "async-trait",
  "http",
@@ -1849,7 +1850,7 @@ dependencies = [
 [[package]]
 name = "did-webkey"
 version = "0.1.1"
-source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#fce1a8b031d21ba8abbb559e791adf4c5f1b3a68"
+source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#e5cf96b7efd899a080b8f2cc763fc39a434ce536"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -2026,9 +2027,9 @@ dependencies = [
 
 [[package]]
 name = "ff"
-version = "0.12.0"
+version = "0.12.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "df689201f395c6b90dfe87127685f8dbfc083a5e779e613575d8bd7314300c3e"
+checksum = "d013fc25338cc558c5c2cfbad646908fb23591e2404481826742b651c9af7160"
 dependencies = [
  "rand_core 0.6.4",
  "subtle",
@@ -2346,9 +2347,9 @@ dependencies = [
 
 [[package]]
 name = "h2"
-version = "0.3.14"
+version = "0.3.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5ca32592cf21ac7ccab1825cd87f6c9b3d9022c44d086172ed0966bec8af30be"
+checksum = "5f9f29bc9dda355256b2916cf526ab02ce0aeaaaf2bad60d65ef3f12f11dd0f4"
 dependencies = [
  "bytes",
  "fnv",
@@ -2489,9 +2490,9 @@ checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421"
 
 [[package]]
 name = "hyper"
-version = "0.14.20"
+version = "0.14.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "02c929dc5c39e335a03c405292728118860721b10190d98c2a0f0efd5baafbac"
+checksum = "abfba89e19b959ca163c7752ba59d737c1ceea53a5d31a149c805446fc958064"
 dependencies = [
  "bytes",
  "futures-channel",
@@ -2521,11 +2522,24 @@ dependencies = [
  "futures-util",
  "hyper",
  "log",
- "rustls",
+ "rustls 0.19.1",
  "rustls-native-certs",
  "tokio",
- "tokio-rustls",
- "webpki",
+ "tokio-rustls 0.22.0",
+ "webpki 0.21.4",
+]
+
+[[package]]
+name = "hyper-rustls"
+version = "0.23.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d87c48c02e0dc5e3b849a2041db3029fd066650f8f717c07bf8ed78ccb895cac"
+dependencies = [
+ "http",
+ "hyper",
+ "rustls 0.20.7",
+ "tokio",
+ "tokio-rustls 0.23.4",
 ]
 
 [[package]]
@@ -2543,9 +2557,9 @@ dependencies = [
 
 [[package]]
 name = "iana-time-zone"
-version = "0.1.51"
+version = "0.1.53"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f5a6ef98976b22b3b7f2f3a806f858cb862044cfa66805aa3ad84cb3d3b785ed"
+checksum = "64c122667b287044802d6ce17ee2ddf13207ed924c712de9a66a5814d5b64765"
 dependencies = [
  "android_system_properties",
  "core-foundation-sys",
@@ -2663,9 +2677,9 @@ checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02"
 
 [[package]]
 name = "io-lifetimes"
-version = "0.7.3"
+version = "0.7.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1ea37f355c05dde75b84bba2d767906ad522e97cd9e2eef2be7a4ab7fb442c06"
+checksum = "59ce5ef949d49ee85593fc4d3f3f95ad61657076395cbbce23e2121fc5542074"
 
 [[package]]
 name = "ipconfig"
@@ -2834,7 +2848,7 @@ dependencies = [
  "sled",
  "tempfile",
  "thiserror",
- "time 0.3.15",
+ "time 0.3.16",
  "tokio",
  "tokio-stream",
  "tokio-util",
@@ -2888,7 +2902,7 @@ dependencies = [
  "serde_json",
  "serde_with 1.14.0",
  "thiserror",
- "time 0.3.15",
+ "time 0.3.16",
  "tokio",
  "tracing",
 ]
@@ -2932,9 +2946,9 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.135"
+version = "0.2.137"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68783febc7782c6c5cb401fbda4de5a9898be1762314da0bb2c10ced61f18b0c"
+checksum = "fc7fcc620a3bff7cdd7a365be3376c97191aeaccc2a603e600951e452615bf89"
 
 [[package]]
 name = "libipld"
@@ -3486,14 +3500,14 @@ dependencies = [
 
 [[package]]
 name = "mio"
-version = "0.8.4"
+version = "0.8.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "57ee1c23c7c63b0c9250c339ffdc69255f110b298b901b9f6c82547b7b87caaf"
+checksum = "e5d732bc30207a6423068df043e3d02e0735b155ad7ce1a6f76fe2baa5b158de"
 dependencies = [
  "libc",
  "log",
  "wasi 0.11.0+wasi-snapshot-preview1",
- "windows-sys 0.36.1",
+ "windows-sys 0.42.0",
 ]
 
 [[package]]
@@ -3617,9 +3631,9 @@ dependencies = [
 
 [[package]]
 name = "native-tls"
-version = "0.2.10"
+version = "0.2.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fd7e2f3618557f980e0b17e8856252eee3c97fa12c54dff0ca290fb6266ca4a9"
+checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e"
 dependencies = [
  "lazy_static",
  "libc",
@@ -3828,9 +3842,9 @@ dependencies = [
 
 [[package]]
 name = "once_cell"
-version = "1.15.0"
+version = "1.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e82dad04139b71a90c080c8463fe0dc7902db5192d939bd0950f074d014339e1"
+checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860"
 
 [[package]]
 name = "opaque-debug"
@@ -3878,18 +3892,18 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
 
 [[package]]
 name = "openssl-src"
-version = "111.22.0+1.1.1q"
+version = "111.24.0+1.1.1s"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f31f0d509d1c1ae9cada2f9539ff8f37933831fd5098879e482aa687d659853"
+checksum = "3498f259dab01178c6228c6b00dcef0ed2a2d5e20d648c017861227773ea4abd"
 dependencies = [
  "cc",
 ]
 
 [[package]]
 name = "openssl-sys"
-version = "0.9.76"
+version = "0.9.77"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5230151e44c0f05157effb743e8d517472843121cf9243e8b81393edb5acd9ce"
+checksum = "b03b84c3b2d099b81f0953422b4d4ad58761589d0229b5506356afca05a3670a"
 dependencies = [
  "autocfg",
  "cc",
@@ -4216,15 +4230,15 @@ dependencies = [
 
 [[package]]
 name = "pkg-config"
-version = "0.3.25"
+version = "0.3.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae"
+checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160"
 
 [[package]]
 name = "polling"
-version = "2.3.0"
+version = "2.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "899b00b9c8ab553c743b3e11e87c5c7d423b2a2de229ba95b24a756344748011"
+checksum = "ab4609a838d88b73d8238967b60dd115cc08d38e2bbaf51ee1e4b695f89122e2"
 dependencies = [
  "autocfg",
  "cfg-if",
@@ -4625,18 +4639,18 @@ dependencies = [
 
 [[package]]
 name = "ref-cast"
-version = "1.0.12"
+version = "1.0.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "12a733f1746c929b4913fe48f8697fcf9c55e3304ba251a79ffb41adfeaf49c2"
+checksum = "53b15debb4f9d60d767cd8ca9ef7abb2452922f3214671ff052defc7f3502c44"
 dependencies = [
  "ref-cast-impl",
 ]
 
 [[package]]
 name = "ref-cast-impl"
-version = "1.0.12"
+version = "1.0.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5887de4a01acafd221861463be6113e6e87275e79804e56779f4cdc131c60368"
+checksum = "abfa8511e9e94fd3de6585a3d3cd00e01ed556dc9814829280af0e8dc72a8f36"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -4693,6 +4707,7 @@ dependencies = [
  "http",
  "http-body",
  "hyper",
+ "hyper-rustls 0.23.0",
  "hyper-tls",
  "ipnet",
  "js-sys",
@@ -4702,17 +4717,21 @@ dependencies = [
  "once_cell",
  "percent-encoding",
  "pin-project-lite",
+ "rustls 0.20.7",
+ "rustls-pemfile",
  "serde",
  "serde_json",
  "serde_urlencoded",
  "tokio",
  "tokio-native-tls",
+ "tokio-rustls 0.23.4",
  "tokio-socks",
  "tower-service",
  "url",
  "wasm-bindgen",
  "wasm-bindgen-futures",
  "web-sys",
+ "webpki-roots",
  "winreg 0.10.1",
 ]
 
@@ -4793,7 +4812,7 @@ dependencies = [
  "serde_json",
  "state",
  "tempfile",
- "time 0.3.15",
+ "time 0.3.16",
  "tokio",
  "tokio-stream",
  "tokio-util",
@@ -4840,7 +4859,7 @@ dependencies = [
  "smallvec",
  "stable-pattern",
  "state",
- "time 0.3.15",
+ "time 0.3.16",
  "tokio",
  "uncased",
 ]
@@ -4891,16 +4910,16 @@ dependencies = [
 
 [[package]]
 name = "rustix"
-version = "0.35.11"
+version = "0.35.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fbb2fda4666def1433b1b05431ab402e42a1084285477222b72d6c564c417cef"
+checksum = "727a1a6d65f786ec22df8a81ca3121107f235970dc1705ed681d3e6e8b9cd5f9"
 dependencies = [
  "bitflags",
  "errno",
  "io-lifetimes",
  "libc",
  "linux-raw-sys",
- "windows-sys 0.36.1",
+ "windows-sys 0.42.0",
 ]
 
 [[package]]
@@ -4912,8 +4931,20 @@ dependencies = [
  "base64 0.13.1",
  "log",
  "ring",
- "sct",
- "webpki",
+ "sct 0.6.1",
+ "webpki 0.21.4",
+]
+
+[[package]]
+name = "rustls"
+version = "0.20.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "539a2bfe908f471bfa933876bd1eb6a19cf2176d375f82ef7f99530a40e48c2c"
+dependencies = [
+ "log",
+ "ring",
+ "sct 0.7.0",
+ "webpki 0.22.0",
 ]
 
 [[package]]
@@ -4923,11 +4954,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5a07b7c1885bd8ed3831c289b7870b13ef46fe0e856d288c30d9cc17d75a2092"
 dependencies = [
  "openssl-probe",
- "rustls",
+ "rustls 0.19.1",
  "schannel",
  "security-framework",
 ]
 
+[[package]]
+name = "rustls-pemfile"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0864aeff53f8c05aa08d86e5ef839d3dfcf07aeba2db32f12db0ef716e87bd55"
+dependencies = [
+ "base64 0.13.1",
+]
+
 [[package]]
 name = "rustversion"
 version = "1.0.9"
@@ -4975,9 +5015,9 @@ dependencies = [
 
 [[package]]
 name = "scoped-tls"
-version = "1.0.0"
+version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ea6a9290e3c9cf0f18145ef7ffa62d68ee0bf5fcd651017e586dc7fd5da448c2"
+checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294"
 
 [[package]]
 name = "scopeguard"
@@ -5001,6 +5041,16 @@ dependencies = [
  "untrusted",
 ]
 
+[[package]]
+name = "sct"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4"
+dependencies = [
+ "ring",
+ "untrusted",
+]
+
 [[package]]
 name = "sec1"
 version = "0.3.0"
@@ -5046,9 +5096,9 @@ checksum = "e25dfac463d778e353db5be2449d1cce89bd6fd23c9f1ea21310ce6e5a1b29c4"
 
 [[package]]
 name = "serde"
-version = "1.0.146"
+version = "1.0.147"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6df50b7a60a0ad48e1b42eb38373eac8ff785d619fb14db917b4e63d5439361f"
+checksum = "d193d69bae983fc11a79df82342761dfbf28a99fc8d203dca4c3c1b590948965"
 dependencies = [
  "serde_derive",
 ]
@@ -5073,9 +5123,9 @@ dependencies = [
 
 [[package]]
 name = "serde_derive"
-version = "1.0.146"
+version = "1.0.147"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a714fd32ba1d66047ce7d53dabd809e9922d538f9047de13cc4cffca47b36205"
+checksum = "4f1d362ca8fc9c3e3a7484440752472d68a6caa98f1ab81d99b5dfe517cec852"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -5141,7 +5191,7 @@ dependencies = [
  "serde",
  "serde_json",
  "serde_with_macros 2.0.1",
- "time 0.3.15",
+ "time 0.3.16",
 ]
 
 [[package]]
@@ -5162,7 +5212,7 @@ version = "2.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1ccadfacf6cf10faad22bbadf55986bdd0856edfb5d9210aa1dcf1f516e84e93"
 dependencies = [
- "darling 0.14.1",
+ "darling 0.14.2",
  "proc-macro2",
  "quote",
  "syn",
@@ -5303,7 +5353,7 @@ dependencies = [
  "rand 0.8.5",
  "sha3 0.10.6",
  "thiserror",
- "time 0.3.15",
+ "time 0.3.16",
 ]
 
 [[package]]
@@ -5424,7 +5474,7 @@ dependencies = [
 [[package]]
 name = "ssi"
 version = "0.4.0"
-source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#fce1a8b031d21ba8abbb559e791adf4c5f1b3a68"
+source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#e5cf96b7efd899a080b8f2cc763fc39a434ce536"
 dependencies = [
  "ssi-caips",
  "ssi-core",
@@ -5445,7 +5495,7 @@ dependencies = [
 [[package]]
 name = "ssi-caips"
 version = "0.1.0"
-source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#fce1a8b031d21ba8abbb559e791adf4c5f1b3a68"
+source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#e5cf96b7efd899a080b8f2cc763fc39a434ce536"
 dependencies = [
  "bs58",
  "ssi-jwk",
@@ -5455,12 +5505,12 @@ dependencies = [
 [[package]]
 name = "ssi-contexts"
 version = "0.1.3"
-source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#fce1a8b031d21ba8abbb559e791adf4c5f1b3a68"
+source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#e5cf96b7efd899a080b8f2cc763fc39a434ce536"
 
 [[package]]
 name = "ssi-core"
 version = "0.1.0"
-source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#fce1a8b031d21ba8abbb559e791adf4c5f1b3a68"
+source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#e5cf96b7efd899a080b8f2cc763fc39a434ce536"
 dependencies = [
  "async-trait",
  "serde",
@@ -5470,7 +5520,7 @@ dependencies = [
 [[package]]
 name = "ssi-crypto"
 version = "0.1.0"
-source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#fce1a8b031d21ba8abbb559e791adf4c5f1b3a68"
+source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#e5cf96b7efd899a080b8f2cc763fc39a434ce536"
 dependencies = [
  "bs58",
  "digest 0.9.0",
@@ -5485,7 +5535,7 @@ dependencies = [
 [[package]]
 name = "ssi-dids"
 version = "0.1.0"
-source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#fce1a8b031d21ba8abbb559e791adf4c5f1b3a68"
+source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#e5cf96b7efd899a080b8f2cc763fc39a434ce536"
 dependencies = [
  "anyhow",
  "async-std",
@@ -5510,7 +5560,7 @@ dependencies = [
 [[package]]
 name = "ssi-json-ld"
 version = "0.1.0"
-source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#fce1a8b031d21ba8abbb559e791adf4c5f1b3a68"
+source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#e5cf96b7efd899a080b8f2cc763fc39a434ce536"
 dependencies = [
  "async-std",
  "chrono",
@@ -5530,7 +5580,7 @@ dependencies = [
 [[package]]
 name = "ssi-jwk"
 version = "0.1.0"
-source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#fce1a8b031d21ba8abbb559e791adf4c5f1b3a68"
+source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#e5cf96b7efd899a080b8f2cc763fc39a434ce536"
 dependencies = [
  "base64 0.12.3",
  "blake2b_simd 0.5.11",
@@ -5554,7 +5604,7 @@ dependencies = [
 [[package]]
 name = "ssi-jws"
 version = "0.1.0"
-source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#fce1a8b031d21ba8abbb559e791adf4c5f1b3a68"
+source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#e5cf96b7efd899a080b8f2cc763fc39a434ce536"
 dependencies = [
  "base64 0.12.3",
  "blake2",
@@ -5575,7 +5625,7 @@ dependencies = [
 [[package]]
 name = "ssi-jwt"
 version = "0.1.0"
-source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#fce1a8b031d21ba8abbb559e791adf4c5f1b3a68"
+source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#e5cf96b7efd899a080b8f2cc763fc39a434ce536"
 dependencies = [
  "chrono",
  "serde",
@@ -5588,12 +5638,13 @@ dependencies = [
 [[package]]
 name = "ssi-ldp"
 version = "0.1.0"
-source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#fce1a8b031d21ba8abbb559e791adf4c5f1b3a68"
+source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#e5cf96b7efd899a080b8f2cc763fc39a434ce536"
 dependencies = [
  "async-trait",
  "bs58",
  "chrono",
  "hex",
+ "k256",
  "lazy_static",
  "multibase 0.8.0",
  "serde",
@@ -5613,7 +5664,7 @@ dependencies = [
 [[package]]
 name = "ssi-ssh"
 version = "0.1.0"
-source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#fce1a8b031d21ba8abbb559e791adf4c5f1b3a68"
+source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#e5cf96b7efd899a080b8f2cc763fc39a434ce536"
 dependencies = [
  "sshkeys",
  "ssi-jwk",
@@ -5623,7 +5674,7 @@ dependencies = [
 [[package]]
 name = "ssi-tzkey"
 version = "0.1.0"
-source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#fce1a8b031d21ba8abbb559e791adf4c5f1b3a68"
+source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#e5cf96b7efd899a080b8f2cc763fc39a434ce536"
 dependencies = [
  "bs58",
  "ed25519-dalek",
@@ -5635,7 +5686,7 @@ dependencies = [
 [[package]]
 name = "ssi-ucan"
 version = "0.1.0"
-source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#fce1a8b031d21ba8abbb559e791adf4c5f1b3a68"
+source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#e5cf96b7efd899a080b8f2cc763fc39a434ce536"
 dependencies = [
  "base64 0.12.3",
  "chrono",
@@ -5655,7 +5706,7 @@ dependencies = [
 [[package]]
 name = "ssi-vc"
 version = "0.1.0"
-source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#fce1a8b031d21ba8abbb559e791adf4c5f1b3a68"
+source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#e5cf96b7efd899a080b8f2cc763fc39a434ce536"
 dependencies = [
  "async-trait",
  "base64 0.12.3",
@@ -5682,7 +5733,7 @@ dependencies = [
 [[package]]
 name = "ssi-zcap-ld"
 version = "0.1.0"
-source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#fce1a8b031d21ba8abbb559e791adf4c5f1b3a68"
+source = "git+https://github.com/spruceid/ssi?branch=feat/ucan-ipld-version#e5cf96b7efd899a080b8f2cc763fc39a434ce536"
 dependencies = [
  "async-trait",
  "serde",
@@ -5874,22 +5925,32 @@ dependencies = [
 
 [[package]]
 name = "time"
-version = "0.3.15"
+version = "0.3.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d634a985c4d4238ec39cacaed2e7ae552fbd3c476b552c1deac3021b7d7eaf0c"
+checksum = "0fab5c8b9980850e06d92ddbe3ab839c062c801f3927c0fb8abd6fc8e918fbca"
 dependencies = [
  "itoa",
  "libc",
  "num_threads",
  "serde",
+ "time-core",
  "time-macros",
 ]
 
+[[package]]
+name = "time-core"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd"
+
 [[package]]
 name = "time-macros"
-version = "0.2.4"
+version = "0.2.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "42657b1a6f4d817cda8e7a0ace261fe0cc946cf3a80314390b22cc61ae080792"
+checksum = "65bb801831d812c562ae7d2bfb531f26e66e4e1f6b17307ba4149c5064710e5b"
+dependencies = [
+ "time-core",
+]
 
 [[package]]
 name = "tiny-keccak"
@@ -5961,9 +6022,20 @@ version = "0.22.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6"
 dependencies = [
- "rustls",
+ "rustls 0.19.1",
+ "tokio",
+ "webpki 0.21.4",
+]
+
+[[package]]
+name = "tokio-rustls"
+version = "0.23.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59"
+dependencies = [
+ "rustls 0.20.7",
  "tokio",
- "webpki",
+ "webpki 0.22.0",
 ]
 
 [[package]]
@@ -6495,6 +6567,25 @@ dependencies = [
  "untrusted",
 ]
 
+[[package]]
+name = "webpki"
+version = "0.22.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd"
+dependencies = [
+ "ring",
+ "untrusted",
+]
+
+[[package]]
+name = "webpki-roots"
+version = "0.22.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "368bfe657969fb01238bb756d351dcade285e0f6fcbd36dcb23359a5169975be"
+dependencies = [
+ "webpki 0.22.0",
+]
+
 [[package]]
 name = "wepoll-ffi"
 version = "0.1.2"
diff --git a/lib/Cargo.toml b/lib/Cargo.toml
index a73a96dd..2877f251 100644
--- a/lib/Cargo.toml
+++ b/lib/Cargo.toml
@@ -18,7 +18,7 @@ lazy_static = "1.4"
 did-method-key = { default-features = false, git = "https://github.com/spruceid/ssi", branch = "feat/ucan-ipld-version" }
 did-tz = { default-features = false, git = "https://github.com/spruceid/ssi", branch = "feat/ucan-ipld-version" }
 did-ethr = { default-features = false, git = "https://github.com/spruceid/ssi", branch = "feat/ucan-ipld-version" }
-did-pkh = { default-features = false, git = "https://github.com/spruceid/ssi", branch = "feat/ucan-ipld-version" }
+did-pkh = { git = "https://github.com/spruceid/ssi", branch = "feat/ucan-ipld-version" }
 did-web = { default-features = false, git = "https://github.com/spruceid/ssi", branch = "feat/ucan-ipld-version" }
 did-webkey = { default-features = false, git = "https://github.com/spruceid/ssi", branch = "feat/ucan-ipld-version" }
 did-onion = { default-features = false, git = "https://github.com/spruceid/ssi", branch = "feat/ucan-ipld-version" }
@@ -34,7 +34,7 @@ uuid = { version = "1", features = ["v4"] }
 git = "https://github.com/spruceid/ssi"
 branch = "feat/ucan-ipld-version"
 default-features = false
-features = ["ed25519", "rsa"]
+features = ["ed25519", "rsa", "secp256k1"]
 
 [dev-dependencies.tokio]
 version = "1"
diff --git a/src/orbit.rs b/src/orbit.rs
index d2323de6..ece9934d 100644
--- a/src/orbit.rs
+++ b/src/orbit.rs
@@ -59,7 +59,7 @@ pub struct OrbitPeerConfig<B, I = config::IndexStorage> {
     identity: Ed25519Keypair,
     #[builder(setter(into))]
     manifest: Manifest,
-    #[builder(setter(into))]
+    #[builder(setter(into, strip_option), default)]
     relay: Option<(PeerId, Multiaddr)>,
     #[builder(setter(into))]
     blocks: B,
@@ -237,31 +237,34 @@ pub fn hash_same<B: AsRef<[u8]>>(c: &Cid, b: B) -> Result<Cid> {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::{
+        config::{IndexStorage, LocalIndexStorage},
+        BlockConfig, FileSystemConfig,
+    };
     use kepler_lib::resolver::DID_METHODS;
     use kepler_lib::ssi::{
         did::{Source, DIDURL},
         jwk::JWK,
     };
     use std::convert::TryInto;
+    use tempfile::{tempdir, TempDir};
 
-    async fn op(_md: Manifest) -> anyhow::Result<Orbit<BlockStores>> {
-        // let dir = Tempfile::new(&md.id().get_cid().to_string())
-        //     .unwrap()
-        //     .path()
-        //     .to_path_buf();
-        // let config = config::Config {
-        //     storage: config::Storage {
-        //         blocks: config::BlockStorage::Local(config::LocalBlockStorage {
-        //             path: dir.clone(),
-        //         }),
-        //         indexes: config::IndexStorage::Local(config::LocalIndexStorage {
-        //             path: dir.clone(),
-        //         }),
-        //     },
-        //     ..Default::default()
-        // };
-        // Orbit::new(&config, Ed25519Keypair::generate(), md, None).await
-        todo!()
+    async fn op(md: Manifest) -> anyhow::Result<(Orbit<BlockStores>, TempDir)> {
+        let dir = tempdir()?;
+        Ok((
+            Orbit::create(
+                &OrbitPeerConfigBuilder::<BlockConfig, IndexStorage>::default()
+                    .identity(Ed25519Keypair::generate())
+                    .manifest(md)
+                    .blocks(BlockConfig::B(FileSystemConfig::new(dir.path())))
+                    .index(IndexStorage::Local(LocalIndexStorage {
+                        path: dir.path().into(),
+                    }))
+                    .build()?,
+            )
+            .await?,
+            dir,
+        ))
     }
 
     #[test]
@@ -281,6 +284,6 @@ mod tests {
 
         let md = Manifest::resolve_dyn(&oid, None).await.unwrap().unwrap();
 
-        let _orbit = op(md).await.unwrap();
+        let (orbit, dir) = op(md).await.unwrap();
     }
 }
diff --git a/src/storage/file_system.rs b/src/storage/file_system.rs
index 64ccb27d..5da30658 100644
--- a/src/storage/file_system.rs
+++ b/src/storage/file_system.rs
@@ -16,7 +16,7 @@ use libp2p::identity::{ed25519::Keypair as Ed25519Keypair, error::DecodingError}
 use serde::{Deserialize, Serialize};
 use std::{
     io::{Error as IoError, ErrorKind},
-    path::PathBuf,
+    path::{Path, PathBuf},
 };
 use tempfile::{NamedTempFile, PersistError};
 use tokio::fs::{create_dir_all, read, remove_file, write, File};
@@ -42,6 +42,17 @@ pub struct FileSystemConfig {
     path: PathBuf,
 }
 
+impl FileSystemConfig {
+    pub fn new<P: AsRef<Path>>(p: P) -> Self {
+        Self {
+            path: p.as_ref().into(),
+        }
+    }
+    pub fn path(&self) -> &Path {
+        &self.path
+    }
+}
+
 #[async_trait]
 impl StorageConfig<FileSystemStore> for FileSystemConfig {
     type Error = IoError;

From 0d5750369015669d8486b3b0cdd656d40eeddb43 Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Fri, 4 Nov 2022 14:09:43 +0100
Subject: [PATCH 47/73] update cached

---
 Cargo.lock | 11 ++++++-----
 Cargo.toml |  2 +-
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 77095d89..2e1b69a4 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -994,9 +994,9 @@ dependencies = [
 
 [[package]]
 name = "cached"
-version = "0.34.1"
+version = "0.40.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "12f5cd208ba696f870238022d81ca1d80ed9d696fd62341c747f2d8f6ecdd9fe"
+checksum = "72b4147cd94d5fbdc2ab71b11d50a2f45493625576b3bb70257f59eedea69f3d"
 dependencies = [
  "async-trait",
  "async_once",
@@ -1004,6 +1004,7 @@ dependencies = [
  "cached_proc_macro_types",
  "futures",
  "hashbrown 0.12.3",
+ "instant",
  "lazy_static",
  "once_cell",
  "thiserror",
@@ -1012,9 +1013,9 @@ dependencies = [
 
 [[package]]
 name = "cached_proc_macro"
-version = "0.12.0"
+version = "0.15.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bce0f37f9b77c6b93cdf3f060c89adca303d2ab052cacb3c3d1ab543e8cecd2f"
+checksum = "751f7f4e7a091545e7f6c65bacc404eaee7e87bfb1f9ece234a1caa173dc16f2"
 dependencies = [
  "cached_proc_macro_types",
  "darling 0.13.4",
@@ -2824,7 +2825,7 @@ dependencies = [
  "base64 0.13.1",
  "bincode",
  "bs58",
- "cached 0.34.1",
+ "cached 0.40.0",
  "derive_builder 0.11.2",
  "futures",
  "hex",
diff --git a/Cargo.toml b/Cargo.toml
index 481b7d28..ceeaa895 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -18,7 +18,7 @@ aws-smithy-http = "0.49"
 base64 = "0.13"
 bincode = "1.3"
 bs58 = "0.4"
-cached = "0.34"
+cached = "0.40"
 derive_builder = "0.11"
 futures = { default-features = false, version = "0.3", features = ["alloc", "std"] }
 hex = "0.4"

From d26f35919f0499280bfecf28b2a8c65f8b44f40f Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Fri, 4 Nov 2022 17:22:08 +0100
Subject: [PATCH 48/73] add hash code choice to store write, add to all usage

---
 src/capabilities/store.rs  |  24 +++++--
 src/kv/store.rs            |  15 +++--
 src/storage/either.rs      |  10 ++-
 src/storage/file_system.rs |  23 +++----
 src/storage/mod.rs         |   9 ++-
 src/storage/s3.rs          |  20 ++----
 src/storage/utils.rs       | 133 +++++++++++++++++++++++++++++++++++--
 7 files changed, 186 insertions(+), 48 deletions(-)

diff --git a/src/capabilities/store.rs b/src/capabilities/store.rs
index ac8a48f6..febea6ac 100644
--- a/src/capabilities/store.rs
+++ b/src/capabilities/store.rs
@@ -104,7 +104,9 @@ where
 
         let cid = Cid::new_v1(
             eb_block.cid().codec(),
-            self.blocks.write(eb_block.data()).await?,
+            self.blocks
+                .write(eb_block.data(), eb_block.cid().hash().code().try_into()?)
+                .await?,
         );
 
         // write element indexes
@@ -115,9 +117,13 @@ where
                 .await?;
             tracing::debug!("applied delegation {:?}", d.1.block.cid());
             // put delegation block (encoded ucan or cacao)
-            self.blocks.write(d.1.block.data()).await?;
+            self.blocks
+                .write(d.1.block.data(), d.1.block.cid().hash().code().try_into()?)
+                .await?;
             // put link block
-            self.blocks.write(d.0.block.data()).await?;
+            self.blocks
+                .write(d.0.block.data(), d.0.block.cid().hash().code().try_into()?)
+                .await?;
             Result::<()>::Ok(())
         }))
         .await?;
@@ -132,9 +138,13 @@ where
                 .await?;
             tracing::debug!("applied revocation {:?}", r.1.block.cid());
             // put revocation block (encoded ucan revocation or cacao)
-            self.blocks.write(r.1.block.data()).await?;
+            self.blocks
+                .write(r.1.block.data(), r.1.block.cid().hash().code().try_into()?)
+                .await?;
             // put link block
-            self.blocks.write(r.0.block.data()).await?;
+            self.blocks
+                .write(r.0.block.data(), r.0.block.cid().hash().code().try_into()?)
+                .await?;
             Result::<()>::Ok(())
         }))
         .await?;
@@ -206,7 +216,9 @@ where
         let eb_block = eb.to_block()?;
         let cid = Cid::new_v1(
             eb_block.cid().codec(),
-            self.blocks.write(eb_block.data()).await?,
+            self.blocks
+                .write(eb_block.data(), eb_block.cid().hash().code().try_into()?)
+                .await?,
         );
 
         for e in event.invoke.iter() {
diff --git a/src/kv/store.rs b/src/kv/store.rs
index 9608c026..953bc52d 100644
--- a/src/kv/store.rs
+++ b/src/kv/store.rs
@@ -6,7 +6,7 @@ use futures::{
     io::AsyncRead,
     stream::{self, StreamExt, TryStreamExt},
 };
-use kepler_lib::libipld::{cid::Cid, multibase::Base, DagCbor};
+use kepler_lib::libipld::{cid::Cid, multibase::Base, multihash::Code, DagCbor};
 use rocket::futures::future::try_join_all;
 use std::{collections::BTreeMap, convert::TryFrom};
 use tracing::instrument;
@@ -208,10 +208,15 @@ where
         let indexes: Vec<(Vec<u8>, Cid)> = try_join_all(add.into_iter().map(|(o, r)| async {
             // tracing::debug!("adding {:#?}", &o.key);
             // store aaalllllll the content bytes under 1 CID
-            let cid = Cid::new_v1(0x55, self.blocks.write(r).await?);
+            let cid = Cid::new_v1(0x55, self.blocks.write(r, Code::Blake3_256).await?);
             let obj = o.add_content(cid);
             let block = obj.to_block()?;
-            let obj_cid = Cid::new_v1(block.cid().codec(), self.blocks.write(block.data()).await?);
+            let obj_cid = Cid::new_v1(
+                block.cid().codec(),
+                self.blocks
+                    .write(block.data(), block.cid().hash().code().try_into()?)
+                    .await?,
+            );
             Ok((obj.key, obj_cid)) as Result<(Vec<u8>, Cid)>
         }))
         .await?
@@ -322,7 +327,9 @@ where
         self.heads
             .new_heads([*block.cid()], delta.prev.clone())
             .await?;
-        self.blocks.write(block.data()).await?;
+        self.blocks
+            .write(block.data(), block.cid().hash().code().try_into()?)
+            .await?;
 
         Ok(())
     }
diff --git a/src/storage/either.rs b/src/storage/either.rs
index 43a77e5d..46706d39 100644
--- a/src/storage/either.rs
+++ b/src/storage/either.rs
@@ -7,7 +7,10 @@ use futures::{
     io::{AsyncRead, Error},
     task::{Context, Poll},
 };
-use kepler_lib::{libipld::cid::multihash::Multihash, resource::OrbitId};
+use kepler_lib::{
+    libipld::cid::multihash::{Code, Multihash},
+    resource::OrbitId,
+};
 use libp2p::identity::ed25519::Keypair as Ed25519Keypair;
 use pin_project::pin_project;
 
@@ -82,10 +85,11 @@ where
     async fn write(
         &self,
         data: impl futures::io::AsyncRead + Send,
+        hash_type: Code,
     ) -> Result<Multihash, Self::Error> {
         match self {
-            Self::A(l) => l.write(data).await.map_err(Self::Error::A),
-            Self::B(r) => r.write(data).await.map_err(Self::Error::B),
+            Self::A(l) => l.write(data, hash_type).await.map_err(Self::Error::A),
+            Self::B(r) => r.write(data, hash_type).await.map_err(Self::Error::B),
         }
     }
     async fn remove(&self, id: &Multihash) -> Result<Option<()>, Self::Error> {
diff --git a/src/storage/file_system.rs b/src/storage/file_system.rs
index 5da30658..38e5b5ff 100644
--- a/src/storage/file_system.rs
+++ b/src/storage/file_system.rs
@@ -1,14 +1,12 @@
 use crate::{
     orbit::ProviderUtils,
-    storage::{utils::HashBuffer, ImmutableStore, StorageConfig},
+    storage::{utils::copy_in, ImmutableStore, StorageConfig},
 };
-use futures::io::{copy, AllowStdIo, AsyncWriteExt};
+use futures::io::AllowStdIo;
 use kepler_lib::{
     libipld::cid::{
         multibase::{encode, Base},
-        multihash::{
-            Blake3Hasher, Code, Error as MultihashError, Hasher, Multihash, MultihashDigest,
-        },
+        multihash::{Code, Error as MultihashError, Multihash},
     },
     resource::OrbitId,
 };
@@ -147,17 +145,14 @@ impl ImmutableStore for FileSystemStore {
     async fn write(
         &self,
         data: impl futures::io::AsyncRead + Send,
+        hash_type: Code,
     ) -> Result<Multihash, Self::Error> {
-        let mut hb = HashBuffer::<Blake3Hasher<32>, AllowStdIo<NamedTempFile>>::new(
+        let (multihash, file) = copy_in(
+            data,
             AllowStdIo::new(NamedTempFile::new_in(&self.path)?),
-        );
-
-        copy(data, &mut hb).await?;
-        hb.flush().await?;
-
-        let (mut hasher, file) = hb.into_inner();
-
-        let multihash = Code::Blake3_256.wrap(hasher.finalize())?;
+            hash_type,
+        )
+        .await?;
 
         let file = file.into_inner();
         file.persist(self.get_path(&multihash))?;
diff --git a/src/storage/mod.rs b/src/storage/mod.rs
index ce57619c..bd13a568 100644
--- a/src/storage/mod.rs
+++ b/src/storage/mod.rs
@@ -1,5 +1,8 @@
 use anyhow::{Error, Result};
-use kepler_lib::libipld::cid::{multihash::Multihash, Cid};
+use kepler_lib::libipld::cid::{
+    multihash::{Code, Multihash},
+    Cid,
+};
 use kepler_lib::resource::OrbitId;
 use std::{collections::HashMap, error::Error as StdError};
 
@@ -37,6 +40,7 @@ pub trait ImmutableStore: Send + Sync {
     async fn write(
         &self,
         data: impl futures::io::AsyncRead + Send,
+        hash_type: Code,
     ) -> Result<Multihash, Self::Error>;
     async fn remove(&self, id: &Multihash) -> Result<Option<()>, Self::Error>;
     async fn read(&self, id: &Multihash) -> Result<Option<Self::Readable>, Self::Error>;
@@ -80,8 +84,9 @@ where
     async fn write(
         &self,
         data: impl futures::io::AsyncRead + Send,
+        hash_type: Code,
     ) -> Result<Multihash, Self::Error> {
-        self.write(data).await
+        self.write(data, hash_type).await
     }
     async fn remove(&self, id: &Multihash) -> Result<Option<()>, Self::Error> {
         self.remove(id).await
diff --git a/src/storage/s3.rs b/src/storage/s3.rs
index 3da24262..feec6f97 100644
--- a/src/storage/s3.rs
+++ b/src/storage/s3.rs
@@ -8,15 +8,13 @@ use aws_sdk_s3::{
 };
 use aws_smithy_http::{body::SdkBody, byte_stream::Error as ByteStreamError, endpoint::Endpoint};
 use futures::{
-    io::{copy, AllowStdIo, AsyncWriteExt},
+    io::AllowStdIo,
     stream::{IntoAsyncRead, MapErr, TryStreamExt},
 };
 use kepler_lib::{
     libipld::cid::{
         multibase::{encode, Base},
-        multihash::{
-            Blake3Hasher, Code, Error as MultihashError, Hasher, Multihash, MultihashDigest,
-        },
+        multihash::{Code, Error as MultihashError, Multihash},
         Cid,
     },
     resource::OrbitId,
@@ -30,7 +28,7 @@ use tempfile::NamedTempFile;
 
 use crate::{
     orbit::ProviderUtils,
-    storage::{utils::HashBuffer, ImmutableStore, StorageConfig},
+    storage::{utils::copy_in, ImmutableStore, StorageConfig},
 };
 
 // TODO we could use the same struct for both the block store and the data
@@ -237,17 +235,11 @@ impl ImmutableStore for S3BlockStore {
     async fn write(
         &self,
         data: impl futures::io::AsyncRead + Send,
+        hash_type: Code,
     ) -> Result<Multihash, Self::Error> {
         // TODO find a way to do this without filesystem access
-        let mut hb = HashBuffer::<Blake3Hasher<32>, AllowStdIo<NamedTempFile>>::new(
-            AllowStdIo::new(NamedTempFile::new()?),
-        );
-
-        copy(data, &mut hb).await?;
-        hb.flush().await?;
-
-        let (mut hasher, file) = hb.into_inner();
-        let multihash = Code::Blake3_256.wrap(hasher.finalize())?;
+        let (multihash, file) =
+            copy_in(data, AllowStdIo::new(NamedTempFile::new()?), hash_type).await?;
 
         let (_, path) = file.into_inner().into_parts();
 
diff --git a/src/storage/utils.rs b/src/storage/utils.rs
index 5f04dcd6..24938407 100644
--- a/src/storage/utils.rs
+++ b/src/storage/utils.rs
@@ -2,11 +2,16 @@ use aws_types::sdk_config::SdkConfig;
 use core::pin::Pin;
 use futures::{
     executor::block_on,
-    io::{AsyncWrite, Error},
+    io::{copy, AsyncRead, AsyncWrite, AsyncWriteExt},
     task::{Context, Poll},
 };
-use libipld::cid::multihash::Hasher;
+use libipld::cid::multihash::{
+    Blake2b256, Blake2b512, Blake2s128, Blake2s256, Blake3_256, Code, Error as MultihashError,
+    Hasher, Keccak224, Keccak256, Keccak384, Keccak512, Multihash, MultihashDigest, Sha2_256,
+    Sha2_512, Sha3_224, Sha3_256, Sha3_384, Sha3_512,
+};
 use pin_project::pin_project;
+use std::io::Error as IoError;
 
 pub fn aws_config() -> SdkConfig {
     block_on(async { aws_config::from_env().load().await })
@@ -38,6 +43,124 @@ where
     }
 }
 
+pub async fn copy_in<B>(
+    data: impl AsyncRead,
+    buffer: B,
+    hash_type: Code,
+) -> Result<(Multihash, B), MultihashError>
+where
+    B: AsyncWrite + Unpin,
+{
+    Ok(match hash_type {
+        Code::Sha2_256 => {
+            let mut hb = HashBuffer::<Sha2_256, B>::new(buffer);
+            copy(data, &mut hb).await?;
+            hb.flush().await?;
+            let (mut h, b) = hb.into_inner();
+            (hash_type.wrap(h.finalize())?, b)
+        }
+        Code::Sha2_512 => {
+            let mut hb = HashBuffer::<Sha2_512, B>::new(buffer);
+            copy(data, &mut hb).await?;
+            hb.flush().await?;
+            let (mut h, b) = hb.into_inner();
+            (hash_type.wrap(h.finalize())?, b)
+        }
+        Code::Sha3_224 => {
+            let mut hb = HashBuffer::<Sha3_224, B>::new(buffer);
+            copy(data, &mut hb).await?;
+            hb.flush().await?;
+            let (mut h, b) = hb.into_inner();
+            (hash_type.wrap(h.finalize())?, b)
+        }
+        Code::Sha3_256 => {
+            let mut hb = HashBuffer::<Sha3_256, B>::new(buffer);
+            copy(data, &mut hb).await?;
+            hb.flush().await?;
+            let (mut h, b) = hb.into_inner();
+            (hash_type.wrap(h.finalize())?, b)
+        }
+        Code::Sha3_384 => {
+            let mut hb = HashBuffer::<Sha3_384, B>::new(buffer);
+            copy(data, &mut hb).await?;
+            hb.flush().await?;
+            let (mut h, b) = hb.into_inner();
+            (hash_type.wrap(h.finalize())?, b)
+        }
+        Code::Sha3_512 => {
+            let mut hb = HashBuffer::<Sha3_512, B>::new(buffer);
+            copy(data, &mut hb).await?;
+            hb.flush().await?;
+            let (mut h, b) = hb.into_inner();
+            (hash_type.wrap(h.finalize())?, b)
+        }
+        Code::Keccak224 => {
+            let mut hb = HashBuffer::<Keccak224, B>::new(buffer);
+            copy(data, &mut hb).await?;
+            hb.flush().await?;
+            let (mut h, b) = hb.into_inner();
+            (hash_type.wrap(h.finalize())?, b)
+        }
+        Code::Keccak256 => {
+            let mut hb = HashBuffer::<Keccak256, B>::new(buffer);
+            copy(data, &mut hb).await?;
+            hb.flush().await?;
+            let (mut h, b) = hb.into_inner();
+            (hash_type.wrap(h.finalize())?, b)
+        }
+        Code::Keccak384 => {
+            let mut hb = HashBuffer::<Keccak384, B>::new(buffer);
+            copy(data, &mut hb).await?;
+            hb.flush().await?;
+            let (mut h, b) = hb.into_inner();
+            (hash_type.wrap(h.finalize())?, b)
+        }
+        Code::Keccak512 => {
+            let mut hb = HashBuffer::<Keccak512, B>::new(buffer);
+            copy(data, &mut hb).await?;
+            hb.flush().await?;
+            let (mut h, b) = hb.into_inner();
+            (hash_type.wrap(h.finalize())?, b)
+        }
+        Code::Blake2b256 => {
+            let mut hb = HashBuffer::<Blake2b256, B>::new(buffer);
+            copy(data, &mut hb).await?;
+            hb.flush().await?;
+            let (mut h, b) = hb.into_inner();
+            (hash_type.wrap(h.finalize())?, b)
+        }
+        Code::Blake2b512 => {
+            let mut hb = HashBuffer::<Blake2b512, B>::new(buffer);
+            copy(data, &mut hb).await?;
+            hb.flush().await?;
+            let (mut h, b) = hb.into_inner();
+            (hash_type.wrap(h.finalize())?, b)
+        }
+        Code::Blake2s128 => {
+            let mut hb = HashBuffer::<Blake2s128, B>::new(buffer);
+            copy(data, &mut hb).await?;
+            hb.flush().await?;
+            let (mut h, b) = hb.into_inner();
+            (hash_type.wrap(h.finalize())?, b)
+        }
+        Code::Blake2s256 => {
+            let mut hb = HashBuffer::<Blake2s256, B>::new(buffer);
+            copy(data, &mut hb).await?;
+            hb.flush().await?;
+            let (mut h, b) = hb.into_inner();
+            (hash_type.wrap(h.finalize())?, b)
+        }
+        Code::Blake3_256 => {
+            let mut hb = HashBuffer::<Blake3_256, B>::new(buffer);
+            copy(data, &mut hb).await?;
+            hb.flush().await?;
+            let (mut h, b) = hb.into_inner();
+            (hash_type.wrap(h.finalize())?, b)
+        }
+        c => return Err(MultihashError::UnsupportedCode(c.into())),
+    })
+}
+
 impl<H, B> AsyncWrite for HashBuffer<H, B>
 where
     H: Hasher,
@@ -47,15 +170,15 @@ where
         self: Pin<&mut Self>,
         cx: &mut Context<'_>,
         buf: &[u8],
-    ) -> Poll<Result<usize, Error>> {
+    ) -> Poll<Result<usize, IoError>> {
         let p = self.project();
         p.hasher.update(buf);
         p.buffer.poll_write(cx, buf)
     }
-    fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), Error>> {
+    fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), IoError>> {
         self.project().buffer.poll_flush(cx)
     }
-    fn poll_close(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), Error>> {
+    fn poll_close(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), IoError>> {
         self.project().buffer.poll_close(cx)
     }
 }

From 125abd894645de24ad71e0fa1c6d64c27156f931 Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Fri, 4 Nov 2022 18:09:32 +0100
Subject: [PATCH 49/73] code golf with macros

---
 src/storage/utils.rs | 130 ++++++++-----------------------------------
 1 file changed, 22 insertions(+), 108 deletions(-)

diff --git a/src/storage/utils.rs b/src/storage/utils.rs
index 24938407..bfa73323 100644
--- a/src/storage/utils.rs
+++ b/src/storage/utils.rs
@@ -43,6 +43,23 @@ where
     }
 }
 
+macro_rules! write_with_multihash {
+    ($data:ident, $buffer:ident, $hash:ident) => {{
+        let mut hb = HashBuffer::<$hash, B>::new($buffer);
+        copy($data, &mut hb).await?;
+        hb.flush().await?;
+        let (mut h, b) = hb.into_inner();
+        Ok((Code::$hash.wrap(h.finalize())?, b))
+    }};
+
+    ($data:ident, $buffer:ident, $code:ident, $($hashes:ident),*) => {
+        match $code {
+            $(Code::$hashes => write_with_multihash!($data, $buffer, $hashes),)*
+            c => Err(MultihashError::UnsupportedCode(c.into()))
+        }
+    };
+}
+
 pub async fn copy_in<B>(
     data: impl AsyncRead,
     buffer: B,
@@ -51,114 +68,11 @@ pub async fn copy_in<B>(
 where
     B: AsyncWrite + Unpin,
 {
-    Ok(match hash_type {
-        Code::Sha2_256 => {
-            let mut hb = HashBuffer::<Sha2_256, B>::new(buffer);
-            copy(data, &mut hb).await?;
-            hb.flush().await?;
-            let (mut h, b) = hb.into_inner();
-            (hash_type.wrap(h.finalize())?, b)
-        }
-        Code::Sha2_512 => {
-            let mut hb = HashBuffer::<Sha2_512, B>::new(buffer);
-            copy(data, &mut hb).await?;
-            hb.flush().await?;
-            let (mut h, b) = hb.into_inner();
-            (hash_type.wrap(h.finalize())?, b)
-        }
-        Code::Sha3_224 => {
-            let mut hb = HashBuffer::<Sha3_224, B>::new(buffer);
-            copy(data, &mut hb).await?;
-            hb.flush().await?;
-            let (mut h, b) = hb.into_inner();
-            (hash_type.wrap(h.finalize())?, b)
-        }
-        Code::Sha3_256 => {
-            let mut hb = HashBuffer::<Sha3_256, B>::new(buffer);
-            copy(data, &mut hb).await?;
-            hb.flush().await?;
-            let (mut h, b) = hb.into_inner();
-            (hash_type.wrap(h.finalize())?, b)
-        }
-        Code::Sha3_384 => {
-            let mut hb = HashBuffer::<Sha3_384, B>::new(buffer);
-            copy(data, &mut hb).await?;
-            hb.flush().await?;
-            let (mut h, b) = hb.into_inner();
-            (hash_type.wrap(h.finalize())?, b)
-        }
-        Code::Sha3_512 => {
-            let mut hb = HashBuffer::<Sha3_512, B>::new(buffer);
-            copy(data, &mut hb).await?;
-            hb.flush().await?;
-            let (mut h, b) = hb.into_inner();
-            (hash_type.wrap(h.finalize())?, b)
-        }
-        Code::Keccak224 => {
-            let mut hb = HashBuffer::<Keccak224, B>::new(buffer);
-            copy(data, &mut hb).await?;
-            hb.flush().await?;
-            let (mut h, b) = hb.into_inner();
-            (hash_type.wrap(h.finalize())?, b)
-        }
-        Code::Keccak256 => {
-            let mut hb = HashBuffer::<Keccak256, B>::new(buffer);
-            copy(data, &mut hb).await?;
-            hb.flush().await?;
-            let (mut h, b) = hb.into_inner();
-            (hash_type.wrap(h.finalize())?, b)
-        }
-        Code::Keccak384 => {
-            let mut hb = HashBuffer::<Keccak384, B>::new(buffer);
-            copy(data, &mut hb).await?;
-            hb.flush().await?;
-            let (mut h, b) = hb.into_inner();
-            (hash_type.wrap(h.finalize())?, b)
-        }
-        Code::Keccak512 => {
-            let mut hb = HashBuffer::<Keccak512, B>::new(buffer);
-            copy(data, &mut hb).await?;
-            hb.flush().await?;
-            let (mut h, b) = hb.into_inner();
-            (hash_type.wrap(h.finalize())?, b)
-        }
-        Code::Blake2b256 => {
-            let mut hb = HashBuffer::<Blake2b256, B>::new(buffer);
-            copy(data, &mut hb).await?;
-            hb.flush().await?;
-            let (mut h, b) = hb.into_inner();
-            (hash_type.wrap(h.finalize())?, b)
-        }
-        Code::Blake2b512 => {
-            let mut hb = HashBuffer::<Blake2b512, B>::new(buffer);
-            copy(data, &mut hb).await?;
-            hb.flush().await?;
-            let (mut h, b) = hb.into_inner();
-            (hash_type.wrap(h.finalize())?, b)
-        }
-        Code::Blake2s128 => {
-            let mut hb = HashBuffer::<Blake2s128, B>::new(buffer);
-            copy(data, &mut hb).await?;
-            hb.flush().await?;
-            let (mut h, b) = hb.into_inner();
-            (hash_type.wrap(h.finalize())?, b)
-        }
-        Code::Blake2s256 => {
-            let mut hb = HashBuffer::<Blake2s256, B>::new(buffer);
-            copy(data, &mut hb).await?;
-            hb.flush().await?;
-            let (mut h, b) = hb.into_inner();
-            (hash_type.wrap(h.finalize())?, b)
-        }
-        Code::Blake3_256 => {
-            let mut hb = HashBuffer::<Blake3_256, B>::new(buffer);
-            copy(data, &mut hb).await?;
-            hb.flush().await?;
-            let (mut h, b) = hb.into_inner();
-            (hash_type.wrap(h.finalize())?, b)
-        }
-        c => return Err(MultihashError::UnsupportedCode(c.into())),
-    })
+    write_with_multihash!(
+        data, buffer, hash_type, Sha2_256, Sha2_512, Sha3_224, Sha3_256, Sha3_384, Sha3_512,
+        Keccak224, Keccak256, Keccak384, Keccak512, Blake2b256, Blake2b512, Blake2s128, Blake2s256,
+        Blake3_256
+    )
 }
 
 impl<H, B> AsyncWrite for HashBuffer<H, B>

From 46c109fc1677cd8b2b1f042fafcffd3956f2f134 Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Fri, 4 Nov 2022 18:18:01 +0100
Subject: [PATCH 50/73] remove unnecessary extra macro case

---
 src/storage/utils.rs | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/src/storage/utils.rs b/src/storage/utils.rs
index bfa73323..f04c5594 100644
--- a/src/storage/utils.rs
+++ b/src/storage/utils.rs
@@ -44,17 +44,15 @@ where
 }
 
 macro_rules! write_with_multihash {
-    ($data:ident, $buffer:ident, $hash:ident) => {{
-        let mut hb = HashBuffer::<$hash, B>::new($buffer);
-        copy($data, &mut hb).await?;
-        hb.flush().await?;
-        let (mut h, b) = hb.into_inner();
-        Ok((Code::$hash.wrap(h.finalize())?, b))
-    }};
-
     ($data:ident, $buffer:ident, $code:ident, $($hashes:ident),*) => {
         match $code {
-            $(Code::$hashes => write_with_multihash!($data, $buffer, $hashes),)*
+            $(Code::$hashes => {
+                let mut hb = HashBuffer::<$hashes, B>::new($buffer);
+                copy($data, &mut hb).await?;
+                hb.flush().await?;
+                let (mut h, b) = hb.into_inner();
+                Ok((Code::$hashes.wrap(h.finalize())?, b))
+            },)*
             c => Err(MultihashError::UnsupportedCode(c.into()))
         }
     };

From 549ca13e7a563d7af8776cab6cdeefcf1f40f17f Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Thu, 10 Nov 2022 10:44:29 +0100
Subject: [PATCH 51/73] remove unnecessary AsyncReadEither type, use
 future::Either

---
 src/storage/either.rs | 53 ++++---------------------------------------
 1 file changed, 4 insertions(+), 49 deletions(-)

diff --git a/src/storage/either.rs b/src/storage/either.rs
index 46706d39..ef55bf55 100644
--- a/src/storage/either.rs
+++ b/src/storage/either.rs
@@ -2,17 +2,12 @@ use crate::{
     orbit::ProviderUtils,
     storage::{ImmutableStore, StorageConfig},
 };
-use core::pin::Pin;
-use futures::{
-    io::{AsyncRead, Error},
-    task::{Context, Poll},
-};
+use futures::future::Either as AsyncReadEither;
 use kepler_lib::{
     libipld::cid::multihash::{Code, Multihash},
     resource::OrbitId,
 };
 use libp2p::identity::ed25519::Keypair as Ed25519Keypair;
-use pin_project::pin_project;
 
 #[derive(Debug, Clone, Hash, PartialEq, Eq)]
 pub enum Either<A, B> {
@@ -20,46 +15,6 @@ pub enum Either<A, B> {
     B(B),
 }
 
-#[pin_project(project = AsyncReadEitherProjection)]
-#[derive(Debug, Clone)]
-pub enum AsyncReadEither<A, B>
-where
-    A: ImmutableStore,
-    B: ImmutableStore,
-{
-    A(#[pin] A::Readable),
-    B(#[pin] B::Readable),
-}
-
-impl<A, B> AsyncRead for AsyncReadEither<A, B>
-where
-    A: ImmutableStore,
-    B: ImmutableStore,
-{
-    #[inline]
-    fn poll_read(
-        self: Pin<&mut Self>,
-        cx: &mut Context<'_>,
-        buf: &mut [u8],
-    ) -> Poll<Result<usize, Error>> {
-        match self.project() {
-            AsyncReadEitherProjection::A(a) => a.poll_read(cx, buf),
-            AsyncReadEitherProjection::B(b) => b.poll_read(cx, buf),
-        }
-    }
-    #[inline]
-    fn poll_read_vectored(
-        self: Pin<&mut Self>,
-        cx: &mut Context<'_>,
-        bufs: &mut [std::io::IoSliceMut<'_>],
-    ) -> Poll<Result<usize, Error>> {
-        match self.project() {
-            AsyncReadEitherProjection::A(a) => a.poll_read_vectored(cx, bufs),
-            AsyncReadEitherProjection::B(b) => b.poll_read_vectored(cx, bufs),
-        }
-    }
-}
-
 #[derive(thiserror::Error, Debug)]
 pub enum EitherError<A, B> {
     #[error(transparent)]
@@ -74,7 +29,7 @@ where
     A: ImmutableStore,
     B: ImmutableStore,
 {
-    type Readable = AsyncReadEither<A, B>;
+    type Readable = AsyncReadEither<A::Readable, B::Readable>;
     type Error = EitherError<A::Error, B::Error>;
     async fn contains(&self, id: &Multihash) -> Result<bool, Self::Error> {
         match self {
@@ -103,12 +58,12 @@ where
             Self::A(l) => l
                 .read(id)
                 .await
-                .map(|o| o.map(Self::Readable::A))
+                .map(|o| o.map(Self::Readable::Left))
                 .map_err(Self::Error::A),
             Self::B(r) => r
                 .read(id)
                 .await
-                .map(|o| o.map(Self::Readable::B))
+                .map(|o| o.map(Self::Readable::Right))
                 .map_err(Self::Error::B),
         }
     }

From f19f8b7ed89981ec51ed65742616f43f4622b86a Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Thu, 3 Nov 2022 18:27:58 +0100
Subject: [PATCH 52/73] impl a basic NetworkBehaviour and config objects

---
 Cargo.lock       |  77 +++++++++++++++++++++++++++--------
 Cargo.toml       |   2 +-
 src/behaviour.rs | 104 +++++++++++++++++++++++++++++++++++++++++++++++
 src/lib.rs       |   1 +
 4 files changed, 167 insertions(+), 17 deletions(-)
 create mode 100644 src/behaviour.rs

diff --git a/Cargo.lock b/Cargo.lock
index 2e1b69a4..b964bee4 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1400,17 +1400,6 @@ dependencies = [
  "cipher 0.4.3",
 ]
 
-[[package]]
-name = "cuckoofilter"
-version = "0.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b810a8449931679f64cd7eef1bbd0fa315801b6d5d9cdc1ace2804d6529eee18"
-dependencies = [
- "byteorder",
- "fnv",
- "rand 0.7.3",
-]
-
 [[package]]
 name = "curve25519-dalek"
 version = "3.2.0"
@@ -2426,6 +2415,12 @@ version = "0.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
 
+[[package]]
+name = "hex_fmt"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b07f60793ff0a4d9cef0f18e63b5357e06209987153a64648c972c1e5aff336f"
+
 [[package]]
 name = "hkdf"
 version = "0.12.3"
@@ -3063,8 +3058,9 @@ dependencies = [
  "instant",
  "lazy_static",
  "libp2p-core",
+ "libp2p-dcutr",
  "libp2p-dns",
- "libp2p-floodsub",
+ "libp2p-gossipsub",
  "libp2p-identify",
  "libp2p-kad",
  "libp2p-metrics",
@@ -3115,6 +3111,28 @@ dependencies = [
  "zeroize",
 ]
 
+[[package]]
+name = "libp2p-dcutr"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "380846ac24e641d38bfdb317cd77cbbb923527ce2eb8f71bce0b107c7249e2a9"
+dependencies = [
+ "asynchronous-codec",
+ "bytes",
+ "either",
+ "futures",
+ "futures-timer",
+ "instant",
+ "libp2p-core",
+ "libp2p-swarm",
+ "log",
+ "prost 0.11.0",
+ "prost-build 0.11.1",
+ "prost-codec",
+ "thiserror",
+ "void",
+]
+
 [[package]]
 name = "libp2p-dns"
 version = "0.37.0"
@@ -3130,21 +3148,31 @@ dependencies = [
 ]
 
 [[package]]
-name = "libp2p-floodsub"
-version = "0.40.1"
+name = "libp2p-gossipsub"
+version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e88a413aac86cd30078192e5c110fd4c859ba36a6d694e040e75f1ef0dc41583"
+checksum = "a7b8d02b9089241196479c6fdae22df4d4444509edb3a8e7ef388218ca9b5e3e"
 dependencies = [
- "cuckoofilter",
+ "asynchronous-codec",
+ "base64 0.13.1",
+ "byteorder",
+ "bytes",
  "fnv",
  "futures",
+ "hex_fmt",
+ "instant",
  "libp2p-core",
  "libp2p-swarm",
  "log",
+ "prometheus-client",
  "prost 0.11.0",
  "prost-build 0.11.1",
  "rand 0.8.5",
+ "regex",
+ "sha2 0.10.6",
  "smallvec",
+ "unsigned-varint",
+ "wasm-timer",
 ]
 
 [[package]]
@@ -3203,6 +3231,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9ee31b08e78b7b8bfd1c4204a9dd8a87b4fcdf6dafc57eb51701c1c264a81cb9"
 dependencies = [
  "libp2p-core",
+ "libp2p-dcutr",
+ "libp2p-gossipsub",
  "libp2p-identify",
  "libp2p-kad",
  "libp2p-ping",
@@ -6548,6 +6578,21 @@ version = "0.2.83"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f"
 
+[[package]]
+name = "wasm-timer"
+version = "0.2.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "be0ecb0db480561e9a7642b5d3e4187c128914e58aa84330b9493e3eb68c5e7f"
+dependencies = [
+ "futures",
+ "js-sys",
+ "parking_lot 0.11.2",
+ "pin-utils",
+ "wasm-bindgen",
+ "wasm-bindgen-futures",
+ "web-sys",
+]
+
 [[package]]
 name = "web-sys"
 version = "0.3.60"
diff --git a/Cargo.toml b/Cargo.toml
index ceeaa895..83b00635 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -26,7 +26,7 @@ hyper = "0.14" # Prometheus server
 iri-string = "0.5"
 lazy_static = "1.4.0"
 libipld = "0.14"
-libp2p = { default-features = false, features = ["floodsub", "identify", "kad", "tcp-tokio", "mplex", "noise", "ping", "yamux", "dns-tokio", "relay"], version = "0.49.0" }
+libp2p = { default-features = false, features = ["gossipsub", "identify", "kad", "tcp-tokio", "mplex", "noise", "ping", "yamux", "dns-tokio", "relay", "dcutr"], version = "0.49.0" }
 opentelemetry = { version = "0.17.0", features = ["rt-tokio"] }
 opentelemetry-jaeger = { version = "0.16.0", features = ["rt-tokio", "reqwest_collector_client"] }
 pin-project = "1"
diff --git a/src/behaviour.rs b/src/behaviour.rs
new file mode 100644
index 00000000..82bf4d82
--- /dev/null
+++ b/src/behaviour.rs
@@ -0,0 +1,104 @@
+use core::time::Duration;
+use derive_builder::Builder;
+use libp2p::{
+    dcutr::behaviour::Behaviour as DcutrBehaviour,
+    gossipsub::{
+        Gossipsub, GossipsubConfig, GossipsubConfigBuilder, MessageAuthenticity, ValidationMode,
+    },
+    identify::{Behaviour as Identify, Config as OIdentifyConfig},
+    identity::{Keypair, PublicKey},
+    kad::{
+        record::store::{MemoryStore, MemoryStoreConfig},
+        Kademlia, KademliaConfig,
+    },
+    ping::{Behaviour as Ping, Config as PingConfig},
+    relay::v2::client::Client,
+    swarm::behaviour::toggle::Toggle,
+    NetworkBehaviour,
+};
+use thiserror::Error;
+
+const PROTOCOL_VERSION: &'static str = "kepler/0.1.0";
+
+#[derive(Builder, Default, Debug)]
+pub struct IdentifyConfig {
+    #[builder(setter(into), default = "Duration::from_millis(500)")]
+    initial_delay: Duration,
+    #[builder(setter(into), default = "Duration::from_secs(300)")]
+    interval: Duration,
+    #[builder(setter(into), default = "false")]
+    push_listen_addr_updates: bool,
+    #[builder(setter(into), default = "0")]
+    cache_size: usize,
+}
+
+impl IdentifyConfig {
+    pub fn to_config(self, key: PublicKey) -> OIdentifyConfig {
+        OIdentifyConfig::new(PROTOCOL_VERSION.to_string(), key)
+            .with_initial_delay(self.initial_delay)
+            .with_interval(self.interval)
+            .with_push_listen_addr_updates(self.push_listen_addr_updates)
+            .with_cache_size(self.cache_size)
+    }
+}
+
+#[derive(Builder)]
+#[builder(pattern = "owned")]
+pub struct OrbitNodeConfig {
+    #[builder(setter(into))]
+    identity: Keypair,
+    #[builder(setter(into), default)]
+    identify: IdentifyConfig,
+    #[builder(setter(into), default)]
+    ping: PingConfig,
+    #[builder(setter(into), default)]
+    gossipsub: GossipsubConfig,
+    #[builder(setter(into), default)]
+    kademlia: KademliaConfig,
+    #[builder(setter(into), default)]
+    kademlia_store: MemoryStoreConfig,
+    #[builder(setter(into, strip_option), default)]
+    relay: Option<Client>,
+}
+
+#[derive(NetworkBehaviour)]
+pub struct OrbitNode {
+    identify: Identify,
+    ping: Ping,
+    gossipsub: Gossipsub,
+    relay: Toggle<Client>,
+    kademlia: Kademlia<MemoryStore>,
+    dcutr: DcutrBehaviour,
+}
+
+#[derive(Error, Debug)]
+pub enum OrbitNodeInitError {
+    #[error("{0}")]
+    Gossipsub(&'static str),
+}
+
+impl OrbitNode {
+    pub fn new(c: OrbitNodeConfig) -> Result<Self, OrbitNodeInitError> {
+        let peer_id = c.identity.public().to_peer_id();
+        Ok(Self {
+            identify: Identify::new(c.identify.to_config(c.identity.public())),
+            ping: Ping::new(c.ping),
+            gossipsub: Gossipsub::new(
+                MessageAuthenticity::Signed(c.identity),
+                GossipsubConfigBuilder::from(c.gossipsub)
+                    // always ensure validation
+                    .validation_mode(ValidationMode::Strict)
+                    .build()
+                    .map_err(OrbitNodeInitError::Gossipsub)?,
+            )
+            .map_err(OrbitNodeInitError::Gossipsub)?,
+            relay: c.relay.into(),
+            kademlia: Kademlia::with_config(
+                peer_id,
+                MemoryStore::with_config(peer_id, c.kademlia_store),
+                c.kademlia,
+            ),
+            dcutr: DcutrBehaviour::new(),
+        })
+    }
+}
diff --git a/src/lib.rs b/src/lib.rs
index b099ca39..b98fb6a0 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -13,6 +13,7 @@ use rocket::{fairing::AdHoc, figment::Figment, http::Header, Build, Rocket};
 pub mod allow_list;
 pub mod auth_guards;
 pub mod authorization;
+pub mod behaviour;
 pub mod capabilities;
 pub mod cas;
 pub mod config;

From cc06cecee82d0e5d39dd9d54424d379219534a38 Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Thu, 3 Nov 2022 19:23:09 +0100
Subject: [PATCH 53/73] move relay out of config

---
 src/behaviour.rs | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/src/behaviour.rs b/src/behaviour.rs
index 82bf4d82..88594a33 100644
--- a/src/behaviour.rs
+++ b/src/behaviour.rs
@@ -1,6 +1,7 @@
 use core::time::Duration;
 use derive_builder::Builder;
 use libp2p::{
+    core::PeerId,
     dcutr::behaviour::Behaviour as DcutrBehaviour,
     gossipsub::{
         Gossipsub, GossipsubConfig, GossipsubConfigBuilder, MessageAuthenticity, ValidationMode,
@@ -57,8 +58,6 @@ pub struct OrbitNodeConfig {
     kademlia: KademliaConfig,
     #[builder(setter(into), default)]
     kademlia_store: MemoryStoreConfig,
-    #[builder(setter(into, strip_option), default)]
-    relay: Option<Client>,
 }
 
 #[derive(NetworkBehaviour)]
@@ -92,13 +91,23 @@ impl OrbitNode {
                     .map_err(OrbitNodeInitError::Gossipsub)?,
             )
             .map_err(OrbitNodeInitError::Gossipsub)?,
-            relay: c.relay.into(),
             kademlia: Kademlia::with_config(
                 peer_id,
                 MemoryStore::with_config(peer_id, c.kademlia_store),
                 c.kademlia,
             ),
+            relay: None.into(),
             dcutr: DcutrBehaviour::new(),
         })
     }
+
+    pub fn new_with_relay(
+        c: OrbitNodeConfig,
+        relay_client: Client,
+    ) -> Result<Self, OrbitNodeInitError> {
+        Ok(Self {
+            relay: Some(relay_client).into(),
+            ..Self::new(c)?
+        })
+    }
 }

From 01c56ba953036fe0255f7d1b7f6acce836cbf8d0 Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Thu, 3 Nov 2022 19:23:19 +0100
Subject: [PATCH 54/73] make generic over kad store and kad store config

---
 src/behaviour.rs | 54 ++++++++++++++++++++++++++++++++++--------------
 1 file changed, 39 insertions(+), 15 deletions(-)

diff --git a/src/behaviour.rs b/src/behaviour.rs
index 88594a33..5c2a0bf0 100644
--- a/src/behaviour.rs
+++ b/src/behaviour.rs
@@ -9,7 +9,7 @@ use libp2p::{
     identify::{Behaviour as Identify, Config as OIdentifyConfig},
     identity::{Keypair, PublicKey},
     kad::{
-        record::store::{MemoryStore, MemoryStoreConfig},
+        record::store::{MemoryStore, MemoryStoreConfig, RecordStore},
         Kademlia, KademliaConfig,
     },
     ping::{Behaviour as Ping, Config as PingConfig},
@@ -45,7 +45,10 @@ impl IdentifyConfig {
 
 #[derive(Builder)]
 #[builder(pattern = "owned")]
-pub struct OrbitNodeConfig {
+pub struct OrbitNodeConfig<KSC = MemoryStoreConfig>
+where
+    KSC: Default,
+{
     #[builder(setter(into))]
     identity: Keypair,
     #[builder(setter(into), default)]
@@ -57,16 +60,19 @@ pub struct OrbitNodeConfig {
     #[builder(setter(into), default)]
     kademlia: KademliaConfig,
     #[builder(setter(into), default)]
-    kademlia_store: MemoryStoreConfig,
+    kademlia_store: KSC,
 }
 
 #[derive(NetworkBehaviour)]
-pub struct OrbitNode {
+pub struct OrbitNode<KS>
+where
+    KS: 'static + for<'a> RecordStore<'a> + Send,
+{
     identify: Identify,
     ping: Ping,
     gossipsub: Gossipsub,
     relay: Toggle<Client>,
-    kademlia: Kademlia<MemoryStore>,
+    kademlia: Kademlia<KS>,
     dcutr: DcutrBehaviour,
 }
 
@@ -76,8 +82,14 @@ pub enum OrbitNodeInitError {
     Gossipsub(&'static str),
 }
 
-impl OrbitNode {
-    pub fn new(c: OrbitNodeConfig) -> Result<Self, OrbitNodeInitError> {
+impl<KS> OrbitNode<KS>
+where
+    KS: 'static + for<'a> RecordStore<'a> + Send,
+{
+    pub fn new<KSC>(c: OrbitNodeConfig<KSC>) -> Result<Self, OrbitNodeInitError>
+    where
+        KSC: RecordStoreConfig<KS> + Default,
+    {
         let peer_id = c.identity.public().to_peer_id();
         Ok(Self {
             identify: Identify::new(c.identify.to_config(c.identity.public())),
@@ -91,23 +103,35 @@ impl OrbitNode {
                     .map_err(OrbitNodeInitError::Gossipsub)?,
             )
             .map_err(OrbitNodeInitError::Gossipsub)?,
-            kademlia: Kademlia::with_config(
-                peer_id,
-                MemoryStore::with_config(peer_id, c.kademlia_store),
-                c.kademlia,
-            ),
             relay: None.into(),
+            kademlia: Kademlia::with_config(peer_id, c.kademlia_store.init(peer_id), c.kademlia),
             dcutr: DcutrBehaviour::new(),
         })
     }
 
-    pub fn new_with_relay(
-        c: OrbitNodeConfig,
+    pub fn new_with_relay<KSC>(
+        c: OrbitNodeConfig<KSC>,
         relay_client: Client,
-    ) -> Result<Self, OrbitNodeInitError> {
+    ) -> Result<Self, OrbitNodeInitError>
+    where
+        KSC: RecordStoreConfig<KS> + Default,
+    {
         Ok(Self {
             relay: Some(relay_client).into(),
             ..Self::new(c)?
         })
     }
 }
+
+pub trait RecordStoreConfig<S>
+where
+    S: for<'a> RecordStore<'a>,
+{
+    fn init(self, id: PeerId) -> S;
+}
+
+impl RecordStoreConfig<MemoryStore> for MemoryStoreConfig {
+    fn init(self, id: PeerId) -> MemoryStore {
+        MemoryStore::with_config(id, self)
+    }
+}

From 7b959e13577e086d5510180040aad4ba22216114 Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Thu, 3 Nov 2022 20:24:58 +0100
Subject: [PATCH 55/73] extra config stuff, add swarm type

---
 src/behaviour.rs | 48 ++++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 42 insertions(+), 6 deletions(-)

diff --git a/src/behaviour.rs b/src/behaviour.rs
index 5c2a0bf0..8fe0c43f 100644
--- a/src/behaviour.rs
+++ b/src/behaviour.rs
@@ -1,7 +1,7 @@
 use core::time::Duration;
 use derive_builder::Builder;
 use libp2p::{
-    core::PeerId,
+    core::{muxing::StreamMuxerBox, transport::Boxed, PeerId},
     dcutr::behaviour::Behaviour as DcutrBehaviour,
     gossipsub::{
         Gossipsub, GossipsubConfig, GossipsubConfigBuilder, MessageAuthenticity, ValidationMode,
@@ -14,13 +14,15 @@ use libp2p::{
     },
     ping::{Behaviour as Ping, Config as PingConfig},
     relay::v2::client::Client,
-    swarm::behaviour::toggle::Toggle,
+    swarm::{behaviour::toggle::Toggle, Swarm},
     NetworkBehaviour,
 };
 use thiserror::Error;
 
 const PROTOCOL_VERSION: &'static str = "kepler/0.1.0";
 
+pub type OrbitSwarm<KS = MemoryStore> = Swarm<OrbitNodeBehaviour<KS>>;
+
 #[derive(Builder, Default, Debug)]
 pub struct IdentifyConfig {
     #[builder(setter(into), default = "Duration::from_millis(500)")]
@@ -63,8 +65,42 @@ where
     kademlia_store: KSC,
 }
 
+impl<KSC> OrbitNodeConfig<KSC>
+where
+    KSC: Default,
+{
+    fn init_behaviour<KS>(self) -> Result<OrbitNodeBehaviour<KS>, OrbitNodeInitError>
+    where
+        KSC: RecordStoreConfig<KS> + Default,
+        KS: for<'a> RecordStore<'a> + Send,
+    {
+        OrbitNodeBehaviour::new(self)
+    }
+    fn init_behaviour_with_relay<KS>(
+        self,
+        relay: Client,
+    ) -> Result<OrbitNodeBehaviour<KS>, OrbitNodeInitError>
+    where
+        KSC: RecordStoreConfig<KS> + Default,
+        KS: for<'a> RecordStore<'a> + Send,
+    {
+        OrbitNodeBehaviour::new_with_relay(self, relay)
+    }
+    pub fn init<KS>(
+        self,
+        transport: Boxed<(PeerId, StreamMuxerBox)>,
+    ) -> Result<OrbitSwarm<KS>, OrbitNodeInitError>
+    where
+        KSC: RecordStoreConfig<KS> + Default,
+        KS: for<'a> RecordStore<'a> + Send,
+    {
+        let peer_id = self.identity.public().to_peer_id();
+        Ok(OrbitSwarm::new(transport, self.init_behaviour()?, peer_id))
+    }
+}
+
 #[derive(NetworkBehaviour)]
-pub struct OrbitNode<KS>
+pub struct OrbitNodeBehaviour<KS>
 where
     KS: 'static + for<'a> RecordStore<'a> + Send,
 {
@@ -82,11 +118,11 @@ pub enum OrbitNodeInitError {
     Gossipsub(&'static str),
 }
 
-impl<KS> OrbitNode<KS>
+impl<KS> OrbitNodeBehaviour<KS>
 where
     KS: 'static + for<'a> RecordStore<'a> + Send,
 {
-    pub fn new<KSC>(c: OrbitNodeConfig<KSC>) -> Result<Self, OrbitNodeInitError>
+    fn new<KSC>(c: OrbitNodeConfig<KSC>) -> Result<Self, OrbitNodeInitError>
     where
         KSC: RecordStoreConfig<KS> + Default,
     {
@@ -109,7 +145,7 @@ where
         })
     }
 
-    pub fn new_with_relay<KSC>(
+    fn new_with_relay<KSC>(
         c: OrbitNodeConfig<KSC>,
         relay_client: Client,
     ) -> Result<Self, OrbitNodeInitError>

From 0f2523fc603654fb889d01a42d824b8d26cbf78d Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Fri, 4 Nov 2022 12:54:01 +0100
Subject: [PATCH 56/73] IdentifyConfig conveniance changes

---
 src/behaviour.rs | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/src/behaviour.rs b/src/behaviour.rs
index 8fe0c43f..b700386a 100644
--- a/src/behaviour.rs
+++ b/src/behaviour.rs
@@ -23,7 +23,7 @@ const PROTOCOL_VERSION: &'static str = "kepler/0.1.0";
 
 pub type OrbitSwarm<KS = MemoryStore> = Swarm<OrbitNodeBehaviour<KS>>;
 
-#[derive(Builder, Default, Debug)]
+#[derive(Builder, Default, Debug, Clone)]
 pub struct IdentifyConfig {
     #[builder(setter(into), default = "Duration::from_millis(500)")]
     initial_delay: Duration,
@@ -36,7 +36,7 @@ pub struct IdentifyConfig {
 }
 
 impl IdentifyConfig {
-    pub fn to_config(self, key: PublicKey) -> OIdentifyConfig {
+    fn to_config(self, key: PublicKey) -> OIdentifyConfig {
         OIdentifyConfig::new(PROTOCOL_VERSION.to_string(), key)
             .with_initial_delay(self.initial_delay)
             .with_interval(self.interval)
@@ -45,8 +45,18 @@ impl IdentifyConfig {
     }
 }
 
+impl From<OIdentifyConfig> for IdentifyConfig {
+    fn from(c: OIdentifyConfig) -> Self {
+        Self {
+            initial_delay: c.initial_delay,
+            interval: c.interval,
+            push_listen_addr_updates: c.push_listen_addr_updates,
+            cache_size: c.cache_size,
+        }
+    }
+}
+
 #[derive(Builder)]
-#[builder(pattern = "owned")]
 pub struct OrbitNodeConfig<KSC = MemoryStoreConfig>
 where
     KSC: Default,

From eebd4d14278746473c58e915b6267298291cecb8 Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Fri, 4 Nov 2022 13:33:50 +0100
Subject: [PATCH 57/73] renames and streamline the builder process

---
 src/behaviour.rs | 221 ++++++++++++++++++++---------------------------
 1 file changed, 93 insertions(+), 128 deletions(-)

diff --git a/src/behaviour.rs b/src/behaviour.rs
index b700386a..f4a16960 100644
--- a/src/behaviour.rs
+++ b/src/behaviour.rs
@@ -2,7 +2,7 @@ use core::time::Duration;
 use derive_builder::Builder;
 use libp2p::{
     core::{muxing::StreamMuxerBox, transport::Boxed, PeerId},
-    dcutr::behaviour::Behaviour as DcutrBehaviour,
+    dcutr::behaviour::Behaviour as Dcutr,
     gossipsub::{
         Gossipsub, GossipsubConfig, GossipsubConfigBuilder, MessageAuthenticity, ValidationMode,
     },
@@ -21,96 +21,75 @@ use thiserror::Error;
 
 const PROTOCOL_VERSION: &'static str = "kepler/0.1.0";
 
-pub type OrbitSwarm<KS = MemoryStore> = Swarm<OrbitNodeBehaviour<KS>>;
+pub type OrbitSwarm<KS = MemoryStore> = Swarm<Behaviour<KS>>;
 
-#[derive(Builder, Default, Debug, Clone)]
-pub struct IdentifyConfig {
-    #[builder(setter(into), default = "Duration::from_millis(500)")]
-    initial_delay: Duration,
-    #[builder(setter(into), default = "Duration::from_secs(300)")]
-    interval: Duration,
-    #[builder(setter(into), default = "false")]
-    push_listen_addr_updates: bool,
-    #[builder(setter(into), default = "0")]
-    cache_size: usize,
-}
-
-impl IdentifyConfig {
-    fn to_config(self, key: PublicKey) -> OIdentifyConfig {
-        OIdentifyConfig::new(PROTOCOL_VERSION.to_string(), key)
-            .with_initial_delay(self.initial_delay)
-            .with_interval(self.interval)
-            .with_push_listen_addr_updates(self.push_listen_addr_updates)
-            .with_cache_size(self.cache_size)
-    }
-}
-
-impl From<OIdentifyConfig> for IdentifyConfig {
-    fn from(c: OIdentifyConfig) -> Self {
-        Self {
-            initial_delay: c.initial_delay,
-            interval: c.interval,
-            push_listen_addr_updates: c.push_listen_addr_updates,
-            cache_size: c.cache_size,
-        }
-    }
-}
-
-#[derive(Builder)]
-pub struct OrbitNodeConfig<KSC = MemoryStoreConfig>
-where
-    KSC: Default,
-{
-    #[builder(setter(into))]
-    identity: Keypair,
-    #[builder(setter(into), default)]
-    identify: IdentifyConfig,
-    #[builder(setter(into), default)]
-    ping: PingConfig,
-    #[builder(setter(into), default)]
-    gossipsub: GossipsubConfig,
-    #[builder(setter(into), default)]
-    kademlia: KademliaConfig,
-    #[builder(setter(into), default)]
-    kademlia_store: KSC,
-}
-
-impl<KSC> OrbitNodeConfig<KSC>
-where
-    KSC: Default,
-{
-    fn init_behaviour<KS>(self) -> Result<OrbitNodeBehaviour<KS>, OrbitNodeInitError>
+mod builder {
+    use super::*;
+    #[derive(Builder, Clone, Debug)]
+    #[builder(build_fn(skip), setter(into), name = "BehaviourBuilder", derive(Debug))]
+    pub struct BehaviourConfig<KSC = MemoryStoreConfig>
     where
-        KSC: RecordStoreConfig<KS> + Default,
-        KS: for<'a> RecordStore<'a> + Send,
+        KSC: Default,
     {
-        OrbitNodeBehaviour::new(self)
+        #[builder(field(type = "IdentifyConfig"))]
+        identify: IdentifyConfig,
+        #[builder(field(type = "PingConfig"))]
+        ping: PingConfig,
+        #[builder(field(type = "GossipsubConfig"))]
+        gossipsub: GossipsubConfig,
+        #[builder(field(type = "KademliaConfig"))]
+        kademlia: KademliaConfig,
+        #[builder(field(type = "KSC"))]
+        kademlia_store: KSC,
     }
-    fn init_behaviour_with_relay<KS>(
-        self,
-        relay: Client,
-    ) -> Result<OrbitNodeBehaviour<KS>, OrbitNodeInitError>
+
+    impl<KSC> BehaviourBuilder<KSC>
     where
-        KSC: RecordStoreConfig<KS> + Default,
-        KS: for<'a> RecordStore<'a> + Send,
+        KSC: Default,
     {
-        OrbitNodeBehaviour::new_with_relay(self, relay)
+        pub fn build<KS>(
+            self,
+            keypair: Keypair,
+            relay: Option<Client>,
+        ) -> Result<Behaviour<KS>, OrbitBehaviourBuildError>
+        where
+            KSC: Default + RecordStoreConfig<KS>,
+            KS: for<'a> RecordStore<'a> + Send,
+        {
+            let peer_id = keypair.public().to_peer_id();
+            Ok(Behaviour {
+                identify: Identify::new(self.identify.to_config(keypair.public())),
+                ping: Ping::new(self.ping),
+                gossipsub: Gossipsub::new(
+                    MessageAuthenticity::Signed(keypair),
+                    GossipsubConfigBuilder::from(self.gossipsub)
+                        // always ensure validation
+                        .validation_mode(ValidationMode::Strict)
+                        .build()
+                        .map_err(OrbitBehaviourBuildError::Gossipsub)?,
+                )
+                .map_err(OrbitBehaviourBuildError::Gossipsub)?,
+                relay: relay.into(),
+                kademlia: Kademlia::with_config(
+                    peer_id,
+                    self.kademlia_store.init(peer_id),
+                    self.kademlia,
+                ),
+                dcutr: Dcutr::new(),
+            })
+        }
     }
-    pub fn init<KS>(
-        self,
-        transport: Boxed<(PeerId, StreamMuxerBox)>,
-    ) -> Result<OrbitSwarm<KS>, OrbitNodeInitError>
-    where
-        KSC: RecordStoreConfig<KS> + Default,
-        KS: for<'a> RecordStore<'a> + Send,
-    {
-        let peer_id = self.identity.public().to_peer_id();
-        Ok(OrbitSwarm::new(transport, self.init_behaviour()?, peer_id))
+
+    #[derive(Error, Debug)]
+    pub enum OrbitBehaviourBuildError {
+        #[error("{0}")]
+        Gossipsub(&'static str),
     }
 }
+pub use builder::{BehaviourBuilder, OrbitBehaviourBuildError};
 
 #[derive(NetworkBehaviour)]
-pub struct OrbitNodeBehaviour<KS>
+pub struct Behaviour<KS>
 where
     KS: 'static + for<'a> RecordStore<'a> + Send,
 {
@@ -119,54 +98,7 @@ where
     gossipsub: Gossipsub,
     relay: Toggle<Client>,
     kademlia: Kademlia<KS>,
-    dcutr: DcutrBehaviour,
-}
-
-#[derive(Error, Debug)]
-pub enum OrbitNodeInitError {
-    #[error("{0}")]
-    Gossipsub(&'static str),
-}
-
-impl<KS> OrbitNodeBehaviour<KS>
-where
-    KS: 'static + for<'a> RecordStore<'a> + Send,
-{
-    fn new<KSC>(c: OrbitNodeConfig<KSC>) -> Result<Self, OrbitNodeInitError>
-    where
-        KSC: RecordStoreConfig<KS> + Default,
-    {
-        let peer_id = c.identity.public().to_peer_id();
-        Ok(Self {
-            identify: Identify::new(c.identify.to_config(c.identity.public())),
-            ping: Ping::new(c.ping),
-            gossipsub: Gossipsub::new(
-                MessageAuthenticity::Signed(c.identity),
-                GossipsubConfigBuilder::from(c.gossipsub)
-                    // always ensure validation
-                    .validation_mode(ValidationMode::Strict)
-                    .build()
-                    .map_err(OrbitNodeInitError::Gossipsub)?,
-            )
-            .map_err(OrbitNodeInitError::Gossipsub)?,
-            relay: None.into(),
-            kademlia: Kademlia::with_config(peer_id, c.kademlia_store.init(peer_id), c.kademlia),
-            dcutr: DcutrBehaviour::new(),
-        })
-    }
-
-    fn new_with_relay<KSC>(
-        c: OrbitNodeConfig<KSC>,
-        relay_client: Client,
-    ) -> Result<Self, OrbitNodeInitError>
-    where
-        KSC: RecordStoreConfig<KS> + Default,
-    {
-        Ok(Self {
-            relay: Some(relay_client).into(),
-            ..Self::new(c)?
-        })
-    }
+    dcutr: Dcutr,
 }
 
 pub trait RecordStoreConfig<S>
@@ -181,3 +113,36 @@ impl RecordStoreConfig<MemoryStore> for MemoryStoreConfig {
         MemoryStore::with_config(id, self)
     }
 }
+
+#[derive(Builder, Default, Debug, Clone)]
+pub struct IdentifyConfig {
+    #[builder(setter(into), default = "Duration::from_millis(500)")]
+    initial_delay: Duration,
+    #[builder(setter(into), default = "Duration::from_secs(300)")]
+    interval: Duration,
+    #[builder(setter(into), default = "false")]
+    push_listen_addr_updates: bool,
+    #[builder(setter(into), default = "0")]
+    cache_size: usize,
+}
+
+impl IdentifyConfig {
+    fn to_config(self, key: PublicKey) -> OIdentifyConfig {
+        OIdentifyConfig::new(PROTOCOL_VERSION.to_string(), key)
+            .with_initial_delay(self.initial_delay)
+            .with_interval(self.interval)
+            .with_push_listen_addr_updates(self.push_listen_addr_updates)
+            .with_cache_size(self.cache_size)
+    }
+}
+
+impl From<OIdentifyConfig> for IdentifyConfig {
+    fn from(c: OIdentifyConfig) -> Self {
+        Self {
+            initial_delay: c.initial_delay,
+            interval: c.interval,
+            push_listen_addr_updates: c.push_listen_addr_updates,
+            cache_size: c.cache_size,
+        }
+    }
+}

From 04ff83e987032d9ef0e2aa77592510c9f4b3921c Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Fri, 4 Nov 2022 13:44:00 +0100
Subject: [PATCH 58/73] move behaviour to p2p mod and split into sub-mods

---
 src/behaviour.rs             | 148 -----------------------------------
 src/lib.rs                   |   2 +-
 src/p2p/behaviour/builder.rs | 129 ++++++++++++++++++++++++++++++
 src/p2p/behaviour/mod.rs     |  40 ++++++++++
 src/p2p/mod.rs               |   1 +
 5 files changed, 171 insertions(+), 149 deletions(-)
 delete mode 100644 src/behaviour.rs
 create mode 100644 src/p2p/behaviour/builder.rs
 create mode 100644 src/p2p/behaviour/mod.rs
 create mode 100644 src/p2p/mod.rs

diff --git a/src/behaviour.rs b/src/behaviour.rs
deleted file mode 100644
index f4a16960..00000000
--- a/src/behaviour.rs
+++ /dev/null
@@ -1,148 +0,0 @@
-use core::time::Duration;
-use derive_builder::Builder;
-use libp2p::{
-    core::{muxing::StreamMuxerBox, transport::Boxed, PeerId},
-    dcutr::behaviour::Behaviour as Dcutr,
-    gossipsub::{
-        Gossipsub, GossipsubConfig, GossipsubConfigBuilder, MessageAuthenticity, ValidationMode,
-    },
-    identify::{Behaviour as Identify, Config as OIdentifyConfig},
-    identity::{Keypair, PublicKey},
-    kad::{
-        record::store::{MemoryStore, MemoryStoreConfig, RecordStore},
-        Kademlia, KademliaConfig,
-    },
-    ping::{Behaviour as Ping, Config as PingConfig},
-    relay::v2::client::Client,
-    swarm::{behaviour::toggle::Toggle, Swarm},
-    NetworkBehaviour,
-};
-use thiserror::Error;
-
-const PROTOCOL_VERSION: &'static str = "kepler/0.1.0";
-
-pub type OrbitSwarm<KS = MemoryStore> = Swarm<Behaviour<KS>>;
-
-mod builder {
-    use super::*;
-    #[derive(Builder, Clone, Debug)]
-    #[builder(build_fn(skip), setter(into), name = "BehaviourBuilder", derive(Debug))]
-    pub struct BehaviourConfig<KSC = MemoryStoreConfig>
-    where
-        KSC: Default,
-    {
-        #[builder(field(type = "IdentifyConfig"))]
-        identify: IdentifyConfig,
-        #[builder(field(type = "PingConfig"))]
-        ping: PingConfig,
-        #[builder(field(type = "GossipsubConfig"))]
-        gossipsub: GossipsubConfig,
-        #[builder(field(type = "KademliaConfig"))]
-        kademlia: KademliaConfig,
-        #[builder(field(type = "KSC"))]
-        kademlia_store: KSC,
-    }
-
-    impl<KSC> BehaviourBuilder<KSC>
-    where
-        KSC: Default,
-    {
-        pub fn build<KS>(
-            self,
-            keypair: Keypair,
-            relay: Option<Client>,
-        ) -> Result<Behaviour<KS>, OrbitBehaviourBuildError>
-        where
-            KSC: Default + RecordStoreConfig<KS>,
-            KS: for<'a> RecordStore<'a> + Send,
-        {
-            let peer_id = keypair.public().to_peer_id();
-            Ok(Behaviour {
-                identify: Identify::new(self.identify.to_config(keypair.public())),
-                ping: Ping::new(self.ping),
-                gossipsub: Gossipsub::new(
-                    MessageAuthenticity::Signed(keypair),
-                    GossipsubConfigBuilder::from(self.gossipsub)
-                        // always ensure validation
-                        .validation_mode(ValidationMode::Strict)
-                        .build()
-                        .map_err(OrbitBehaviourBuildError::Gossipsub)?,
-                )
-                .map_err(OrbitBehaviourBuildError::Gossipsub)?,
-                relay: relay.into(),
-                kademlia: Kademlia::with_config(
-                    peer_id,
-                    self.kademlia_store.init(peer_id),
-                    self.kademlia,
-                ),
-                dcutr: Dcutr::new(),
-            })
-        }
-    }
-
-    #[derive(Error, Debug)]
-    pub enum OrbitBehaviourBuildError {
-        #[error("{0}")]
-        Gossipsub(&'static str),
-    }
-}
-pub use builder::{BehaviourBuilder, OrbitBehaviourBuildError};
-
-#[derive(NetworkBehaviour)]
-pub struct Behaviour<KS>
-where
-    KS: 'static + for<'a> RecordStore<'a> + Send,
-{
-    identify: Identify,
-    ping: Ping,
-    gossipsub: Gossipsub,
-    relay: Toggle<Client>,
-    kademlia: Kademlia<KS>,
-    dcutr: Dcutr,
-}
-
-pub trait RecordStoreConfig<S>
-where
-    S: for<'a> RecordStore<'a>,
-{
-    fn init(self, id: PeerId) -> S;
-}
-
-impl RecordStoreConfig<MemoryStore> for MemoryStoreConfig {
-    fn init(self, id: PeerId) -> MemoryStore {
-        MemoryStore::with_config(id, self)
-    }
-}
-
-#[derive(Builder, Default, Debug, Clone)]
-pub struct IdentifyConfig {
-    #[builder(setter(into), default = "Duration::from_millis(500)")]
-    initial_delay: Duration,
-    #[builder(setter(into), default = "Duration::from_secs(300)")]
-    interval: Duration,
-    #[builder(setter(into), default = "false")]
-    push_listen_addr_updates: bool,
-    #[builder(setter(into), default = "0")]
-    cache_size: usize,
-}
-
-impl IdentifyConfig {
-    fn to_config(self, key: PublicKey) -> OIdentifyConfig {
-        OIdentifyConfig::new(PROTOCOL_VERSION.to_string(), key)
-            .with_initial_delay(self.initial_delay)
-            .with_interval(self.interval)
-            .with_push_listen_addr_updates(self.push_listen_addr_updates)
-            .with_cache_size(self.cache_size)
-    }
-}
-
-impl From<OIdentifyConfig> for IdentifyConfig {
-    fn from(c: OIdentifyConfig) -> Self {
-        Self {
-            initial_delay: c.initial_delay,
-            interval: c.interval,
-            push_listen_addr_updates: c.push_listen_addr_updates,
-            cache_size: c.cache_size,
-        }
-    }
-}
diff --git a/src/lib.rs b/src/lib.rs
index b98fb6a0..a8e30b53 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -13,7 +13,6 @@ use rocket::{fairing::AdHoc, figment::Figment, http::Header, Build, Rocket};
 pub mod allow_list;
 pub mod auth_guards;
 pub mod authorization;
-pub mod behaviour;
 pub mod capabilities;
 pub mod cas;
 pub mod config;
@@ -21,6 +20,7 @@ pub mod indexes;
 pub mod kv;
 pub mod manifest;
 pub mod orbit;
+pub mod p2p;
 pub mod prometheus;
 pub mod relay;
 pub mod routes;
diff --git a/src/p2p/behaviour/builder.rs b/src/p2p/behaviour/builder.rs
new file mode 100644
index 00000000..6617766e
--- /dev/null
+++ b/src/p2p/behaviour/builder.rs
@@ -0,0 +1,129 @@
+use core::time::Duration;
+use derive_builder::Builder;
+use libp2p::{
+    core::{muxing::StreamMuxerBox, transport::Boxed, PeerId},
+    dcutr::behaviour::Behaviour as Dcutr,
+    gossipsub::{
+        Gossipsub, GossipsubConfig, GossipsubConfigBuilder, MessageAuthenticity, ValidationMode,
+    },
+    identify::{Behaviour as Identify, Config as OIdentifyConfig},
+    identity::{Keypair, PublicKey},
+    kad::{
+        record::store::{MemoryStore, MemoryStoreConfig, RecordStore},
+        Kademlia, KademliaConfig,
+    },
+    ping::{Behaviour as Ping, Config as PingConfig},
+    relay::v2::client::Client,
+    swarm::{behaviour::toggle::Toggle, Swarm},
+    NetworkBehaviour,
+};
+use thiserror::Error;
+
+const PROTOCOL_VERSION: &'static str = "kepler/0.1.0";
+
+#[derive(Builder, Clone, Debug)]
+#[builder(build_fn(skip), setter(into), name = "BehaviourBuilder", derive(Debug))]
+pub struct BehaviourConfig<KSC = MemoryStoreConfig>
+where
+    KSC: Default,
+{
+    #[builder(field(type = "IdentifyConfig"))]
+    identify: IdentifyConfig,
+    #[builder(field(type = "PingConfig"))]
+    ping: PingConfig,
+    #[builder(field(type = "GossipsubConfig"))]
+    gossipsub: GossipsubConfig,
+    #[builder(field(type = "KademliaConfig"))]
+    kademlia: KademliaConfig,
+    #[builder(field(type = "KSC"))]
+    kademlia_store: KSC,
+}
+
+impl<KSC> BehaviourBuilder<KSC>
+where
+    KSC: Default,
+{
+    pub fn build<KS>(
+        self,
+        keypair: Keypair,
+        relay: Option<Client>,
+    ) -> Result<Behaviour<KS>, OrbitBehaviourBuildError>
+    where
+        KSC: Default + RecordStoreConfig<KS>,
+        KS: for<'a> RecordStore<'a> + Send,
+    {
+        let peer_id = keypair.public().to_peer_id();
+        Ok(Behaviour {
+            identify: Identify::new(self.identify.to_config(keypair.public())),
+            ping: Ping::new(self.ping),
+            gossipsub: Gossipsub::new(
+                MessageAuthenticity::Signed(keypair),
+                GossipsubConfigBuilder::from(self.gossipsub)
+                    // always ensure validation
+                    .validation_mode(ValidationMode::Strict)
+                    .build()
+                    .map_err(OrbitBehaviourBuildError::Gossipsub)?,
+            )
+            .map_err(OrbitBehaviourBuildError::Gossipsub)?,
+            relay: relay.into(),
+            kademlia: Kademlia::with_config(
+                peer_id,
+                self.kademlia_store.init(peer_id),
+                self.kademlia,
+            ),
+            dcutr: Dcutr::new(),
+        })
+    }
+}
+
+#[derive(Error, Debug)]
+pub enum OrbitBehaviourBuildError {
+    #[error("{0}")]
+    Gossipsub(&'static str),
+}
+
+pub trait RecordStoreConfig<S>
+where
+    S: for<'a> RecordStore<'a>,
+{
+    fn init(self, id: PeerId) -> S;
+}
+
+impl RecordStoreConfig<MemoryStore> for MemoryStoreConfig {
+    fn init(self, id: PeerId) -> MemoryStore {
+        MemoryStore::with_config(id, self)
+    }
+}
+
+#[derive(Builder, Default, Debug, Clone)]
+pub struct IdentifyConfig {
+    #[builder(setter(into), default = "Duration::from_millis(500)")]
+    initial_delay: Duration,
+    #[builder(setter(into), default = "Duration::from_secs(300)")]
+    interval: Duration,
+    #[builder(setter(into), default = "false")]
+    push_listen_addr_updates: bool,
+    #[builder(setter(into), default = "0")]
+    cache_size: usize,
+}
+
+impl IdentifyConfig {
+    fn to_config(self, key: PublicKey) -> OIdentifyConfig {
+        OIdentifyConfig::new(PROTOCOL_VERSION.to_string(), key)
+            .with_initial_delay(self.initial_delay)
+            .with_interval(self.interval)
+            .with_push_listen_addr_updates(self.push_listen_addr_updates)
+            .with_cache_size(self.cache_size)
+    }
+}
+
+impl From<OIdentifyConfig> for IdentifyConfig {
+    fn from(c: OIdentifyConfig) -> Self {
+        Self {
+            initial_delay: c.initial_delay,
+            interval: c.interval,
+            push_listen_addr_updates: c.push_listen_addr_updates,
+            cache_size: c.cache_size,
+        }
+    }
+}
diff --git a/src/p2p/behaviour/mod.rs b/src/p2p/behaviour/mod.rs
new file mode 100644
index 00000000..b60cd7ff
--- /dev/null
+++ b/src/p2p/behaviour/mod.rs
@@ -0,0 +1,40 @@
+use core::time::Duration;
+use derive_builder::Builder;
+use libp2p::{
+    core::{muxing::StreamMuxerBox, transport::Boxed, PeerId},
+    dcutr::behaviour::Behaviour as Dcutr,
+    gossipsub::{
+        Gossipsub, GossipsubConfig, GossipsubConfigBuilder, MessageAuthenticity, ValidationMode,
+    },
+    identify::{Behaviour as Identify, Config as OIdentifyConfig},
+    identity::{Keypair, PublicKey},
+    kad::{
+        record::store::{MemoryStore, MemoryStoreConfig, RecordStore},
+        Kademlia, KademliaConfig,
+    },
+    ping::{Behaviour as Ping, Config as PingConfig},
+    relay::v2::client::Client,
+    swarm::{behaviour::toggle::Toggle, Swarm},
+    NetworkBehaviour,
+};
+use thiserror::Error;
+
+const PROTOCOL_VERSION: &'static str = "kepler/0.1.0";
+
+pub type OrbitSwarm<KS = MemoryStore> = Swarm<Behaviour<KS>>;
+mod builder;
+
+pub use builder::{BehaviourBuilder, OrbitBehaviourBuildError};
+
+#[derive(NetworkBehaviour)]
+pub struct Behaviour<KS>
+where
+    KS: 'static + for<'a> RecordStore<'a> + Send,
+{
+    identify: Identify,
+    ping: Ping,
+    gossipsub: Gossipsub,
+    relay: Toggle<Client>,
+    kademlia: Kademlia<KS>,
+    dcutr: Dcutr,
+}
diff --git a/src/p2p/mod.rs b/src/p2p/mod.rs
new file mode 100644
index 00000000..833e52df
--- /dev/null
+++ b/src/p2p/mod.rs
@@ -0,0 +1 @@
+pub mod behaviour;

From 0fbfefeb9a41afaa8a164e0c3feefdfa40cd1595 Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Fri, 4 Nov 2022 13:58:06 +0100
Subject: [PATCH 59/73] unused warns and stuff

---
 src/p2p/behaviour/builder.rs | 37 ++++++++++++++++++------------------
 1 file changed, 19 insertions(+), 18 deletions(-)

diff --git a/src/p2p/behaviour/builder.rs b/src/p2p/behaviour/builder.rs
index 6617766e..634f1d56 100644
--- a/src/p2p/behaviour/builder.rs
+++ b/src/p2p/behaviour/builder.rs
@@ -1,7 +1,8 @@
+use super::Behaviour;
 use core::time::Duration;
 use derive_builder::Builder;
 use libp2p::{
-    core::{muxing::StreamMuxerBox, transport::Boxed, PeerId},
+    core::PeerId,
     dcutr::behaviour::Behaviour as Dcutr,
     gossipsub::{
         Gossipsub, GossipsubConfig, GossipsubConfigBuilder, MessageAuthenticity, ValidationMode,
@@ -14,29 +15,29 @@ use libp2p::{
     },
     ping::{Behaviour as Ping, Config as PingConfig},
     relay::v2::client::Client,
-    swarm::{behaviour::toggle::Toggle, Swarm},
-    NetworkBehaviour,
 };
 use thiserror::Error;
 
 const PROTOCOL_VERSION: &'static str = "kepler/0.1.0";
 
+// we use derive_builder here to make a conveniant builder, but we do not export
+// the actual config struct
 #[derive(Builder, Clone, Debug)]
 #[builder(build_fn(skip), setter(into), name = "BehaviourBuilder", derive(Debug))]
 pub struct BehaviourConfig<KSC = MemoryStoreConfig>
 where
     KSC: Default,
 {
-    #[builder(field(type = "IdentifyConfig"))]
-    identify: IdentifyConfig,
-    #[builder(field(type = "PingConfig"))]
-    ping: PingConfig,
-    #[builder(field(type = "GossipsubConfig"))]
-    gossipsub: GossipsubConfig,
-    #[builder(field(type = "KademliaConfig"))]
-    kademlia: KademliaConfig,
-    #[builder(field(type = "KSC"))]
-    kademlia_store: KSC,
+    #[builder(field(type = "IdentifyConfig"), setter(name = "identify"))]
+    _identify: IdentifyConfig,
+    #[builder(field(type = "PingConfig"), setter(name = "ping"))]
+    _ping: PingConfig,
+    #[builder(field(type = "GossipsubConfig"), setter(name = "gossipsub"))]
+    _gossipsub: GossipsubConfig,
+    #[builder(field(type = "KademliaConfig"), setter(name = "kademlia"))]
+    _kademlia: KademliaConfig,
+    #[builder(field(type = "KSC"), setter(name = "kademlia_store"))]
+    _kademlia_store: KSC,
 }
 
 impl<KSC> BehaviourBuilder<KSC>
@@ -54,11 +55,11 @@ where
     {
         let peer_id = keypair.public().to_peer_id();
         Ok(Behaviour {
-            identify: Identify::new(self.identify.to_config(keypair.public())),
-            ping: Ping::new(self.ping),
+            identify: Identify::new(self._identify.to_config(keypair.public())),
+            ping: Ping::new(self._ping),
             gossipsub: Gossipsub::new(
                 MessageAuthenticity::Signed(keypair),
-                GossipsubConfigBuilder::from(self.gossipsub)
+                GossipsubConfigBuilder::from(self._gossipsub)
                     // always ensure validation
                     .validation_mode(ValidationMode::Strict)
                     .build()
@@ -68,8 +69,8 @@ where
             relay: relay.into(),
             kademlia: Kademlia::with_config(
                 peer_id,
-                self.kademlia_store.init(peer_id),
-                self.kademlia,
+                self._kademlia_store.init(peer_id),
+                self._kademlia,
             ),
             dcutr: Dcutr::new(),
         })

From 46a41c1ddb28549c33e4c4ce5ca5bef854ce53f3 Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Fri, 4 Nov 2022 15:00:18 +0100
Subject: [PATCH 60/73] add autonat to behaviour

---
 Cargo.lock                   | 40 +++++++++++++++++++++++++++++++
 Cargo.toml                   |  2 +-
 src/p2p/behaviour/builder.rs | 46 ++++++------------------------------
 src/p2p/behaviour/mod.rs     | 21 ++++++----------
 src/p2p/mod.rs               | 38 +++++++++++++++++++++++++++++
 5 files changed, 93 insertions(+), 54 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index b964bee4..8a6bb4a6 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3057,6 +3057,7 @@ dependencies = [
  "getrandom 0.2.8",
  "instant",
  "lazy_static",
+ "libp2p-autonat",
  "libp2p-core",
  "libp2p-dcutr",
  "libp2p-dns",
@@ -3078,6 +3079,25 @@ dependencies = [
  "smallvec",
 ]
 
+[[package]]
+name = "libp2p-autonat"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5a72e3fc1050ee2a206e30bf81d946df6ddb7b6930ae599027e3272d9035b157"
+dependencies = [
+ "async-trait",
+ "futures",
+ "futures-timer",
+ "instant",
+ "libp2p-core",
+ "libp2p-request-response",
+ "libp2p-swarm",
+ "log",
+ "prost 0.11.0",
+ "prost-build 0.11.1",
+ "rand 0.8.5",
+]
+
 [[package]]
 name = "libp2p-core"
 version = "0.37.0"
@@ -3323,6 +3343,24 @@ dependencies = [
  "void",
 ]
 
+[[package]]
+name = "libp2p-request-response"
+version = "0.22.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8827af16a017b65311a410bb626205a9ad92ec0473967618425039fa5231adc1"
+dependencies = [
+ "async-trait",
+ "bytes",
+ "futures",
+ "instant",
+ "libp2p-core",
+ "libp2p-swarm",
+ "log",
+ "rand 0.8.5",
+ "smallvec",
+ "unsigned-varint",
+]
+
 [[package]]
 name = "libp2p-swarm"
 version = "0.40.1"
@@ -6403,6 +6441,8 @@ checksum = "d86a8dc7f45e4c1b0d30e43038c38f274e77af056aa5f74b93c2cf9eb3c1c836"
 dependencies = [
  "asynchronous-codec",
  "bytes",
+ "futures-io",
+ "futures-util",
 ]
 
 [[package]]
diff --git a/Cargo.toml b/Cargo.toml
index 83b00635..8f670b25 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -26,7 +26,7 @@ hyper = "0.14" # Prometheus server
 iri-string = "0.5"
 lazy_static = "1.4.0"
 libipld = "0.14"
-libp2p = { default-features = false, features = ["gossipsub", "identify", "kad", "tcp-tokio", "mplex", "noise", "ping", "yamux", "dns-tokio", "relay", "dcutr"], version = "0.49.0" }
+libp2p = { default-features = false, features = ["autonat", "gossipsub", "identify", "kad", "tcp-tokio", "mplex", "noise", "ping", "yamux", "dns-tokio", "relay", "dcutr"], version = "0.49.0" }
 opentelemetry = { version = "0.17.0", features = ["rt-tokio"] }
 opentelemetry-jaeger = { version = "0.16.0", features = ["rt-tokio", "reqwest_collector_client"] }
 pin-project = "1"
diff --git a/src/p2p/behaviour/builder.rs b/src/p2p/behaviour/builder.rs
index 634f1d56..f2d7c9ee 100644
--- a/src/p2p/behaviour/builder.rs
+++ b/src/p2p/behaviour/builder.rs
@@ -1,14 +1,14 @@
-use super::Behaviour;
-use core::time::Duration;
+use crate::p2p::{behaviour::Behaviour, IdentifyConfig};
 use derive_builder::Builder;
 use libp2p::{
+    autonat::{Behaviour as AutoNat, Config as AutoNatConfig},
     core::PeerId,
     dcutr::behaviour::Behaviour as Dcutr,
     gossipsub::{
         Gossipsub, GossipsubConfig, GossipsubConfigBuilder, MessageAuthenticity, ValidationMode,
     },
-    identify::{Behaviour as Identify, Config as OIdentifyConfig},
-    identity::{Keypair, PublicKey},
+    identify::Behaviour as Identify,
+    identity::Keypair,
     kad::{
         record::store::{MemoryStore, MemoryStoreConfig, RecordStore},
         Kademlia, KademliaConfig,
@@ -18,8 +18,6 @@ use libp2p::{
 };
 use thiserror::Error;
 
-const PROTOCOL_VERSION: &'static str = "kepler/0.1.0";
-
 // we use derive_builder here to make a conveniant builder, but we do not export
 // the actual config struct
 #[derive(Builder, Clone, Debug)]
@@ -38,6 +36,8 @@ where
     _kademlia: KademliaConfig,
     #[builder(field(type = "KSC"), setter(name = "kademlia_store"))]
     _kademlia_store: KSC,
+    #[builder(field(type = "AutoNatConfig"), setter(name = "autonat"))]
+    _autonat: AutoNatConfig,
 }
 
 impl<KSC> BehaviourBuilder<KSC>
@@ -73,6 +73,7 @@ where
                 self._kademlia,
             ),
             dcutr: Dcutr::new(),
+            autonat: AutoNat::new(peer_id, self._autonat),
         })
     }
 }
@@ -95,36 +96,3 @@ impl RecordStoreConfig<MemoryStore> for MemoryStoreConfig {
         MemoryStore::with_config(id, self)
     }
 }
-
-#[derive(Builder, Default, Debug, Clone)]
-pub struct IdentifyConfig {
-    #[builder(setter(into), default = "Duration::from_millis(500)")]
-    initial_delay: Duration,
-    #[builder(setter(into), default = "Duration::from_secs(300)")]
-    interval: Duration,
-    #[builder(setter(into), default = "false")]
-    push_listen_addr_updates: bool,
-    #[builder(setter(into), default = "0")]
-    cache_size: usize,
-}
-
-impl IdentifyConfig {
-    fn to_config(self, key: PublicKey) -> OIdentifyConfig {
-        OIdentifyConfig::new(PROTOCOL_VERSION.to_string(), key)
-            .with_initial_delay(self.initial_delay)
-            .with_interval(self.interval)
-            .with_push_listen_addr_updates(self.push_listen_addr_updates)
-            .with_cache_size(self.cache_size)
-    }
-}
-
-impl From<OIdentifyConfig> for IdentifyConfig {
-    fn from(c: OIdentifyConfig) -> Self {
-        Self {
-            initial_delay: c.initial_delay,
-            interval: c.interval,
-            push_listen_addr_updates: c.push_listen_addr_updates,
-            cache_size: c.cache_size,
-        }
-    }
-}
diff --git a/src/p2p/behaviour/mod.rs b/src/p2p/behaviour/mod.rs
index b60cd7ff..3343ce55 100644
--- a/src/p2p/behaviour/mod.rs
+++ b/src/p2p/behaviour/mod.rs
@@ -1,25 +1,17 @@
-use core::time::Duration;
-use derive_builder::Builder;
 use libp2p::{
-    core::{muxing::StreamMuxerBox, transport::Boxed, PeerId},
+    autonat::Behaviour as AutoNat,
     dcutr::behaviour::Behaviour as Dcutr,
-    gossipsub::{
-        Gossipsub, GossipsubConfig, GossipsubConfigBuilder, MessageAuthenticity, ValidationMode,
-    },
-    identify::{Behaviour as Identify, Config as OIdentifyConfig},
-    identity::{Keypair, PublicKey},
+    gossipsub::Gossipsub,
+    identify::Behaviour as Identify,
     kad::{
-        record::store::{MemoryStore, MemoryStoreConfig, RecordStore},
-        Kademlia, KademliaConfig,
+        record::store::{MemoryStore, RecordStore},
+        Kademlia,
     },
-    ping::{Behaviour as Ping, Config as PingConfig},
+    ping::Behaviour as Ping,
     relay::v2::client::Client,
     swarm::{behaviour::toggle::Toggle, Swarm},
     NetworkBehaviour,
 };
-use thiserror::Error;
-
-const PROTOCOL_VERSION: &'static str = "kepler/0.1.0";
 
 pub type OrbitSwarm<KS = MemoryStore> = Swarm<Behaviour<KS>>;
 mod builder;
@@ -37,4 +29,5 @@ where
     relay: Toggle<Client>,
     kademlia: Kademlia<KS>,
     dcutr: Dcutr,
+    autonat: AutoNat,
 }
diff --git a/src/p2p/mod.rs b/src/p2p/mod.rs
index 833e52df..0e62dfc4 100644
--- a/src/p2p/mod.rs
+++ b/src/p2p/mod.rs
@@ -1 +1,39 @@
+use core::time::Duration;
+use derive_builder::Builder;
+use libp2p::{identify::Config as OIdentifyConfig, identity::PublicKey};
 pub mod behaviour;
+
+#[derive(Builder, Default, Debug, Clone)]
+pub struct IdentifyConfig {
+    #[builder(setter(into), default = "Duration::from_millis(500)")]
+    initial_delay: Duration,
+    #[builder(setter(into), default = "Duration::from_secs(300)")]
+    interval: Duration,
+    #[builder(setter(into), default = "false")]
+    push_listen_addr_updates: bool,
+    #[builder(setter(into), default = "0")]
+    cache_size: usize,
+}
+
+const PROTOCOL_VERSION: &'static str = "kepler/0.1.0";
+
+impl IdentifyConfig {
+    fn to_config(self, key: PublicKey) -> OIdentifyConfig {
+        OIdentifyConfig::new(PROTOCOL_VERSION.to_string(), key)
+            .with_initial_delay(self.initial_delay)
+            .with_interval(self.interval)
+            .with_push_listen_addr_updates(self.push_listen_addr_updates)
+            .with_cache_size(self.cache_size)
+    }
+}
+
+impl From<OIdentifyConfig> for IdentifyConfig {
+    fn from(c: OIdentifyConfig) -> Self {
+        Self {
+            initial_delay: c.initial_delay,
+            interval: c.interval,
+            push_listen_addr_updates: c.push_listen_addr_updates,
+            cache_size: c.cache_size,
+        }
+    }
+}

From 800226a9887d5afc73f7c873686e4cdb39704190 Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Fri, 4 Nov 2022 16:13:02 +0100
Subject: [PATCH 61/73] add an incomplete relay impl

---
 src/auth_guards.rs     |  2 +-
 src/lib.rs             |  3 +--
 src/p2p/mod.rs         |  1 +
 src/{ => p2p}/relay.rs | 54 +++++++++++++++++++++++++++++++++++++++++-
 src/routes/mod.rs      |  2 +-
 5 files changed, 57 insertions(+), 5 deletions(-)
 rename src/{ => p2p}/relay.rs (77%)

diff --git a/src/auth_guards.rs b/src/auth_guards.rs
index cf6fb05d..66c03bd2 100644
--- a/src/auth_guards.rs
+++ b/src/auth_guards.rs
@@ -2,7 +2,7 @@ use crate::authorization::{Delegation, Invocation, Verifiable};
 use crate::capabilities::store::{InvokeError, ToBlock, Updates};
 use crate::config;
 use crate::orbit::{create_orbit, load_orbit, Orbit};
-use crate::relay::RelayNode;
+use crate::p2p::relay::RelayNode;
 use crate::routes::Metadata;
 use crate::BlockStores;
 use anyhow::Result;
diff --git a/src/lib.rs b/src/lib.rs
index a8e30b53..70f922d8 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -22,7 +22,6 @@ pub mod manifest;
 pub mod orbit;
 pub mod p2p;
 pub mod prometheus;
-pub mod relay;
 pub mod routes;
 pub mod storage;
 mod tracing;
@@ -34,7 +33,7 @@ use libp2p::{
     PeerId,
 };
 use orbit::ProviderUtils;
-use relay::RelayNode;
+use p2p::relay::RelayNode;
 use routes::{delegate, invoke, open_host_key, relay_addr, util_routes::*};
 use std::{collections::HashMap, sync::RwLock};
 use storage::{
diff --git a/src/p2p/mod.rs b/src/p2p/mod.rs
index 0e62dfc4..3ad452a7 100644
--- a/src/p2p/mod.rs
+++ b/src/p2p/mod.rs
@@ -2,6 +2,7 @@ use core::time::Duration;
 use derive_builder::Builder;
 use libp2p::{identify::Config as OIdentifyConfig, identity::PublicKey};
 pub mod behaviour;
+pub mod relay;
 
 #[derive(Builder, Default, Debug, Clone)]
 pub struct IdentifyConfig {
diff --git a/src/relay.rs b/src/p2p/relay.rs
similarity index 77%
rename from src/relay.rs
rename to src/p2p/relay.rs
index c3ead904..56c5c68f 100644
--- a/src/relay.rs
+++ b/src/p2p/relay.rs
@@ -1,11 +1,63 @@
+use crate::p2p::IdentifyConfig;
 use anyhow::Result;
-use libp2p::core::{identity::Keypair, multiaddr::multiaddr, Multiaddr, PeerId};
+use derive_builder::Builder;
+use libp2p::{
+    autonat::{Behaviour as AutoNat, Config as AutoNatConfig},
+    core::{identity::Keypair, multiaddr::multiaddr, Multiaddr, PeerId},
+    identify::Behaviour as Identify,
+    identity::PublicKey,
+    ping::{Behaviour as Ping, Config as PingConfig},
+    relay::v2::relay::{Config as RelayConfig, Relay},
+    swarm::Swarm,
+    NetworkBehaviour,
+};
+
+pub type RelaySwarm = Swarm<Behaviour>;
 
 pub struct RelayNode {
     pub port: u16,
     pub id: PeerId,
 }
 
+#[derive(NetworkBehaviour)]
+pub struct Behaviour {
+    identify: Identify,
+    ping: Ping,
+    relay: Relay,
+    autonat: AutoNat,
+}
+
+#[derive(Builder, Debug)]
+#[builder(
+    build_fn(skip),
+    setter(into),
+    name = "BehaviourBuilder",
+    derive(Debug),
+    pattern = "owned"
+)]
+pub struct BehaviourConfig {
+    #[builder(field(type = "IdentifyConfig"), setter(name = "identify"))]
+    _identify: IdentifyConfig,
+    #[builder(field(type = "PingConfig"), setter(name = "ping"))]
+    _ping: PingConfig,
+    #[builder(field(type = "RelayConfig"), setter(name = "relay"))]
+    _relay: RelayConfig,
+    #[builder(field(type = "AutoNatConfig"), setter(name = "autonat"))]
+    _autonat: AutoNatConfig,
+}
+
+impl BehaviourBuilder {
+    pub fn build(self, pubkey: PublicKey) -> Behaviour {
+        let peer_id = pubkey.to_peer_id();
+        Behaviour {
+            identify: Identify::new(self._identify.to_config(pubkey)),
+            ping: Ping::new(self._ping),
+            relay: Relay::new(peer_id, self._relay),
+            autonat: AutoNat::new(peer_id, self._autonat),
+        }
+    }
+}
+
 impl RelayNode {
     pub async fn new(port: u16, keypair: Keypair) -> Result<Self> {
         let local_public_key = keypair.public();
diff --git a/src/routes/mod.rs b/src/routes/mod.rs
index 958fe778..4b2596f2 100644
--- a/src/routes/mod.rs
+++ b/src/routes/mod.rs
@@ -23,7 +23,7 @@ use tracing::{info_span, Instrument};
 use crate::{
     auth_guards::{DelegateAuthWrapper, InvokeAuthWrapper, KVAction},
     kv::{ObjectBuilder, ReadResponse},
-    relay::RelayNode,
+    p2p::relay::RelayNode,
     storage::ImmutableStore,
     tracing::TracingSpan,
     BlockStores,

From 303db102d3e98ea9a1e927dcf83f0c1ee01f61c7 Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Thu, 10 Nov 2022 10:38:31 +0100
Subject: [PATCH 62/73] impl partially stubbed relay node with task

---
 src/auth_guards.rs |   2 +-
 src/lib.rs         |   2 +-
 src/p2p/relay.rs   | 225 +++++++++++++++++++++++++++++----------------
 src/routes/mod.rs  |   2 +-
 4 files changed, 149 insertions(+), 82 deletions(-)

diff --git a/src/auth_guards.rs b/src/auth_guards.rs
index 66c03bd2..b4d84d7f 100644
--- a/src/auth_guards.rs
+++ b/src/auth_guards.rs
@@ -81,7 +81,7 @@ fn get_state(req: &Request<'_>) -> Result<(config::Config, (PeerId, Multiaddr))>
             .ok_or_else(|| anyhow!("Could not retrieve config"))?,
         req.rocket()
             .state::<RelayNode>()
-            .map(|r| (r.id, r.internal()))
+            .map(|r| (r.id().clone(), r.internal()))
             .ok_or_else(|| anyhow!("Could not retrieve relay node information"))?,
     ))
 }
diff --git a/src/lib.rs b/src/lib.rs
index 70f922d8..36920921 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -33,7 +33,7 @@ use libp2p::{
     PeerId,
 };
 use orbit::ProviderUtils;
-use p2p::relay::RelayNode;
+use p2p::relay::{Config as RelayConfig, RelayNode};
 use routes::{delegate, invoke, open_host_key, relay_addr, util_routes::*};
 use std::{collections::HashMap, sync::RwLock};
 use storage::{
diff --git a/src/p2p/relay.rs b/src/p2p/relay.rs
index 56c5c68f..1ff9542c 100644
--- a/src/p2p/relay.rs
+++ b/src/p2p/relay.rs
@@ -1,103 +1,68 @@
-use crate::p2p::IdentifyConfig;
+use crate::{orbit::AbortOnDrop, p2p::IdentifyConfig};
 use anyhow::Result;
-use derive_builder::Builder;
+use futures::{
+    channel::{mpsc, oneshot},
+    future::{select, Either},
+    io::{AsyncRead, AsyncWrite},
+    sink::{Sink, SinkExt},
+    stream::{iter, Stream, StreamExt},
+};
 use libp2p::{
-    autonat::{Behaviour as AutoNat, Config as AutoNatConfig},
-    core::{identity::Keypair, multiaddr::multiaddr, Multiaddr, PeerId},
+    autonat::Behaviour as AutoNat,
+    core::{
+        identity::Keypair, multiaddr::multiaddr, transport::upgrade::Builder, upgrade, Multiaddr,
+        PeerId,
+    },
     identify::Behaviour as Identify,
-    identity::PublicKey,
-    ping::{Behaviour as Ping, Config as PingConfig},
-    relay::v2::relay::{Config as RelayConfig, Relay},
-    swarm::Swarm,
-    NetworkBehaviour,
+    mplex, noise,
+    ping::Behaviour as Ping,
+    relay::v2::relay::Relay,
+    swarm::{Swarm, SwarmBuilder},
+    yamux, NetworkBehaviour,
 };
 
+pub use builder::Config;
+
 pub type RelaySwarm = Swarm<Behaviour>;
 
+#[derive(Clone, Debug)]
 pub struct RelayNode {
-    pub port: u16,
-    pub id: PeerId,
+    id: PeerId,
+    sender: mpsc::Sender<Message>,
+    port: u16,
 }
 
 #[derive(NetworkBehaviour)]
-pub struct Behaviour {
+struct Behaviour {
     identify: Identify,
     ping: Ping,
     relay: Relay,
     autonat: AutoNat,
 }
 
-#[derive(Builder, Debug)]
-#[builder(
-    build_fn(skip),
-    setter(into),
-    name = "BehaviourBuilder",
-    derive(Debug),
-    pattern = "owned"
-)]
-pub struct BehaviourConfig {
-    #[builder(field(type = "IdentifyConfig"), setter(name = "identify"))]
-    _identify: IdentifyConfig,
-    #[builder(field(type = "PingConfig"), setter(name = "ping"))]
-    _ping: PingConfig,
-    #[builder(field(type = "RelayConfig"), setter(name = "relay"))]
-    _relay: RelayConfig,
-    #[builder(field(type = "AutoNatConfig"), setter(name = "autonat"))]
-    _autonat: AutoNatConfig,
+#[derive(Debug)]
+pub enum Message {
+    GetAddresses(oneshot::Sender<Vec<Multiaddr>>),
+    ListenOn(Multiaddr),
 }
 
-impl BehaviourBuilder {
-    pub fn build(self, pubkey: PublicKey) -> Behaviour {
-        let peer_id = pubkey.to_peer_id();
-        Behaviour {
-            identify: Identify::new(self._identify.to_config(pubkey)),
-            ping: Ping::new(self._ping),
-            relay: Relay::new(peer_id, self._relay),
-            autonat: AutoNat::new(peer_id, self._autonat),
-        }
+impl RelayNode {
+    pub fn id(&self) -> &PeerId {
+        &self.id
+    }
+    async fn get_addresses(&mut self) -> Result<Vec<Multiaddr>> {
+        let (s, r) = oneshot::channel();
+        self.sender.send(Message::GetAddresses(s)).await?;
+        Ok(r.await?)
     }
-}
 
-impl RelayNode {
-    pub async fn new(port: u16, keypair: Keypair) -> Result<Self> {
-        let local_public_key = keypair.public();
-        let id = local_public_key.to_peer_id();
-        // let relay_tcp_addr = Self::_external(port);
-        // let relay_mem_addr = Self::_internal(port);
-
-        // let (transport_builder, relay_behaviour) = TransportBuilder::new(keypair.clone())?
-        //     .or(MemoryTransport::default())
-        //     .relay();
-
-        // let ipfs_opts = IpfsOptions {
-        //     ipfs_path: std::env::temp_dir(),
-        //     keypair,
-        //     bootstrap: vec![],
-        //     mdns: false,
-        //     kad_protocol: "/kepler/relay".to_string().into(),
-        //     listening_addrs: vec![relay_tcp_addr, relay_mem_addr],
-        //     span: None,
-        // };
-
-        // // TestTypes designates an in-memory Ipfs instance, but this peer won't store data anyway.
-        // let (_ipfs, ipfs_task) =
-        //     UninitializedIpfs::new(ipfs_opts, transport_builder.build(), Some(relay_behaviour))
-        //         .start()
-        //         .await?;
-
-        // tracing::debug!(
-        //     "opened relay: {} at {}, {}",
-        //     id,
-        //     Self::_internal(port),
-        //     Self::_external(port),
-        // );
-
-        // let task = spawn(ipfs_task);
-        Ok(Self {
-            port,
-            // _task: AbortOnDrop::new(task),
-            id,
-        })
+    async fn listen_on(&mut self, addr: impl IntoIterator<Item = &Multiaddr>) -> Result<()> {
+        Ok(self
+            .sender
+            .send_all(&mut iter(
+                addr.into_iter().map(|a| Ok(Message::ListenOn(a.clone()))),
+            ))
+            .await?)
     }
 
     fn _internal(port: u16) -> Multiaddr {
@@ -117,6 +82,108 @@ impl RelayNode {
     }
 }
 
+mod builder {
+    use super::*;
+    use derive_builder::Builder;
+    use libp2p::{
+        autonat::Config as AutoNatConfig, core::Transport, identity::PublicKey,
+        ping::Config as PingConfig, relay::v2::relay::Config as RelayConfig,
+    };
+
+    #[derive(Builder, Debug)]
+    #[builder(
+        build_fn(skip),
+        setter(into),
+        name = "Config",
+        derive(Debug),
+        pattern = "owned"
+    )]
+    pub struct BehaviourConfigDummy {
+        #[builder(field(type = "IdentifyConfig"))]
+        identify: IdentifyConfig,
+        #[builder(field(type = "PingConfig"))]
+        ping: PingConfig,
+        #[builder(field(type = "RelayConfig"))]
+        relay: RelayConfig,
+        #[builder(field(type = "AutoNatConfig"))]
+        autonat: AutoNatConfig,
+    }
+
+    impl Config {
+        fn build(self, pubkey: PublicKey) -> Behaviour {
+            let peer_id = pubkey.to_peer_id();
+            Behaviour {
+                identify: Identify::new(self.identify.to_config(pubkey)),
+                ping: Ping::new(self.ping),
+                relay: Relay::new(peer_id, self.relay),
+                autonat: AutoNat::new(peer_id, self.autonat),
+            }
+        }
+
+        pub fn launch<T>(
+            self,
+            keypair: Keypair,
+            transport: Builder<T>,
+            port: u16,
+        ) -> Result<RelayNode>
+        where
+            T: Transport + Send,
+            T::Output: AsyncRead + AsyncWrite + Unpin + Send,
+            T::Error: Send + Sync,
+            T: Unpin,
+            T::Dial: Send,
+            T::ListenerUpgrade: Send,
+        {
+            let local_public_key = keypair.public();
+            let id = local_public_key.to_peer_id();
+            let b = self.build(local_public_key);
+            let (sender, reciever) = mpsc::channel(100);
+            let r = RelayNode { id, sender, port };
+
+            let mut swarm = SwarmBuilder::new(
+                transport
+                    .authenticate(noise::NoiseAuthenticated::xx(&keypair).unwrap())
+                    .multiplex(upgrade::SelectUpgrade::new(
+                        yamux::YamuxConfig::default(),
+                        mplex::MplexConfig::default(),
+                    ))
+                    .timeout(std::time::Duration::from_secs(20))
+                    .boxed(),
+                b,
+                id,
+            )
+            .build();
+
+            swarm.listen_on(r.external())?;
+            swarm.listen_on(r.internal())?;
+
+            tokio::spawn(async move {
+                loop {
+                    match select(reciever.next(), swarm.next()).await {
+                        // if the swarm or the channel are closed, close the relay
+                        Either::Right((None, _)) | Either::Left((None, _)) => {
+                            break;
+                        }
+                        // process command
+                        Either::Left((Some(e), _)) => match e {
+                            Message::ListenOn(a) => swarm.listen_on(a).map(|_| ())?,
+                            Message::GetAddresses(s) => {
+                                s.send(swarm.listeners().map(|a| a.clone()).collect())
+                                    .map_err(|_| anyhow!("failed to return listeners"))?;
+                            }
+                        },
+                        Either::Right((Some(e), _)) => {
+                            // process swarm event
+                        }
+                    }
+                }
+                Result::<(), anyhow::Error>::Ok(())
+            });
+            Ok(r)
+        }
+    }
+}
+
 #[cfg(test)]
 pub mod test {
     // use super::*;
diff --git a/src/routes/mod.rs b/src/routes/mod.rs
index 4b2596f2..2185e48a 100644
--- a/src/routes/mod.rs
+++ b/src/routes/mod.rs
@@ -90,7 +90,7 @@ pub mod util_routes {
 pub fn relay_addr(relay: &State<RelayNode>) -> String {
     relay
         .external()
-        .with(Protocol::P2p(relay.id.into()))
+        .with(Protocol::P2p(relay.id().clone().into()))
         .to_string()
 }
 

From 7312d4627be27c0eec16ebda6b4527ba85d4ae83 Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Thu, 10 Nov 2022 10:38:56 +0100
Subject: [PATCH 63/73] minor builder refactors

---
 src/p2p/behaviour/builder.rs | 42 ++++++++++++++---------------
 src/p2p/behaviour/mod.rs     |  2 +-
 src/p2p/mod.rs               | 51 ++++++++++++++++++++++--------------
 3 files changed, 54 insertions(+), 41 deletions(-)

diff --git a/src/p2p/behaviour/builder.rs b/src/p2p/behaviour/builder.rs
index f2d7c9ee..86db2b30 100644
--- a/src/p2p/behaviour/builder.rs
+++ b/src/p2p/behaviour/builder.rs
@@ -21,26 +21,26 @@ use thiserror::Error;
 // we use derive_builder here to make a conveniant builder, but we do not export
 // the actual config struct
 #[derive(Builder, Clone, Debug)]
-#[builder(build_fn(skip), setter(into), name = "BehaviourBuilder", derive(Debug))]
-pub struct BehaviourConfig<KSC = MemoryStoreConfig>
+#[builder(build_fn(skip), setter(into), name = "BehaviourConfig", derive(Debug))]
+pub struct BehaviourConfigDummy<KSC = MemoryStoreConfig>
 where
     KSC: Default,
 {
-    #[builder(field(type = "IdentifyConfig"), setter(name = "identify"))]
-    _identify: IdentifyConfig,
-    #[builder(field(type = "PingConfig"), setter(name = "ping"))]
-    _ping: PingConfig,
-    #[builder(field(type = "GossipsubConfig"), setter(name = "gossipsub"))]
-    _gossipsub: GossipsubConfig,
-    #[builder(field(type = "KademliaConfig"), setter(name = "kademlia"))]
-    _kademlia: KademliaConfig,
-    #[builder(field(type = "KSC"), setter(name = "kademlia_store"))]
-    _kademlia_store: KSC,
-    #[builder(field(type = "AutoNatConfig"), setter(name = "autonat"))]
-    _autonat: AutoNatConfig,
+    #[builder(field(type = "IdentifyConfig"))]
+    identify: IdentifyConfig,
+    #[builder(field(type = "PingConfig"))]
+    ping: PingConfig,
+    #[builder(field(type = "GossipsubConfig"))]
+    gossipsub: GossipsubConfig,
+    #[builder(field(type = "KademliaConfig"))]
+    kademlia: KademliaConfig,
+    #[builder(field(type = "KSC"))]
+    kademlia_store: KSC,
+    #[builder(field(type = "AutoNatConfig"))]
+    autonat: AutoNatConfig,
 }
 
-impl<KSC> BehaviourBuilder<KSC>
+impl<KSC> BehaviourConfig<KSC>
 where
     KSC: Default,
 {
@@ -55,11 +55,11 @@ where
     {
         let peer_id = keypair.public().to_peer_id();
         Ok(Behaviour {
-            identify: Identify::new(self._identify.to_config(keypair.public())),
-            ping: Ping::new(self._ping),
+            identify: Identify::new(self.identify.to_config(keypair.public())),
+            ping: Ping::new(self.ping),
             gossipsub: Gossipsub::new(
                 MessageAuthenticity::Signed(keypair),
-                GossipsubConfigBuilder::from(self._gossipsub)
+                GossipsubConfigBuilder::from(self.gossipsub)
                     // always ensure validation
                     .validation_mode(ValidationMode::Strict)
                     .build()
@@ -69,11 +69,11 @@ where
             relay: relay.into(),
             kademlia: Kademlia::with_config(
                 peer_id,
-                self._kademlia_store.init(peer_id),
-                self._kademlia,
+                self.kademlia_store.init(peer_id),
+                self.kademlia,
             ),
             dcutr: Dcutr::new(),
-            autonat: AutoNat::new(peer_id, self._autonat),
+            autonat: AutoNat::new(peer_id, self.autonat),
         })
     }
 }
diff --git a/src/p2p/behaviour/mod.rs b/src/p2p/behaviour/mod.rs
index 3343ce55..e6ae179f 100644
--- a/src/p2p/behaviour/mod.rs
+++ b/src/p2p/behaviour/mod.rs
@@ -16,7 +16,7 @@ use libp2p::{
 pub type OrbitSwarm<KS = MemoryStore> = Swarm<Behaviour<KS>>;
 mod builder;
 
-pub use builder::{BehaviourBuilder, OrbitBehaviourBuildError};
+pub use builder::{BehaviourConfig, OrbitBehaviourBuildError};
 
 #[derive(NetworkBehaviour)]
 pub struct Behaviour<KS>
diff --git a/src/p2p/mod.rs b/src/p2p/mod.rs
index 3ad452a7..aa2ffcf5 100644
--- a/src/p2p/mod.rs
+++ b/src/p2p/mod.rs
@@ -1,36 +1,49 @@
-use core::time::Duration;
-use derive_builder::Builder;
 use libp2p::{identify::Config as OIdentifyConfig, identity::PublicKey};
 pub mod behaviour;
 pub mod relay;
-
-#[derive(Builder, Default, Debug, Clone)]
-pub struct IdentifyConfig {
-    #[builder(setter(into), default = "Duration::from_millis(500)")]
-    initial_delay: Duration,
-    #[builder(setter(into), default = "Duration::from_secs(300)")]
-    interval: Duration,
-    #[builder(setter(into), default = "false")]
-    push_listen_addr_updates: bool,
-    #[builder(setter(into), default = "0")]
-    cache_size: usize,
-}
+pub mod transport;
 
 const PROTOCOL_VERSION: &'static str = "kepler/0.1.0";
 
+pub use builder::IdentifyConfig;
+
+mod builder {
+    use core::time::Duration;
+    use derive_builder::Builder;
+    use libp2p::{identify::Config as OIdentifyConfig, identity::PublicKey};
+    #[derive(Builder, Default, Debug, Clone)]
+    #[builder(build_fn(skip), setter(into), name = "IdentifyConfig", derive(Debug))]
+    pub struct IdentifyConfigDummy {
+        #[builder(field(type = "String"), default = "PROTOCOL_VERSION.into()")]
+        protocol_version: String,
+        #[builder(field(type = "Duration"), default = "Duration::from_millis(500)")]
+        initial_delay: Duration,
+        #[builder(field(type = "Duration"), default = "Duration::from_secs(300)")]
+        interval: Duration,
+        #[builder(field(type = "bool"), default = "false")]
+        push_listen_addr_updates: bool,
+        #[builder(field(type = "usize"), default = "0")]
+        cache_size: usize,
+    }
+    pub fn convert(config: IdentifyConfig, key: PublicKey) -> OIdentifyConfig {
+        OIdentifyConfig::new(config.protocol_version, key)
+            .with_initial_delay(config.initial_delay)
+            .with_interval(config.interval)
+            .with_push_listen_addr_updates(config.push_listen_addr_updates)
+            .with_cache_size(config.cache_size)
+    }
+}
+
 impl IdentifyConfig {
     fn to_config(self, key: PublicKey) -> OIdentifyConfig {
-        OIdentifyConfig::new(PROTOCOL_VERSION.to_string(), key)
-            .with_initial_delay(self.initial_delay)
-            .with_interval(self.interval)
-            .with_push_listen_addr_updates(self.push_listen_addr_updates)
-            .with_cache_size(self.cache_size)
+        builder::convert(self, key)
     }
 }
 
 impl From<OIdentifyConfig> for IdentifyConfig {
     fn from(c: OIdentifyConfig) -> Self {
         Self {
+            protocol_version: c.protocol_version,
             initial_delay: c.initial_delay,
             interval: c.interval,
             push_listen_addr_updates: c.push_listen_addr_updates,

From 44f5ec0e000c5abd61ad393cd44a225014c61584 Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Thu, 10 Nov 2022 10:40:19 +0100
Subject: [PATCH 64/73] wip

---
 Cargo.lock           | 137 ++++++++++++++++++++++++++++++++++++-------
 Cargo.toml           |   2 +-
 src/p2p/transport.rs |  88 +++++++++++++++++++++++++++
 3 files changed, 206 insertions(+), 21 deletions(-)
 create mode 100644 src/p2p/transport.rs

diff --git a/Cargo.lock b/Cargo.lock
index 8a6bb4a6..76a52f42 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -249,7 +249,7 @@ dependencies = [
  "log",
  "memchr",
  "once_cell",
- "pin-project-lite",
+ "pin-project-lite 0.2.9",
  "pin-utils",
  "slab",
  "wasm-bindgen-futures",
@@ -309,7 +309,7 @@ dependencies = [
  "futures-sink",
  "futures-util",
  "memchr",
- "pin-project-lite",
+ "pin-project-lite 0.2.9",
 ]
 
 [[package]]
@@ -400,7 +400,7 @@ dependencies = [
  "http-body",
  "lazy_static",
  "percent-encoding",
- "pin-project-lite",
+ "pin-project-lite 0.2.9",
  "tracing",
 ]
 
@@ -540,7 +540,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bc604f278bae64bbd15854baa9c46ed69a56dfb0669d04aab80974749f2d6599"
 dependencies = [
  "futures-util",
- "pin-project-lite",
+ "pin-project-lite 0.2.9",
  "tokio",
  "tokio-stream",
 ]
@@ -560,7 +560,7 @@ dependencies = [
  "http",
  "http-body",
  "md-5 0.10.5",
- "pin-project-lite",
+ "pin-project-lite 0.2.9",
  "sha1",
  "sha2 0.10.6",
  "tracing",
@@ -583,7 +583,7 @@ dependencies = [
  "hyper",
  "hyper-rustls 0.22.1",
  "lazy_static",
- "pin-project-lite",
+ "pin-project-lite 0.2.9",
  "tokio",
  "tower",
  "tracing",
@@ -616,7 +616,7 @@ dependencies = [
  "hyper",
  "once_cell",
  "percent-encoding",
- "pin-project-lite",
+ "pin-project-lite 0.2.9",
  "tokio",
  "tokio-util",
  "tracing",
@@ -632,7 +632,7 @@ dependencies = [
  "bytes",
  "http",
  "http-body",
- "pin-project-lite",
+ "pin-project-lite 0.2.9",
  "tower",
  "tracing",
 ]
@@ -2061,6 +2061,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f82b0f4c27ad9f8bfd1f3208d882da2b09c301bc1c828fd3a00d0216d2fbbff6"
 dependencies = [
  "crc32fast",
+ "libz-sys",
  "miniz_oxide",
 ]
 
@@ -2170,7 +2171,7 @@ dependencies = [
  "futures-io",
  "memchr",
  "parking",
- "pin-project-lite",
+ "pin-project-lite 0.2.9",
  "waker-fn",
 ]
 
@@ -2185,6 +2186,17 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "futures-rustls"
+version = "0.22.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d2411eed028cdf8c8034eaf21f9915f956b6c3abec4d4c7949ee67f0721127bd"
+dependencies = [
+ "futures-io",
+ "rustls 0.20.7",
+ "webpki 0.22.0",
+]
+
 [[package]]
 name = "futures-sink"
 version = "0.3.25"
@@ -2216,7 +2228,7 @@ dependencies = [
  "futures-sink",
  "futures-task",
  "memchr",
- "pin-project-lite",
+ "pin-project-lite 0.2.9",
  "pin-utils",
  "slab",
 ]
@@ -2469,7 +2481,7 @@ checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1"
 dependencies = [
  "bytes",
  "http",
- "pin-project-lite",
+ "pin-project-lite 0.2.9",
 ]
 
 [[package]]
@@ -2500,7 +2512,7 @@ dependencies = [
  "httparse",
  "httpdate",
  "itoa",
- "pin-project-lite",
+ "pin-project-lite 0.2.9",
  "socket2",
  "tokio",
  "tower-service",
@@ -3072,6 +3084,8 @@ dependencies = [
  "libp2p-swarm",
  "libp2p-swarm-derive",
  "libp2p-tcp",
+ "libp2p-wasm-ext",
+ "libp2p-websocket",
  "libp2p-yamux",
  "multiaddr",
  "parking_lot 0.12.1",
@@ -3408,6 +3422,39 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "libp2p-wasm-ext"
+version = "0.37.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a17b5b8e7a73e379e47b1b77f8a82c4721e97eca01abcd18e9cd91a23ca6ce97"
+dependencies = [
+ "futures",
+ "js-sys",
+ "libp2p-core",
+ "parity-send-wrapper",
+ "wasm-bindgen",
+ "wasm-bindgen-futures",
+]
+
+[[package]]
+name = "libp2p-websocket"
+version = "0.39.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3758ae6f89b2531a24b6d9f5776bda6a626b60a57600d7185d43dfa75ca5ecc4"
+dependencies = [
+ "either",
+ "futures",
+ "futures-rustls",
+ "libp2p-core",
+ "log",
+ "parking_lot 0.12.1",
+ "quicksink",
+ "rw-stream-sink",
+ "soketto",
+ "url",
+ "webpki-roots",
+]
+
 [[package]]
 name = "libp2p-yamux"
 version = "0.41.0"
@@ -3422,6 +3469,17 @@ dependencies = [
  "yamux",
 ]
 
+[[package]]
+name = "libz-sys"
+version = "1.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9702761c3935f8cc2f101793272e202c72b99da8f4224a19ddcf1279a6450bbf"
+dependencies = [
+ "cc",
+ "pkg-config",
+ "vcpkg",
+]
+
 [[package]]
 name = "link-cplusplus"
 version = "1.0.7"
@@ -4070,6 +4128,12 @@ dependencies = [
  "sha2 0.10.6",
 ]
 
+[[package]]
+name = "parity-send-wrapper"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aa9777aa91b8ad9dd5aaa04a9b6bcb02c7f1deb952fca5a66034d5e63afc5c6f"
+
 [[package]]
 name = "parking"
 version = "2.0.0"
@@ -4253,6 +4317,12 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "pin-project-lite"
+version = "0.1.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "257b64915a082f7811703966789728173279bdebb956b143dbcd23f6f970a777"
+
 [[package]]
 name = "pin-project-lite"
 version = "0.2.9"
@@ -4611,6 +4681,17 @@ version = "1.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0"
 
+[[package]]
+name = "quicksink"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "77de3c815e5a160b1539c6592796801df2043ae35e123b46d73380cfa57af858"
+dependencies = [
+ "futures-core",
+ "futures-sink",
+ "pin-project-lite 0.1.12",
+]
+
 [[package]]
 name = "quote"
 version = "1.0.21"
@@ -4785,7 +4866,7 @@ dependencies = [
  "native-tls",
  "once_cell",
  "percent-encoding",
- "pin-project-lite",
+ "pin-project-lite 0.2.9",
  "rustls 0.20.7",
  "rustls-pemfile",
  "serde",
@@ -4872,7 +4953,7 @@ dependencies = [
  "multer",
  "num_cpus",
  "parking_lot 0.12.1",
- "pin-project-lite",
+ "pin-project-lite 0.2.9",
  "rand 0.8.5",
  "ref-cast",
  "rocket_codegen",
@@ -4922,7 +5003,7 @@ dependencies = [
  "memchr",
  "pear",
  "percent-encoding",
- "pin-project-lite",
+ "pin-project-lite 0.2.9",
  "ref-cast",
  "serde",
  "smallvec",
@@ -5497,6 +5578,22 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "soketto"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "41d1c5305e39e09653383c2c7244f2f78b3bcae37cf50c64cb4789c9f5096ec2"
+dependencies = [
+ "base64 0.13.1",
+ "bytes",
+ "flate2",
+ "futures",
+ "httparse",
+ "log",
+ "rand 0.8.5",
+ "sha-1",
+]
+
 [[package]]
 name = "spin"
 version = "0.5.2"
@@ -6057,7 +6154,7 @@ dependencies = [
  "memchr",
  "mio",
  "num_cpus",
- "pin-project-lite",
+ "pin-project-lite 0.2.9",
  "signal-hook-registry",
  "socket2",
  "tokio-macros",
@@ -6126,7 +6223,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d660770404473ccd7bc9f8b28494a811bc18542b915c0855c51e8f419d5223ce"
 dependencies = [
  "futures-core",
- "pin-project-lite",
+ "pin-project-lite 0.2.9",
  "tokio",
 ]
 
@@ -6140,7 +6237,7 @@ dependencies = [
  "futures-core",
  "futures-io",
  "futures-sink",
- "pin-project-lite",
+ "pin-project-lite 0.2.9",
  "tokio",
  "tracing",
 ]
@@ -6163,7 +6260,7 @@ dependencies = [
  "futures-core",
  "futures-util",
  "pin-project",
- "pin-project-lite",
+ "pin-project-lite 0.2.9",
  "tokio",
  "tower-layer",
  "tower-service",
@@ -6190,7 +6287,7 @@ checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8"
 dependencies = [
  "cfg-if",
  "log",
- "pin-project-lite",
+ "pin-project-lite 0.2.9",
  "tracing-attributes",
  "tracing-core",
 ]
diff --git a/Cargo.toml b/Cargo.toml
index 8f670b25..152370b6 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -26,7 +26,7 @@ hyper = "0.14" # Prometheus server
 iri-string = "0.5"
 lazy_static = "1.4.0"
 libipld = "0.14"
-libp2p = { default-features = false, features = ["autonat", "gossipsub", "identify", "kad", "tcp-tokio", "mplex", "noise", "ping", "yamux", "dns-tokio", "relay", "dcutr"], version = "0.49.0" }
+libp2p = { default-features = false, features = ["autonat", "websocket", "wasm-ext", "gossipsub", "identify", "kad", "tcp-tokio", "mplex", "noise", "ping", "yamux", "dns-tokio", "relay", "dcutr"], version = "0.49.0" }
 opentelemetry = { version = "0.17.0", features = ["rt-tokio"] }
 opentelemetry-jaeger = { version = "0.16.0", features = ["rt-tokio", "reqwest_collector_client"] }
 pin-project = "1"
diff --git a/src/p2p/transport.rs b/src/p2p/transport.rs
new file mode 100644
index 00000000..24f89439
--- /dev/null
+++ b/src/p2p/transport.rs
@@ -0,0 +1,88 @@
+use derive_builder::Builder;
+use libp2p::{
+    core::transport::{dummy::DummyTransport, MemoryTransport},
+    dns::{ResolverConfig, ResolverOpts},
+    identity::Keypair,
+    mplex::MplexConfig,
+    tcp::GenTcpConfig,
+    wasm_ext::ffi,
+    websocket::tls::Config as WsTlsConfig,
+    yamux::YamuxConfig,
+    relay::v2::client::{Client, transport::ClientTransport}
+};
+use std::time::Duration;
+
+#[derive(Clone, Debug)]
+pub enum DnsResolver {
+    System,
+    Custom(DnsConfig),
+}
+
+impl Default for DnsResolver {
+    fn default() -> Self {
+        Self::System
+    }
+}
+
+#[derive(Builder, Debug, Clone)]
+pub struct DnsConfig {
+    conf: ResolverConfig,
+    opts: ResolverOpts,
+}
+
+pub const MAX_DATA_SIZE: usize = 256 * 1024 * 1024;
+
+fn client() -> WsTlsConfig {
+    WsTlsConfig::client()
+}
+
+#[derive(Builder, Debug, Clone)]
+#[builder(
+    build_fn(skip),
+    setter(into),
+    derive(Debug),
+    name = "WsConfig",
+    pattern = "owned"
+)]
+pub struct WsConfigDummy {
+    #[builder(field(type = "u8"), default = "0")]
+    max_redirects: u8,
+    #[builder(field(type = "usize"), default = "MAX_DATA_SIZE")]
+    max_data_size: usize,
+    #[builder(field(type = "bool"), default = "false")]
+    deflate: bool,
+    #[builder(field(type = "WsTlsConfig"), default = "client()")]
+    tls: WsTlsConfig,
+}
+
+#[derive(Builder)]
+#[builder(
+    build_fn(skip),
+    setter(into),
+    name = "TransportConfig",
+    pattern = "owned"
+)]
+pub struct TransportConfigDummy {
+    #[builder(field(type = "bool"), default = "false")]
+    memory: bool,
+    tcp: GenTcpConfig,
+    external: ffi::Transport,
+    dns: DnsResolver,
+    websocket: WsConfig,
+    #[builder(field(type = "bool"), default = "false")]
+    relay: bool,
+    mplex: MplexConfig,
+    yamux: YamuxConfig,
+    timeout_incoming: Duration,
+    timeout_outgoing: Duration,
+}
+
+impl TransportConfig {
+    fn build(self, keypair: Keypair) -> (Boxed<(PeerId, StreamMuxerBox)>, Option<Client>) {
+        let d = match (&self.memory, &self.tcp, &self.external, &self.relay) {
+            (false, None, None, false) => DummyTransport::new(),
+            (true, None) => MemoryTransport::new(),
+
+        }
+    }
+}

From 23fdec11c10d32be063c757291f7abc19aa406a4 Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Thu, 10 Nov 2022 14:41:43 +0100
Subject: [PATCH 65/73] make transport configs actually convenient

---
 src/p2p/transport.rs | 251 ++++++++++++++++++++++++++++++++-----------
 1 file changed, 190 insertions(+), 61 deletions(-)

diff --git a/src/p2p/transport.rs b/src/p2p/transport.rs
index 24f89439..73366bb5 100644
--- a/src/p2p/transport.rs
+++ b/src/p2p/transport.rs
@@ -1,21 +1,54 @@
+use crate::storage::either::EitherError;
 use derive_builder::Builder;
+use futures::io::{AsyncRead, AsyncWrite};
 use libp2p::{
-    core::transport::{dummy::DummyTransport, MemoryTransport},
-    dns::{ResolverConfig, ResolverOpts},
-    identity::Keypair,
-    mplex::MplexConfig,
-    tcp::GenTcpConfig,
-    wasm_ext::ffi,
-    websocket::tls::Config as WsTlsConfig,
-    yamux::YamuxConfig,
-    relay::v2::client::{Client, transport::ClientTransport}
+    core::transport::{dummy::DummyTransport, MemoryTransport, OrTransport, Transport},
+    dns::{ResolverConfig, ResolverOpts, TokioDnsConfig as DnsTransport},
+    tcp::TokioTcpTransport as TcpTransport,
+    wasm_ext::ExtTransport,
+    websocket::{tls::Config as WsTlsConfig, WsConfig as WsTransport},
 };
-use std::time::Duration;
+use std::io::Error as IoError;
+
+pub trait IntoTransport {
+    type T: Transport;
+    type Error;
+    fn into_transport(self) -> Result<Self::T, Self::Error>;
+}
+
+pub use dns::{CustomDnsResolver, DnsConfig};
+pub use libp2p::tcp::GenTcpConfig as TcpConfig;
+pub use libp2p::wasm_ext::ffi::Transport as ExtConfig;
+pub use ws::{WsConfig, WS_MAX_DATA_SIZE};
+
+#[derive(Default, Debug, Clone, Hash, PartialEq, Eq)]
+pub struct MemoryConfig;
+
+#[derive(Default, Debug, Clone, Hash, PartialEq, Eq)]
+pub struct DummyConfig;
+
+#[derive(Default, Debug, Clone, Hash, PartialEq, Eq)]
+pub struct Both<A, B>(pub A, pub B);
+
+impl<A, B> IntoTransport for Both<A, B>
+where
+    A: IntoTransport,
+    B: IntoTransport,
+{
+    type T = OrTransport<A::T, B::T>;
+    type Error = EitherError<A::Error, B::Error>;
+    fn into_transport(self) -> Result<Self::T, Self::Error> {
+        Ok(OrTransport::new(
+            self.0.into_transport().map_err(EitherError::A)?,
+            self.1.into_transport().map_err(EitherError::B)?,
+        ))
+    }
+}
 
 #[derive(Clone, Debug)]
 pub enum DnsResolver {
     System,
-    Custom(DnsConfig),
+    Custom(CustomDnsResolver),
 }
 
 impl Default for DnsResolver {
@@ -24,65 +57,161 @@ impl Default for DnsResolver {
     }
 }
 
-#[derive(Builder, Debug, Clone)]
-pub struct DnsConfig {
-    conf: ResolverConfig,
-    opts: ResolverOpts,
+mod dns {
+    use super::*;
+    #[derive(Builder)]
+    #[builder(
+        build_fn(skip),
+        setter(into),
+        derive(Debug),
+        name = "CustomDnsResolver"
+    )]
+    pub struct CustomDnsResolverDummy {
+        #[builder(field(type = "ResolverConfig"))]
+        conf: ResolverConfig,
+        #[builder(field(type = "ResolverOpts"))]
+        opts: ResolverOpts,
+    }
+
+    #[derive(Builder)]
+    #[builder(build_fn(skip), setter(into), derive(Debug), name = "DnsConfig")]
+    pub struct DnsConfigDummy<B>
+    where
+        B: Default,
+    {
+        #[builder(field(type = "DnsResolver"))]
+        resolver: DnsResolver,
+        #[builder(field(type = "B"))]
+        base: B,
+    }
+
+    pub fn convert<B>(c: DnsConfig<B>) -> Result<DnsTransport<B::T>, EitherError<IoError, B::Error>>
+    where
+        B: Default + IntoTransport,
+        B::T: 'static + Send + Unpin,
+        <B::T as Transport>::Output: 'static + AsyncRead + AsyncWrite + Send + Unpin,
+        <B::T as Transport>::Dial: Send,
+        <B::T as Transport>::Error: Send,
+    {
+        match c.resolver {
+            DnsResolver::System => {
+                DnsTransport::system(c.base.into_transport().map_err(EitherError::B)?)
+                    .map_err(EitherError::A)
+            }
+            DnsResolver::Custom(custom) => DnsTransport::custom(
+                c.base.into_transport().map_err(EitherError::B)?,
+                custom.conf,
+                custom.opts,
+            )
+            .map_err(EitherError::A),
+        }
+    }
 }
 
-pub const MAX_DATA_SIZE: usize = 256 * 1024 * 1024;
+impl<B> IntoTransport for DnsConfig<B>
+where
+    B: Default + IntoTransport,
+    B::T: 'static + Send + Unpin,
+    <B::T as Transport>::Output: 'static + AsyncRead + AsyncWrite + Send + Unpin,
+    <B::T as Transport>::Dial: Send,
+    <B::T as Transport>::Error: Send,
+{
+    type T = DnsTransport<B::T>;
+    type Error = EitherError<IoError, B::Error>;
+    fn into_transport(self) -> Result<Self::T, Self::Error> {
+        dns::convert(self)
+    }
+}
+
+mod ws {
+    use super::*;
+    pub const WS_MAX_DATA_SIZE: usize = 256 * 1024 * 1024;
+
+    fn client() -> WsTlsConfig {
+        WsTlsConfig::client()
+    }
 
-fn client() -> WsTlsConfig {
-    WsTlsConfig::client()
+    #[derive(Builder)]
+    #[builder(build_fn(skip), setter(into), derive(Debug), name = "WsConfig")]
+    pub struct WsConfigDummy<T>
+    where
+        T: Default,
+    {
+        #[builder(field(type = "T"))]
+        base: T,
+        #[builder(field(type = "u8"), default = "0")]
+        max_redirects: u8,
+        #[builder(field(type = "usize"), default = "WS_MAX_DATA_SIZE")]
+        max_data_size: usize,
+        #[builder(field(type = "bool"), default = "false")]
+        deflate: bool,
+        // TODO this is cause some kind of error cos it has no Default
+        // #[builder(field(type = "WsTlsConfig"), default = "client()")]
+        // tls: WsTlsConfig,
+    }
+
+    pub fn convert<B>(c: WsConfig<B>) -> Result<WsTransport<B::T>, B::Error>
+    where
+        B: Default + IntoTransport,
+        B::T: 'static + Send + Unpin,
+        <B::T as Transport>::Output: 'static + AsyncRead + AsyncWrite + Send + Unpin,
+        <B::T as Transport>::Dial: Send,
+        <B::T as Transport>::Error: Send,
+        <B::T as Transport>::ListenerUpgrade: Send,
+    {
+        let mut ws = WsTransport::new(c.base.into_transport()?);
+        ws.set_max_redirects(c.max_redirects)
+            .set_max_data_size(c.max_data_size)
+            // .set_tls_config(c.tls)
+            .use_deflate(c.deflate);
+        Ok(ws)
+    }
 }
 
-#[derive(Builder, Debug, Clone)]
-#[builder(
-    build_fn(skip),
-    setter(into),
-    derive(Debug),
-    name = "WsConfig",
-    pattern = "owned"
-)]
-pub struct WsConfigDummy {
-    #[builder(field(type = "u8"), default = "0")]
-    max_redirects: u8,
-    #[builder(field(type = "usize"), default = "MAX_DATA_SIZE")]
-    max_data_size: usize,
-    #[builder(field(type = "bool"), default = "false")]
-    deflate: bool,
-    #[builder(field(type = "WsTlsConfig"), default = "client()")]
-    tls: WsTlsConfig,
+impl<B> IntoTransport for WsConfig<B>
+where
+    B: Default + IntoTransport,
+    B::T: 'static + Send + Unpin,
+    <B::T as Transport>::Output: 'static + AsyncRead + AsyncWrite + Send + Unpin,
+    <B::T as Transport>::Dial: Send,
+    <B::T as Transport>::Error: Send,
+    <B::T as Transport>::ListenerUpgrade: Send,
+{
+    type T = WsTransport<B::T>;
+    type Error = B::Error;
+    fn into_transport(self) -> Result<Self::T, Self::Error> {
+        ws::convert(self)
+    }
 }
 
-#[derive(Builder)]
-#[builder(
-    build_fn(skip),
-    setter(into),
-    name = "TransportConfig",
-    pattern = "owned"
-)]
-pub struct TransportConfigDummy {
-    #[builder(field(type = "bool"), default = "false")]
-    memory: bool,
-    tcp: GenTcpConfig,
-    external: ffi::Transport,
-    dns: DnsResolver,
-    websocket: WsConfig,
-    #[builder(field(type = "bool"), default = "false")]
-    relay: bool,
-    mplex: MplexConfig,
-    yamux: YamuxConfig,
-    timeout_incoming: Duration,
-    timeout_outgoing: Duration,
+impl IntoTransport for ExtConfig {
+    type T = ExtTransport;
+    type Error = std::convert::Infallible;
+    fn into_transport(self) -> Result<Self::T, Self::Error> {
+        Ok(Self::T::new(self))
+    }
 }
 
-impl TransportConfig {
-    fn build(self, keypair: Keypair) -> (Boxed<(PeerId, StreamMuxerBox)>, Option<Client>) {
-        let d = match (&self.memory, &self.tcp, &self.external, &self.relay) {
-            (false, None, None, false) => DummyTransport::new(),
-            (true, None) => MemoryTransport::new(),
+impl IntoTransport for MemoryConfig {
+    type T = MemoryTransport;
+    type Error = std::convert::Infallible;
+    fn into_transport(self) -> Result<Self::T, Self::Error> {
+        Ok(Self::T::new())
+    }
+}
 
-        }
+impl IntoTransport for TcpConfig {
+    type T = TcpTransport;
+    type Error = std::convert::Infallible;
+    fn into_transport(self) -> Result<Self::T, Self::Error> {
+        Ok(Self::T::new(self))
+    }
+}
+
+impl IntoTransport for DummyConfig {
+    type T = DummyTransport;
+    type Error = std::convert::Infallible;
+    fn into_transport(self) -> Result<Self::T, Self::Error> {
+        Ok(Self::T::new())
     }
 }

From c170f2655ee8236d0244a8487b1f2efd053687df Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Thu, 10 Nov 2022 14:52:36 +0100
Subject: [PATCH 66/73] fix errors

---
 src/p2p/mod.rs   | 21 ++++++++++++---------
 src/p2p/relay.rs |  9 ++++++---
 2 files changed, 18 insertions(+), 12 deletions(-)

diff --git a/src/p2p/mod.rs b/src/p2p/mod.rs
index aa2ffcf5..6f3fd3a4 100644
--- a/src/p2p/mod.rs
+++ b/src/p2p/mod.rs
@@ -25,29 +25,32 @@ mod builder {
         #[builder(field(type = "usize"), default = "0")]
         cache_size: usize,
     }
-    pub fn convert(config: IdentifyConfig, key: PublicKey) -> OIdentifyConfig {
+    pub fn to_config(config: IdentifyConfig, key: PublicKey) -> OIdentifyConfig {
         OIdentifyConfig::new(config.protocol_version, key)
             .with_initial_delay(config.initial_delay)
             .with_interval(config.interval)
             .with_push_listen_addr_updates(config.push_listen_addr_updates)
             .with_cache_size(config.cache_size)
     }
+    pub fn convert(c: OIdentifyConfig) -> IdentifyConfig {
+        IdentifyConfig {
+            protocol_version: c.protocol_version,
+            initial_delay: c.initial_delay,
+            interval: c.interval,
+            push_listen_addr_updates: c.push_listen_addr_updates,
+            cache_size: c.cache_size,
+        }
+    }
 }
 
 impl IdentifyConfig {
     fn to_config(self, key: PublicKey) -> OIdentifyConfig {
-        builder::convert(self, key)
+        builder::to_config(self, key)
     }
 }
 
 impl From<OIdentifyConfig> for IdentifyConfig {
     fn from(c: OIdentifyConfig) -> Self {
-        Self {
-            protocol_version: c.protocol_version,
-            initial_delay: c.initial_delay,
-            interval: c.interval,
-            push_listen_addr_updates: c.push_listen_addr_updates,
-            cache_size: c.cache_size,
-        }
+        builder::convert(c)
     }
 }
diff --git a/src/p2p/relay.rs b/src/p2p/relay.rs
index 1ff9542c..98a5a79e 100644
--- a/src/p2p/relay.rs
+++ b/src/p2p/relay.rs
@@ -1,4 +1,7 @@
-use crate::{orbit::AbortOnDrop, p2p::IdentifyConfig};
+use crate::{
+    orbit::AbortOnDrop,
+    p2p::{transport::IntoTransport, IdentifyConfig},
+};
 use anyhow::Result;
 use futures::{
     channel::{mpsc, oneshot},
@@ -33,7 +36,7 @@ pub struct RelayNode {
 }
 
 #[derive(NetworkBehaviour)]
-struct Behaviour {
+pub struct Behaviour {
     identify: Identify,
     ping: Ping,
     relay: Relay,
@@ -137,7 +140,7 @@ mod builder {
             let local_public_key = keypair.public();
             let id = local_public_key.to_peer_id();
             let b = self.build(local_public_key);
-            let (sender, reciever) = mpsc::channel(100);
+            let (sender, mut reciever) = mpsc::channel(100);
             let r = RelayNode { id, sender, port };
 
             let mut swarm = SwarmBuilder::new(

From ee2a7eb7aa921f5d601fda95e916c6b7b20f0275 Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Thu, 10 Nov 2022 14:52:44 +0100
Subject: [PATCH 67/73] use IntoTransport in relay::launch

---
 src/lib.rs       | 11 +++++++++--
 src/p2p/relay.rs | 22 ++++++++++------------
 2 files changed, 19 insertions(+), 14 deletions(-)

diff --git a/src/lib.rs b/src/lib.rs
index 36920921..75ccbce2 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -33,7 +33,10 @@ use libp2p::{
     PeerId,
 };
 use orbit::ProviderUtils;
-use p2p::relay::{Config as RelayConfig, RelayNode};
+use p2p::{
+    relay::{Config as RelayConfig, RelayNode},
+    transport::{Both, DnsConfig, MemoryConfig, TcpConfig, WsConfig},
+};
 use routes::{delegate, invoke, open_host_key, relay_addr, util_routes::*};
 use std::{collections::HashMap, sync::RwLock};
 use storage::{
@@ -78,7 +81,11 @@ pub async fn app(config: &Figment) -> Result<Rocket<Build>> {
     storage::KV::healthcheck(kepler_config.storage.indexes.clone()).await?;
     let kp = kepler_config.storage.blocks.relay_key_pair().await?;
 
-    let relay_node = RelayNode::new(kepler_config.relay.port, Keypair::Ed25519(kp)).await?;
+    let relay_node = RelayConfig::default().launch(
+        Keypair::Ed25519(kp),
+        Both::<MemoryConfig, WsConfig<DnsConfig<TcpConfig>>>::default(),
+        kepler_config.relay.port,
+    )?;
 
     let routes = routes![
         healthcheck,
diff --git a/src/p2p/relay.rs b/src/p2p/relay.rs
index 98a5a79e..e2ad96ac 100644
--- a/src/p2p/relay.rs
+++ b/src/p2p/relay.rs
@@ -123,19 +123,15 @@ mod builder {
             }
         }
 
-        pub fn launch<T>(
-            self,
-            keypair: Keypair,
-            transport: Builder<T>,
-            port: u16,
-        ) -> Result<RelayNode>
+        pub fn launch<T>(self, keypair: Keypair, transport: T, port: u16) -> Result<RelayNode>
         where
-            T: Transport + Send,
-            T::Output: AsyncRead + AsyncWrite + Unpin + Send,
-            T::Error: Send + Sync,
-            T: Unpin,
-            T::Dial: Send,
-            T::ListenerUpgrade: Send,
+            T: IntoTransport,
+            T::T: 'static + Send + Unpin,
+            T::Error: 'static + std::error::Error + Send + Sync,
+            <T::T as Transport>::Output: 'static + AsyncRead + AsyncWrite + Unpin + Send,
+            <T::T as Transport>::Error: 'static + Send + Sync,
+            <T::T as Transport>::Dial: Send,
+            <T::T as Transport>::ListenerUpgrade: Send,
         {
             let local_public_key = keypair.public();
             let id = local_public_key.to_peer_id();
@@ -145,6 +141,8 @@ mod builder {
 
             let mut swarm = SwarmBuilder::new(
                 transport
+                    .into_transport()?
+                    .upgrade(upgrade::Version::V1)
                     .authenticate(noise::NoiseAuthenticated::xx(&keypair).unwrap())
                     .multiplex(upgrade::SelectUpgrade::new(
                         yamux::YamuxConfig::default(),

From b8c7c86c5bec811972d2d7b6ef4f89d80ad54fdb Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Thu, 10 Nov 2022 16:07:59 +0100
Subject: [PATCH 68/73] clean up some unused builder stuff

---
 src/p2p/behaviour/builder.rs |  2 +-
 src/p2p/mod.rs               |  4 ++--
 src/p2p/relay.rs             | 11 ++++-------
 3 files changed, 7 insertions(+), 10 deletions(-)

diff --git a/src/p2p/behaviour/builder.rs b/src/p2p/behaviour/builder.rs
index 86db2b30..ca9809fc 100644
--- a/src/p2p/behaviour/builder.rs
+++ b/src/p2p/behaviour/builder.rs
@@ -20,7 +20,7 @@ use thiserror::Error;
 
 // we use derive_builder here to make a conveniant builder, but we do not export
 // the actual config struct
-#[derive(Builder, Clone, Debug)]
+#[derive(Builder)]
 #[builder(build_fn(skip), setter(into), name = "BehaviourConfig", derive(Debug))]
 pub struct BehaviourConfigDummy<KSC = MemoryStoreConfig>
 where
diff --git a/src/p2p/mod.rs b/src/p2p/mod.rs
index 6f3fd3a4..51258dee 100644
--- a/src/p2p/mod.rs
+++ b/src/p2p/mod.rs
@@ -3,7 +3,7 @@ pub mod behaviour;
 pub mod relay;
 pub mod transport;
 
-const PROTOCOL_VERSION: &'static str = "kepler/0.1.0";
+pub const PROTOCOL_VERSION: &'static str = "kepler/0.1.0";
 
 pub use builder::IdentifyConfig;
 
@@ -11,7 +11,7 @@ mod builder {
     use core::time::Duration;
     use derive_builder::Builder;
     use libp2p::{identify::Config as OIdentifyConfig, identity::PublicKey};
-    #[derive(Builder, Default, Debug, Clone)]
+    #[derive(Builder)]
     #[builder(build_fn(skip), setter(into), name = "IdentifyConfig", derive(Debug))]
     pub struct IdentifyConfigDummy {
         #[builder(field(type = "String"), default = "PROTOCOL_VERSION.into()")]
diff --git a/src/p2p/relay.rs b/src/p2p/relay.rs
index e2ad96ac..c45e7553 100644
--- a/src/p2p/relay.rs
+++ b/src/p2p/relay.rs
@@ -7,15 +7,12 @@ use futures::{
     channel::{mpsc, oneshot},
     future::{select, Either},
     io::{AsyncRead, AsyncWrite},
-    sink::{Sink, SinkExt},
-    stream::{iter, Stream, StreamExt},
+    sink::SinkExt,
+    stream::{iter, StreamExt},
 };
 use libp2p::{
     autonat::Behaviour as AutoNat,
-    core::{
-        identity::Keypair, multiaddr::multiaddr, transport::upgrade::Builder, upgrade, Multiaddr,
-        PeerId,
-    },
+    core::{identity::Keypair, upgrade, Multiaddr, PeerId},
     identify::Behaviour as Identify,
     mplex, noise,
     ping::Behaviour as Ping,
@@ -93,7 +90,7 @@ mod builder {
         ping::Config as PingConfig, relay::v2::relay::Config as RelayConfig,
     };
 
-    #[derive(Builder, Debug)]
+    #[derive(Builder)]
     #[builder(
         build_fn(skip),
         setter(into),

From 3c9902d890005b0a1ff8142ea3974bbc73e61388 Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Thu, 10 Nov 2022 16:08:21 +0100
Subject: [PATCH 69/73] get addrs and listen with relay via the event channels

---
 src/auth_guards.rs | 15 ++++++-----
 src/lib.rs         | 13 +++++++---
 src/p2p/relay.rs   | 62 ++++++++++++++++++++++------------------------
 src/routes/mod.rs  | 15 ++++++-----
 4 files changed, 58 insertions(+), 47 deletions(-)

diff --git a/src/auth_guards.rs b/src/auth_guards.rs
index b4d84d7f..954ec60d 100644
--- a/src/auth_guards.rs
+++ b/src/auth_guards.rs
@@ -11,7 +11,7 @@ use kepler_lib::{
     resource::{OrbitId, ResourceId},
 };
 use libp2p::{
-    core::{Multiaddr, PeerId},
+    core::{multiaddr::multiaddr, Multiaddr, PeerId},
     identity::ed25519::Keypair as Ed25519Keypair,
 };
 use rocket::{
@@ -74,14 +74,17 @@ pub fn check_orbit_and_service(
 }
 
 fn get_state(req: &Request<'_>) -> Result<(config::Config, (PeerId, Multiaddr))> {
+    let config = req
+        .rocket()
+        .state::<config::Config>()
+        .cloned()
+        .ok_or_else(|| anyhow!("Could not retrieve config"))?;
+    let port = config.relay.port;
     Ok((
-        req.rocket()
-            .state::<config::Config>()
-            .cloned()
-            .ok_or_else(|| anyhow!("Could not retrieve config"))?,
+        config,
         req.rocket()
             .state::<RelayNode>()
-            .map(|r| (r.id().clone(), r.internal()))
+            .map(|r| (r.id().clone(), multiaddr!(Memory(port))))
             .ok_or_else(|| anyhow!("Could not retrieve relay node information"))?,
     ))
 }
diff --git a/src/lib.rs b/src/lib.rs
index 75ccbce2..2f7278a1 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -29,12 +29,13 @@ pub mod transport;
 
 use config::{BlockStorage, Config};
 use libp2p::{
+    core::multiaddr::multiaddr,
     identity::{ed25519::Keypair as Ed25519Keypair, Keypair},
     PeerId,
 };
 use orbit::ProviderUtils;
 use p2p::{
-    relay::{Config as RelayConfig, RelayNode},
+    relay::Config as RelayConfig,
     transport::{Both, DnsConfig, MemoryConfig, TcpConfig, WsConfig},
 };
 use routes::{delegate, invoke, open_host_key, relay_addr, util_routes::*};
@@ -81,12 +82,18 @@ pub async fn app(config: &Figment) -> Result<Rocket<Build>> {
     storage::KV::healthcheck(kepler_config.storage.indexes.clone()).await?;
     let kp = kepler_config.storage.blocks.relay_key_pair().await?;
 
-    let relay_node = RelayConfig::default().launch(
+    let mut relay_node = RelayConfig::default().launch(
         Keypair::Ed25519(kp),
         Both::<MemoryConfig, WsConfig<DnsConfig<TcpConfig>>>::default(),
-        kepler_config.relay.port,
     )?;
 
+    relay_node
+        .listen_on([
+            multiaddr!(Memory(kepler_config.relay.port)),
+            multiaddr!(Ip4([127, 0, 0, 1]), Tcp(kepler_config.relay.port)),
+        ])
+        .await?;
+
     let routes = routes![
         healthcheck,
         cors,
diff --git a/src/p2p/relay.rs b/src/p2p/relay.rs
index c45e7553..6cd9ed83 100644
--- a/src/p2p/relay.rs
+++ b/src/p2p/relay.rs
@@ -29,7 +29,6 @@ pub type RelaySwarm = Swarm<Behaviour>;
 pub struct RelayNode {
     id: PeerId,
     sender: mpsc::Sender<Message>,
-    port: u16,
 }
 
 #[derive(NetworkBehaviour)]
@@ -43,42 +42,25 @@ pub struct Behaviour {
 #[derive(Debug)]
 pub enum Message {
     GetAddresses(oneshot::Sender<Vec<Multiaddr>>),
-    ListenOn(Multiaddr),
+    ListenOn(Vec<Multiaddr>, oneshot::Sender<Result<()>>),
 }
 
 impl RelayNode {
     pub fn id(&self) -> &PeerId {
         &self.id
     }
-    async fn get_addresses(&mut self) -> Result<Vec<Multiaddr>> {
+    pub async fn get_addresses(&mut self) -> Result<Vec<Multiaddr>> {
         let (s, r) = oneshot::channel();
         self.sender.send(Message::GetAddresses(s)).await?;
         Ok(r.await?)
     }
 
-    async fn listen_on(&mut self, addr: impl IntoIterator<Item = &Multiaddr>) -> Result<()> {
-        Ok(self
-            .sender
-            .send_all(&mut iter(
-                addr.into_iter().map(|a| Ok(Message::ListenOn(a.clone()))),
-            ))
-            .await?)
-    }
-
-    fn _internal(port: u16) -> Multiaddr {
-        multiaddr!(Memory(port))
-    }
-
-    pub fn internal(&self) -> Multiaddr {
-        Self::_internal(self.port)
-    }
-
-    fn _external(port: u16) -> Multiaddr {
-        multiaddr!(Ip4([127, 0, 0, 1]), Tcp(port))
-    }
-
-    pub fn external(&self) -> Multiaddr {
-        Self::_external(self.port)
+    pub async fn listen_on(&mut self, addr: impl IntoIterator<Item = Multiaddr>) -> Result<()> {
+        let (s, r) = oneshot::channel();
+        self.sender
+            .send(Message::ListenOn(addr.into_iter().collect(), s))
+            .await?;
+        r.await?
     }
 }
 
@@ -120,7 +102,7 @@ mod builder {
             }
         }
 
-        pub fn launch<T>(self, keypair: Keypair, transport: T, port: u16) -> Result<RelayNode>
+        pub fn launch<T>(self, keypair: Keypair, transport: T) -> Result<RelayNode>
         where
             T: IntoTransport,
             T::T: 'static + Send + Unpin,
@@ -134,7 +116,7 @@ mod builder {
             let id = local_public_key.to_peer_id();
             let b = self.build(local_public_key);
             let (sender, mut reciever) = mpsc::channel(100);
-            let r = RelayNode { id, sender, port };
+            let r = RelayNode { id, sender };
 
             let mut swarm = SwarmBuilder::new(
                 transport
@@ -152,9 +134,6 @@ mod builder {
             )
             .build();
 
-            swarm.listen_on(r.external())?;
-            swarm.listen_on(r.internal())?;
-
             tokio::spawn(async move {
                 loop {
                     match select(reciever.next(), swarm.next()).await {
@@ -164,7 +143,26 @@ mod builder {
                         }
                         // process command
                         Either::Left((Some(e), _)) => match e {
-                            Message::ListenOn(a) => swarm.listen_on(a).map(|_| ())?,
+                            Message::ListenOn(a, s) => {
+                                match a.into_iter().try_fold(Vec::new(), |mut listeners, addr| {
+                                    match swarm.listen_on(addr) {
+                                        Ok(l) => {
+                                            listeners.push(l);
+                                            Ok(listeners)
+                                        }
+                                        Err(e) => Err((e, listeners)),
+                                    }
+                                }) {
+                                    Ok(_) => s.send(Ok(())),
+                                    Err((e, listeners)) => {
+                                        for l in listeners {
+                                            swarm.remove_listener(l);
+                                        }
+                                        s.send(Err(e.into()))
+                                    }
+                                }
+                                .map_err(|_| anyhow!("failed to return listening result"))?
+                            }
                             Message::GetAddresses(s) => {
                                 s.send(swarm.listeners().map(|a| a.clone()).collect())
                                     .map_err(|_| anyhow!("failed to return listeners"))?;
diff --git a/src/routes/mod.rs b/src/routes/mod.rs
index 2185e48a..bc699d0f 100644
--- a/src/routes/mod.rs
+++ b/src/routes/mod.rs
@@ -4,7 +4,7 @@ use kepler_lib::libipld::Cid;
 use libp2p::{
     core::PeerId,
     identity::{ed25519::Keypair as Ed25519Keypair, PublicKey},
-    multiaddr::Protocol,
+    multiaddr::multiaddr,
 };
 use rocket::{
     data::{Data, ToByteUnit},
@@ -22,6 +22,7 @@ use tracing::{info_span, Instrument};
 
 use crate::{
     auth_guards::{DelegateAuthWrapper, InvokeAuthWrapper, KVAction},
+    config::Config,
     kv::{ObjectBuilder, ReadResponse},
     p2p::relay::RelayNode,
     storage::ImmutableStore,
@@ -87,11 +88,13 @@ pub mod util_routes {
 }
 
 #[get("/peer/relay")]
-pub fn relay_addr(relay: &State<RelayNode>) -> String {
-    relay
-        .external()
-        .with(Protocol::P2p(relay.id().clone().into()))
-        .to_string()
+pub fn relay_addr(relay: &State<RelayNode>, config: &State<Config>) -> String {
+    multiaddr!(
+        Ip4([127, 0, 0, 1]),
+        Tcp(config.relay.port),
+        P2p(relay.id().clone())
+    )
+    .to_string()
 }
 
 #[get("/peer/generate")]

From 2c179e46d8ba098414c692766ca90172d9bb161c Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Wed, 23 Nov 2022 12:00:58 +0100
Subject: [PATCH 70/73] comments, add vanilla dns to kepler relay peer

---
 src/lib.rs           | 3 ++-
 src/p2p/relay.rs     | 6 ++++--
 src/p2p/transport.rs | 6 ++++++
 3 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/src/lib.rs b/src/lib.rs
index 2f7278a1..b3111160 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -84,7 +84,8 @@ pub async fn app(config: &Figment) -> Result<Rocket<Build>> {
 
     let mut relay_node = RelayConfig::default().launch(
         Keypair::Ed25519(kp),
-        Both::<MemoryConfig, WsConfig<DnsConfig<TcpConfig>>>::default(),
+        // transport for memory, dns + tcp and websocket over dns + tcp
+        Both::<MemoryConfig, Both<DnsConfig<TcpConfig>, WsConfig<DnsConfig<TcpConfig>>>>::default(),
     )?;
 
     relay_node
diff --git a/src/p2p/relay.rs b/src/p2p/relay.rs
index 6cd9ed83..995dcd0e 100644
--- a/src/p2p/relay.rs
+++ b/src/p2p/relay.rs
@@ -102,7 +102,7 @@ mod builder {
             }
         }
 
-        pub fn launch<T>(self, keypair: Keypair, transport: T) -> Result<RelayNode>
+        pub fn launch<T>(self, keypair: Keypair, transport: T) -> Result<RelayNode, T::Error>
         where
             T: IntoTransport,
             T::T: 'static + Send + Unpin,
@@ -144,6 +144,7 @@ mod builder {
                         // process command
                         Either::Left((Some(e), _)) => match e {
                             Message::ListenOn(a, s) => {
+                                // try listen on each given address
                                 match a.into_iter().try_fold(Vec::new(), |mut listeners, addr| {
                                     match swarm.listen_on(addr) {
                                         Ok(l) => {
@@ -154,6 +155,7 @@ mod builder {
                                     }
                                 }) {
                                     Ok(_) => s.send(Ok(())),
+                                    // if one fails, roll back all of them
                                     Err((e, listeners)) => {
                                         for l in listeners {
                                             swarm.remove_listener(l);
@@ -161,7 +163,7 @@ mod builder {
                                         s.send(Err(e.into()))
                                     }
                                 }
-                                .map_err(|_| anyhow!("failed to return listening result"))?
+                                .map_err(|_| anyhow!("failed to return listening result"))?;
                             }
                             Message::GetAddresses(s) => {
                                 s.send(swarm.listeners().map(|a| a.clone()).collect())
diff --git a/src/p2p/transport.rs b/src/p2p/transport.rs
index 73366bb5..a046ef98 100644
--- a/src/p2p/transport.rs
+++ b/src/p2p/transport.rs
@@ -14,6 +14,12 @@ pub trait IntoTransport {
     type T: Transport;
     type Error;
     fn into_transport(self) -> Result<Self::T, Self::Error>;
+    fn and<O: IntoTransport>(self, other: O) -> Both<Self, O>
+    where
+        Self: Sized,
+    {
+        Both(self, other)
+    }
 }
 
 pub use dns::{CustomDnsResolver, DnsConfig};

From 94432acc1341945126dac86b7f5c896d9855f666 Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Wed, 23 Nov 2022 12:34:13 +0100
Subject: [PATCH 71/73] remove currently unnecessary modules

---
 src/p2p/behaviour/builder.rs | 98 ------------------------------------
 src/p2p/behaviour/mod.rs     | 33 ------------
 src/p2p/mod.rs               |  1 -
 src/p2p/relay.rs             | 98 ------------------------------------
 4 files changed, 230 deletions(-)
 delete mode 100644 src/p2p/behaviour/builder.rs
 delete mode 100644 src/p2p/behaviour/mod.rs

diff --git a/src/p2p/behaviour/builder.rs b/src/p2p/behaviour/builder.rs
deleted file mode 100644
index ca9809fc..00000000
--- a/src/p2p/behaviour/builder.rs
+++ /dev/null
@@ -1,98 +0,0 @@
-use crate::p2p::{behaviour::Behaviour, IdentifyConfig};
-use derive_builder::Builder;
-use libp2p::{
-    autonat::{Behaviour as AutoNat, Config as AutoNatConfig},
-    core::PeerId,
-    dcutr::behaviour::Behaviour as Dcutr,
-    gossipsub::{
-        Gossipsub, GossipsubConfig, GossipsubConfigBuilder, MessageAuthenticity, ValidationMode,
-    },
-    identify::Behaviour as Identify,
-    identity::Keypair,
-    kad::{
-        record::store::{MemoryStore, MemoryStoreConfig, RecordStore},
-        Kademlia, KademliaConfig,
-    },
-    ping::{Behaviour as Ping, Config as PingConfig},
-    relay::v2::client::Client,
-};
-use thiserror::Error;
-
-// we use derive_builder here to make a conveniant builder, but we do not export
-// the actual config struct
-#[derive(Builder)]
-#[builder(build_fn(skip), setter(into), name = "BehaviourConfig", derive(Debug))]
-pub struct BehaviourConfigDummy<KSC = MemoryStoreConfig>
-where
-    KSC: Default,
-{
-    #[builder(field(type = "IdentifyConfig"))]
-    identify: IdentifyConfig,
-    #[builder(field(type = "PingConfig"))]
-    ping: PingConfig,
-    #[builder(field(type = "GossipsubConfig"))]
-    gossipsub: GossipsubConfig,
-    #[builder(field(type = "KademliaConfig"))]
-    kademlia: KademliaConfig,
-    #[builder(field(type = "KSC"))]
-    kademlia_store: KSC,
-    #[builder(field(type = "AutoNatConfig"))]
-    autonat: AutoNatConfig,
-}
-
-impl<KSC> BehaviourConfig<KSC>
-where
-    KSC: Default,
-{
-    pub fn build<KS>(
-        self,
-        keypair: Keypair,
-        relay: Option<Client>,
-    ) -> Result<Behaviour<KS>, OrbitBehaviourBuildError>
-    where
-        KSC: Default + RecordStoreConfig<KS>,
-        KS: for<'a> RecordStore<'a> + Send,
-    {
-        let peer_id = keypair.public().to_peer_id();
-        Ok(Behaviour {
-            identify: Identify::new(self.identify.to_config(keypair.public())),
-            ping: Ping::new(self.ping),
-            gossipsub: Gossipsub::new(
-                MessageAuthenticity::Signed(keypair),
-                GossipsubConfigBuilder::from(self.gossipsub)
-                    // always ensure validation
-                    .validation_mode(ValidationMode::Strict)
-                    .build()
-                    .map_err(OrbitBehaviourBuildError::Gossipsub)?,
-            )
-            .map_err(OrbitBehaviourBuildError::Gossipsub)?,
-            relay: relay.into(),
-            kademlia: Kademlia::with_config(
-                peer_id,
-                self.kademlia_store.init(peer_id),
-                self.kademlia,
-            ),
-            dcutr: Dcutr::new(),
-            autonat: AutoNat::new(peer_id, self.autonat),
-        })
-    }
-}
-
-#[derive(Error, Debug)]
-pub enum OrbitBehaviourBuildError {
-    #[error("{0}")]
-    Gossipsub(&'static str),
-}
-
-pub trait RecordStoreConfig<S>
-where
-    S: for<'a> RecordStore<'a>,
-{
-    fn init(self, id: PeerId) -> S;
-}
-
-impl RecordStoreConfig<MemoryStore> for MemoryStoreConfig {
-    fn init(self, id: PeerId) -> MemoryStore {
-        MemoryStore::with_config(id, self)
-    }
-}
diff --git a/src/p2p/behaviour/mod.rs b/src/p2p/behaviour/mod.rs
deleted file mode 100644
index e6ae179f..00000000
--- a/src/p2p/behaviour/mod.rs
+++ /dev/null
@@ -1,33 +0,0 @@
-use libp2p::{
-    autonat::Behaviour as AutoNat,
-    dcutr::behaviour::Behaviour as Dcutr,
-    gossipsub::Gossipsub,
-    identify::Behaviour as Identify,
-    kad::{
-        record::store::{MemoryStore, RecordStore},
-        Kademlia,
-    },
-    ping::Behaviour as Ping,
-    relay::v2::client::Client,
-    swarm::{behaviour::toggle::Toggle, Swarm},
-    NetworkBehaviour,
-};
-
-pub type OrbitSwarm<KS = MemoryStore> = Swarm<Behaviour<KS>>;
-mod builder;
-
-pub use builder::{BehaviourConfig, OrbitBehaviourBuildError};
-
-#[derive(NetworkBehaviour)]
-pub struct Behaviour<KS>
-where
-    KS: 'static + for<'a> RecordStore<'a> + Send,
-{
-    identify: Identify,
-    ping: Ping,
-    gossipsub: Gossipsub,
-    relay: Toggle<Client>,
-    kademlia: Kademlia<KS>,
-    dcutr: Dcutr,
-    autonat: AutoNat,
-}
diff --git a/src/p2p/mod.rs b/src/p2p/mod.rs
index 51258dee..fa723a21 100644
--- a/src/p2p/mod.rs
+++ b/src/p2p/mod.rs
@@ -1,5 +1,4 @@
 use libp2p::{identify::Config as OIdentifyConfig, identity::PublicKey};
-pub mod behaviour;
 pub mod relay;
 pub mod transport;
 
diff --git a/src/p2p/relay.rs b/src/p2p/relay.rs
index 995dcd0e..53e41340 100644
--- a/src/p2p/relay.rs
+++ b/src/p2p/relay.rs
@@ -181,101 +181,3 @@ mod builder {
         }
     }
 }
-
-#[cfg(test)]
-pub mod test {
-    // use super::*;
-    // use ipfs::{
-    //     p2p::transport::TransportBuilder, IpfsOptions, MultiaddrWithoutPeerId, Types,
-    //     UninitializedIpfs,
-    // };
-    // use libp2p::core::multiaddr::{multiaddr, Protocol};
-    // use std::{
-    //     convert::TryFrom,
-    //     sync::atomic::{AtomicU16, Ordering},
-    //     time::Duration,
-    // };
-
-    // static PORT: AtomicU16 = AtomicU16::new(10000);
-
-    // pub async fn test_relay() -> Result<RelayNode> {
-    //     RelayNode::new(
-    //         PORT.fetch_add(1, Ordering::SeqCst),
-    //         Keypair::generate_ed25519(),
-    //     )
-    //     .await
-    // }
-
-    // #[tokio::test(flavor = "multi_thread")]
-    // async fn relay() -> Result<()> {
-    //     let relay = test_relay().await?;
-
-    //     let dir = tempfile::TempDir::new("relay")?;
-    //     let alice_path = dir.path().join("alice");
-    //     std::fs::create_dir(&alice_path)?;
-    //     let bob_path = dir.path().join("bob");
-    //     std::fs::create_dir(&bob_path)?;
-
-    //     // Isn't actually in-memory, just provides useful defaults.
-    //     let mut alice_opts = IpfsOptions::inmemory_with_generated_keys();
-    //     alice_opts.ipfs_path = alice_path;
-    //     alice_opts.listening_addrs = vec![multiaddr!(P2pCircuit)];
-    //     let mut bob_opts = IpfsOptions::inmemory_with_generated_keys();
-    //     bob_opts.ipfs_path = bob_path;
-
-    //     let alice_peer_id = alice_opts.keypair.public().to_peer_id();
-    //     let bob_peer_id = bob_opts.keypair.public().to_peer_id();
-
-    //     let (alice_builder, alice_relay) = TransportBuilder::new(alice_opts.keypair.clone())?
-    //         .or(MemoryTransport::default())
-    //         .relay();
-    //     let alice_transport = alice_builder
-    //         .map_auth()
-    //         .map(crate::transport::auth_mapper([bob_peer_id.clone()]))
-    //         .build();
-    //     let (alice, task) =
-    //         UninitializedIpfs::<Types>::new(alice_opts, alice_transport, Some(alice_relay))
-    //             .start()
-    //             .await?;
-    //     tokio::spawn(task);
-
-    //     let (bob_builder, bob_relay) = TransportBuilder::new(bob_opts.keypair.clone())?
-    //         .or(MemoryTransport::default())
-    //         .relay();
-    //     let (bob, task) =
-    //         UninitializedIpfs::<Types>::new(bob_opts, bob_builder.build(), Some(bob_relay))
-    //             .start()
-    //             .await?;
-    //     tokio::spawn(task);
-
-    //     alice
-    //         .connect(MultiaddrWithoutPeerId::try_from(relay.internal())?.with(relay.id.clone()))
-    //         .await
-    //         .expect("alice failed to connect to relay");
-
-    //     bob.connect(
-    //         MultiaddrWithoutPeerId::try_from(
-    //             relay
-    //                 .external()
-    //                 .with(Protocol::P2p(relay.id.clone().into()))
-    //                 .with(Protocol::P2pCircuit),
-    //         )?
-    //         .with(alice_peer_id.clone()),
-    //     )
-    //     .await
-    //     .expect("bob failed to connect to alice");
-
-    //     tokio::time::sleep(Duration::from_millis(1000)).await;
-
-    //     let alice_peers = alice.peers().await?;
-    //     let bob_peers = bob.peers().await?;
-    //     assert!(alice_peers
-    //         .iter()
-    //         .any(|conn| conn.addr.peer_id == bob_peer_id));
-    //     assert!(bob_peers
-    //         .iter()
-    //         .any(|conn| conn.addr.peer_id == alice_peer_id));
-
-    //     Ok(())
-    // }
-}

From ecd60a207b74bd7729e953a9653ec870a13a772b Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Wed, 23 Nov 2022 13:41:51 +0100
Subject: [PATCH 72/73] remove janky builder derive workaround

---
 src/p2p/mod.rs       |  88 ++++++++++--------
 src/p2p/relay.rs     | 210 +++++++++++++++++++++----------------------
 src/p2p/transport.rs | 181 +++++++++++++++++++------------------
 3 files changed, 248 insertions(+), 231 deletions(-)

diff --git a/src/p2p/mod.rs b/src/p2p/mod.rs
index fa723a21..1b256444 100644
--- a/src/p2p/mod.rs
+++ b/src/p2p/mod.rs
@@ -1,55 +1,73 @@
+use core::time::Duration;
 use libp2p::{identify::Config as OIdentifyConfig, identity::PublicKey};
+
 pub mod relay;
 pub mod transport;
 
 pub const PROTOCOL_VERSION: &'static str = "kepler/0.1.0";
 
-pub use builder::IdentifyConfig;
+#[derive(Debug, Clone, Hash, PartialEq, Eq)]
+pub struct IdentifyConfig {
+    protocol_version: String,
+    initial_delay: Duration,
+    interval: Duration,
+    push_listen_addr_updates: bool,
+    cache_size: usize,
+}
 
-mod builder {
-    use core::time::Duration;
-    use derive_builder::Builder;
-    use libp2p::{identify::Config as OIdentifyConfig, identity::PublicKey};
-    #[derive(Builder)]
-    #[builder(build_fn(skip), setter(into), name = "IdentifyConfig", derive(Debug))]
-    pub struct IdentifyConfigDummy {
-        #[builder(field(type = "String"), default = "PROTOCOL_VERSION.into()")]
-        protocol_version: String,
-        #[builder(field(type = "Duration"), default = "Duration::from_millis(500)")]
-        initial_delay: Duration,
-        #[builder(field(type = "Duration"), default = "Duration::from_secs(300)")]
-        interval: Duration,
-        #[builder(field(type = "bool"), default = "false")]
-        push_listen_addr_updates: bool,
-        #[builder(field(type = "usize"), default = "0")]
-        cache_size: usize,
-    }
-    pub fn to_config(config: IdentifyConfig, key: PublicKey) -> OIdentifyConfig {
-        OIdentifyConfig::new(config.protocol_version, key)
-            .with_initial_delay(config.initial_delay)
-            .with_interval(config.interval)
-            .with_push_listen_addr_updates(config.push_listen_addr_updates)
-            .with_cache_size(config.cache_size)
-    }
-    pub fn convert(c: OIdentifyConfig) -> IdentifyConfig {
-        IdentifyConfig {
-            protocol_version: c.protocol_version,
-            initial_delay: c.initial_delay,
-            interval: c.interval,
-            push_listen_addr_updates: c.push_listen_addr_updates,
-            cache_size: c.cache_size,
+impl IdentifyConfig {
+    pub fn protocol_version(&mut self, i: impl Into<String>) -> &mut Self {
+        self.protocol_version = i.into();
+        self
+    }
+    pub fn initial_delay(&mut self, i: impl Into<Duration>) -> &mut Self {
+        self.initial_delay = i.into();
+        self
+    }
+    pub fn interval(&mut self, i: impl Into<Duration>) -> &mut Self {
+        self.interval = i.into();
+        self
+    }
+    pub fn push_listen_addr_updates(&mut self, i: impl Into<bool>) -> &mut Self {
+        self.push_listen_addr_updates = i.into();
+        self
+    }
+    pub fn cache_size(&mut self, i: impl Into<usize>) -> &mut Self {
+        self.cache_size = i.into();
+        self
+    }
+}
+
+impl Default for IdentifyConfig {
+    fn default() -> Self {
+        Self {
+            protocol_version: PROTOCOL_VERSION.into(),
+            initial_delay: Duration::from_millis(500),
+            interval: Duration::from_secs(300),
+            push_listen_addr_updates: false,
+            cache_size: 0,
         }
     }
 }
 
 impl IdentifyConfig {
     fn to_config(self, key: PublicKey) -> OIdentifyConfig {
-        builder::to_config(self, key)
+        OIdentifyConfig::new(self.protocol_version, key)
+            .with_initial_delay(self.initial_delay)
+            .with_interval(self.interval)
+            .with_push_listen_addr_updates(self.push_listen_addr_updates)
+            .with_cache_size(self.cache_size)
     }
 }
 
 impl From<OIdentifyConfig> for IdentifyConfig {
     fn from(c: OIdentifyConfig) -> Self {
-        builder::convert(c)
+        IdentifyConfig {
+            protocol_version: c.protocol_version,
+            initial_delay: c.initial_delay,
+            interval: c.interval,
+            push_listen_addr_updates: c.push_listen_addr_updates,
+            cache_size: c.cache_size,
+        }
     }
 }
diff --git a/src/p2p/relay.rs b/src/p2p/relay.rs
index 53e41340..ae8aefc7 100644
--- a/src/p2p/relay.rs
+++ b/src/p2p/relay.rs
@@ -11,18 +11,17 @@ use futures::{
     stream::{iter, StreamExt},
 };
 use libp2p::{
-    autonat::Behaviour as AutoNat,
-    core::{identity::Keypair, upgrade, Multiaddr, PeerId},
+    autonat::{Behaviour as AutoNat, Config as AutoNatConfig},
+    core::{identity::Keypair, upgrade, Multiaddr, PeerId, Transport},
     identify::Behaviour as Identify,
+    identity::PublicKey,
     mplex, noise,
-    ping::Behaviour as Ping,
-    relay::v2::relay::Relay,
+    ping::{Behaviour as Ping, Config as PingConfig},
+    relay::v2::relay::{Config as RelayConfig, Relay},
     swarm::{Swarm, SwarmBuilder},
     yamux, NetworkBehaviour,
 };
 
-pub use builder::Config;
-
 pub type RelaySwarm = Swarm<Behaviour>;
 
 #[derive(Clone, Debug)]
@@ -64,120 +63,117 @@ impl RelayNode {
     }
 }
 
-mod builder {
-    use super::*;
-    use derive_builder::Builder;
-    use libp2p::{
-        autonat::Config as AutoNatConfig, core::Transport, identity::PublicKey,
-        ping::Config as PingConfig, relay::v2::relay::Config as RelayConfig,
-    };
+#[derive(Debug, Default)]
+pub struct Config {
+    identify: IdentifyConfig,
+    ping: PingConfig,
+    relay: RelayConfig,
+    autonat: AutoNatConfig,
+}
 
-    #[derive(Builder)]
-    #[builder(
-        build_fn(skip),
-        setter(into),
-        name = "Config",
-        derive(Debug),
-        pattern = "owned"
-    )]
-    pub struct BehaviourConfigDummy {
-        #[builder(field(type = "IdentifyConfig"))]
-        identify: IdentifyConfig,
-        #[builder(field(type = "PingConfig"))]
-        ping: PingConfig,
-        #[builder(field(type = "RelayConfig"))]
-        relay: RelayConfig,
-        #[builder(field(type = "AutoNatConfig"))]
-        autonat: AutoNatConfig,
+impl Config {
+    pub fn identify(&mut self, i: impl Into<IdentifyConfig>) -> &mut Self {
+        self.identify = i.into();
+        self
+    }
+    pub fn ping(&mut self, i: impl Into<PingConfig>) -> &mut Self {
+        self.ping = i.into();
+        self
+    }
+    pub fn relay(&mut self, i: impl Into<RelayConfig>) -> &mut Self {
+        self.relay = i.into();
+        self
+    }
+    pub fn autonat(&mut self, i: impl Into<AutoNatConfig>) -> &mut Self {
+        self.autonat = i.into();
+        self
     }
 
-    impl Config {
-        fn build(self, pubkey: PublicKey) -> Behaviour {
-            let peer_id = pubkey.to_peer_id();
-            Behaviour {
-                identify: Identify::new(self.identify.to_config(pubkey)),
-                ping: Ping::new(self.ping),
-                relay: Relay::new(peer_id, self.relay),
-                autonat: AutoNat::new(peer_id, self.autonat),
-            }
+    fn build(self, pubkey: PublicKey) -> Behaviour {
+        let peer_id = pubkey.to_peer_id();
+        Behaviour {
+            identify: Identify::new(self.identify.to_config(pubkey)),
+            ping: Ping::new(self.ping),
+            relay: Relay::new(peer_id, self.relay),
+            autonat: AutoNat::new(peer_id, self.autonat),
         }
+    }
 
-        pub fn launch<T>(self, keypair: Keypair, transport: T) -> Result<RelayNode, T::Error>
-        where
-            T: IntoTransport,
-            T::T: 'static + Send + Unpin,
-            T::Error: 'static + std::error::Error + Send + Sync,
-            <T::T as Transport>::Output: 'static + AsyncRead + AsyncWrite + Unpin + Send,
-            <T::T as Transport>::Error: 'static + Send + Sync,
-            <T::T as Transport>::Dial: Send,
-            <T::T as Transport>::ListenerUpgrade: Send,
-        {
-            let local_public_key = keypair.public();
-            let id = local_public_key.to_peer_id();
-            let b = self.build(local_public_key);
-            let (sender, mut reciever) = mpsc::channel(100);
-            let r = RelayNode { id, sender };
+    pub fn launch<T>(self, keypair: Keypair, transport: T) -> Result<RelayNode, T::Error>
+    where
+        T: IntoTransport,
+        T::T: 'static + Send + Unpin,
+        T::Error: 'static + std::error::Error + Send + Sync,
+        <T::T as Transport>::Output: 'static + AsyncRead + AsyncWrite + Unpin + Send,
+        <T::T as Transport>::Error: 'static + Send + Sync,
+        <T::T as Transport>::Dial: Send,
+        <T::T as Transport>::ListenerUpgrade: Send,
+    {
+        let local_public_key = keypair.public();
+        let id = local_public_key.to_peer_id();
+        let b = self.build(local_public_key);
+        let (sender, mut reciever) = mpsc::channel(100);
+        let r = RelayNode { id, sender };
 
-            let mut swarm = SwarmBuilder::new(
-                transport
-                    .into_transport()?
-                    .upgrade(upgrade::Version::V1)
-                    .authenticate(noise::NoiseAuthenticated::xx(&keypair).unwrap())
-                    .multiplex(upgrade::SelectUpgrade::new(
-                        yamux::YamuxConfig::default(),
-                        mplex::MplexConfig::default(),
-                    ))
-                    .timeout(std::time::Duration::from_secs(20))
-                    .boxed(),
-                b,
-                id,
-            )
-            .build();
+        let mut swarm = SwarmBuilder::new(
+            transport
+                .into_transport()?
+                .upgrade(upgrade::Version::V1)
+                .authenticate(noise::NoiseAuthenticated::xx(&keypair).unwrap())
+                .multiplex(upgrade::SelectUpgrade::new(
+                    yamux::YamuxConfig::default(),
+                    mplex::MplexConfig::default(),
+                ))
+                .timeout(std::time::Duration::from_secs(20))
+                .boxed(),
+            b,
+            id,
+        )
+        .build();
 
-            tokio::spawn(async move {
-                loop {
-                    match select(reciever.next(), swarm.next()).await {
-                        // if the swarm or the channel are closed, close the relay
-                        Either::Right((None, _)) | Either::Left((None, _)) => {
-                            break;
-                        }
-                        // process command
-                        Either::Left((Some(e), _)) => match e {
-                            Message::ListenOn(a, s) => {
-                                // try listen on each given address
-                                match a.into_iter().try_fold(Vec::new(), |mut listeners, addr| {
-                                    match swarm.listen_on(addr) {
-                                        Ok(l) => {
-                                            listeners.push(l);
-                                            Ok(listeners)
-                                        }
-                                        Err(e) => Err((e, listeners)),
+        tokio::spawn(async move {
+            loop {
+                match select(reciever.next(), swarm.next()).await {
+                    // if the swarm or the channel are closed, close the relay
+                    Either::Right((None, _)) | Either::Left((None, _)) => {
+                        break;
+                    }
+                    // process command
+                    Either::Left((Some(e), _)) => match e {
+                        Message::ListenOn(a, s) => {
+                            // try listen on each given address
+                            match a.into_iter().try_fold(Vec::new(), |mut listeners, addr| {
+                                match swarm.listen_on(addr) {
+                                    Ok(l) => {
+                                        listeners.push(l);
+                                        Ok(listeners)
                                     }
-                                }) {
-                                    Ok(_) => s.send(Ok(())),
-                                    // if one fails, roll back all of them
-                                    Err((e, listeners)) => {
-                                        for l in listeners {
-                                            swarm.remove_listener(l);
-                                        }
-                                        s.send(Err(e.into()))
+                                    Err(e) => Err((e, listeners)),
+                                }
+                            }) {
+                                Ok(_) => s.send(Ok(())),
+                                // if one fails, roll back all of them
+                                Err((e, listeners)) => {
+                                    for l in listeners {
+                                        swarm.remove_listener(l);
                                     }
+                                    s.send(Err(e.into()))
                                 }
-                                .map_err(|_| anyhow!("failed to return listening result"))?;
-                            }
-                            Message::GetAddresses(s) => {
-                                s.send(swarm.listeners().map(|a| a.clone()).collect())
-                                    .map_err(|_| anyhow!("failed to return listeners"))?;
                             }
-                        },
-                        Either::Right((Some(e), _)) => {
-                            // process swarm event
+                            .map_err(|_| anyhow!("failed to return listening result"))?;
                         }
+                        Message::GetAddresses(s) => {
+                            s.send(swarm.listeners().map(|a| a.clone()).collect())
+                                .map_err(|_| anyhow!("failed to return listeners"))?;
+                        }
+                    },
+                    Either::Right((Some(e), _)) => {
+                        // process swarm event
                     }
                 }
-                Result::<(), anyhow::Error>::Ok(())
-            });
-            Ok(r)
-        }
+            }
+            Result::<(), anyhow::Error>::Ok(())
+        });
+        Ok(r)
     }
 }
diff --git a/src/p2p/transport.rs b/src/p2p/transport.rs
index a046ef98..c4b27aa9 100644
--- a/src/p2p/transport.rs
+++ b/src/p2p/transport.rs
@@ -1,5 +1,4 @@
 use crate::storage::either::EitherError;
-use derive_builder::Builder;
 use futures::io::{AsyncRead, AsyncWrite};
 use libp2p::{
     core::transport::{dummy::DummyTransport, MemoryTransport, OrTransport, Transport},
@@ -22,10 +21,8 @@ pub trait IntoTransport {
     }
 }
 
-pub use dns::{CustomDnsResolver, DnsConfig};
 pub use libp2p::tcp::GenTcpConfig as TcpConfig;
 pub use libp2p::wasm_ext::ffi::Transport as ExtConfig;
-pub use ws::{WsConfig, WS_MAX_DATA_SIZE};
 
 #[derive(Default, Debug, Clone, Hash, PartialEq, Eq)]
 pub struct MemoryConfig;
@@ -63,54 +60,37 @@ impl Default for DnsResolver {
     }
 }
 
-mod dns {
-    use super::*;
-    #[derive(Builder)]
-    #[builder(
-        build_fn(skip),
-        setter(into),
-        derive(Debug),
-        name = "CustomDnsResolver"
-    )]
-    pub struct CustomDnsResolverDummy {
-        #[builder(field(type = "ResolverConfig"))]
-        conf: ResolverConfig,
-        #[builder(field(type = "ResolverOpts"))]
-        opts: ResolverOpts,
-    }
+#[derive(Debug, Clone, PartialEq, Eq, Default)]
+pub struct CustomDnsResolver {
+    conf: ResolverConfig,
+    opts: ResolverOpts,
+}
 
-    #[derive(Builder)]
-    #[builder(build_fn(skip), setter(into), derive(Debug), name = "DnsConfig")]
-    pub struct DnsConfigDummy<B>
-    where
-        B: Default,
-    {
-        #[builder(field(type = "DnsResolver"))]
-        resolver: DnsResolver,
-        #[builder(field(type = "B"))]
-        base: B,
+impl CustomDnsResolver {
+    pub fn config(&mut self, i: impl Into<ResolverConfig>) -> &mut Self {
+        self.conf = i.into();
+        self
+    }
+    pub fn options(&mut self, i: impl Into<ResolverOpts>) -> &mut Self {
+        self.opts = i.into();
+        self
     }
+}
 
-    pub fn convert<B>(c: DnsConfig<B>) -> Result<DnsTransport<B::T>, EitherError<IoError, B::Error>>
-    where
-        B: Default + IntoTransport,
-        B::T: 'static + Send + Unpin,
-        <B::T as Transport>::Output: 'static + AsyncRead + AsyncWrite + Send + Unpin,
-        <B::T as Transport>::Dial: Send,
-        <B::T as Transport>::Error: Send,
-    {
-        match c.resolver {
-            DnsResolver::System => {
-                DnsTransport::system(c.base.into_transport().map_err(EitherError::B)?)
-                    .map_err(EitherError::A)
-            }
-            DnsResolver::Custom(custom) => DnsTransport::custom(
-                c.base.into_transport().map_err(EitherError::B)?,
-                custom.conf,
-                custom.opts,
-            )
-            .map_err(EitherError::A),
-        }
+#[derive(Debug, Clone, Default)]
+pub struct DnsConfig<B> {
+    resolver: DnsResolver,
+    base: B,
+}
+
+impl<B> DnsConfig<B> {
+    pub fn resolver(&mut self, i: impl Into<DnsResolver>) -> &mut Self {
+        self.resolver = i.into();
+        self
+    }
+    pub fn base(&mut self, i: impl Into<B>) -> &mut Self {
+        self.base = i.into();
+        self
     }
 }
 
@@ -125,52 +105,70 @@ where
     type T = DnsTransport<B::T>;
     type Error = EitherError<IoError, B::Error>;
     fn into_transport(self) -> Result<Self::T, Self::Error> {
-        dns::convert(self)
+        match self.resolver {
+            DnsResolver::System => {
+                DnsTransport::system(self.base.into_transport().map_err(EitherError::B)?)
+                    .map_err(EitherError::A)
+            }
+            DnsResolver::Custom(custom) => DnsTransport::custom(
+                self.base.into_transport().map_err(EitherError::B)?,
+                custom.conf,
+                custom.opts,
+            )
+            .map_err(EitherError::A),
+        }
     }
 }
 
-mod ws {
-    use super::*;
-    pub const WS_MAX_DATA_SIZE: usize = 256 * 1024 * 1024;
+pub const WS_MAX_DATA_SIZE: usize = 256 * 1024 * 1024;
+
+#[derive(Debug, Clone)]
+pub struct WsConfig<T> {
+    base: T,
+    max_redirects: u8,
+    max_data_size: usize,
+    deflate: bool,
+    tls: WsTlsConfig,
+}
 
-    fn client() -> WsTlsConfig {
-        WsTlsConfig::client()
+impl<T> WsConfig<T> {
+    pub fn from_base(b: impl Into<T>) -> Self {
+        Self {
+            base: b.into(),
+            max_redirects: 0,
+            max_data_size: WS_MAX_DATA_SIZE,
+            deflate: false,
+            tls: WsTlsConfig::client(),
+        }
+    }
+    pub fn base(&mut self, i: impl Into<T>) -> &mut Self {
+        self.base = i.into();
+        self
+    }
+    pub fn max_redirects(&mut self, i: impl Into<u8>) -> &mut Self {
+        self.max_redirects = i.into();
+        self
+    }
+    pub fn max_data_size(&mut self, i: impl Into<usize>) -> &mut Self {
+        self.max_data_size = i.into();
+        self
+    }
+    pub fn deflate(&mut self, i: impl Into<bool>) -> &mut Self {
+        self.deflate = i.into();
+        self
     }
+    pub fn tls(&mut self, i: impl Into<WsTlsConfig>) -> &mut Self {
+        self.tls = i.into();
+        self
+    }
+}
 
-    #[derive(Builder)]
-    #[builder(build_fn(skip), setter(into), derive(Debug), name = "WsConfig")]
-    pub struct WsConfigDummy<T>
-    where
-        T: Default,
-    {
-        #[builder(field(type = "T"))]
-        base: T,
-        #[builder(field(type = "u8"), default = "0")]
-        max_redirects: u8,
-        #[builder(field(type = "usize"), default = "WS_MAX_DATA_SIZE")]
-        max_data_size: usize,
-        #[builder(field(type = "bool"), default = "false")]
-        deflate: bool,
-        // TODO this is cause some kind of error cos it has no Default
-        // #[builder(field(type = "WsTlsConfig"), default = "client()")]
-        // tls: WsTlsConfig,
-    }
-
-    pub fn convert<B>(c: WsConfig<B>) -> Result<WsTransport<B::T>, B::Error>
-    where
-        B: Default + IntoTransport,
-        B::T: 'static + Send + Unpin,
-        <B::T as Transport>::Output: 'static + AsyncRead + AsyncWrite + Send + Unpin,
-        <B::T as Transport>::Dial: Send,
-        <B::T as Transport>::Error: Send,
-        <B::T as Transport>::ListenerUpgrade: Send,
-    {
-        let mut ws = WsTransport::new(c.base.into_transport()?);
-        ws.set_max_redirects(c.max_redirects)
-            .set_max_data_size(c.max_data_size)
-            // .set_tls_config(c.tls)
-            .use_deflate(c.deflate);
-        Ok(ws)
+impl<T> Default for WsConfig<T>
+where
+    T: Default,
+{
+    fn default() -> Self {
+        Self::from_base(T::default())
     }
 }
 
@@ -186,7 +184,12 @@ where
     type T = WsTransport<B::T>;
     type Error = B::Error;
     fn into_transport(self) -> Result<Self::T, Self::Error> {
-        ws::convert(self)
+        let mut ws = WsTransport::new(self.base.into_transport()?);
+        ws.set_max_redirects(self.max_redirects)
+            .set_max_data_size(self.max_data_size)
+            .set_tls_config(self.tls)
+            .use_deflate(self.deflate);
+        Ok(ws)
     }
 }
 

From ea5d08fb1cf7fb330668558989a32be0ff026c90 Mon Sep 17 00:00:00 2001
From: chunningham <c.a.cunningham6@gmail.com>
Date: Wed, 23 Nov 2022 13:43:15 +0100
Subject: [PATCH 73/73] warnings

---
 src/authorization.rs | 6 +++---
 src/orbit.rs         | 2 +-
 src/p2p/relay.rs     | 9 +++------
 3 files changed, 7 insertions(+), 10 deletions(-)

diff --git a/src/authorization.rs b/src/authorization.rs
index c26042b2..fa4dba02 100644
--- a/src/authorization.rs
+++ b/src/authorization.rs
@@ -515,7 +515,7 @@ mod test {
         },
     };
 
-    async fn gen(
+    async fn _gen(
         iss: &JWK,
         aud: String,
         caps: Vec<Capability>,
@@ -531,10 +531,10 @@ mod test {
                 .await
                 .1
                 .unwrap(),
-            gen_ucan((iss, did), aud, caps, exp, prf).await,
+            _gen_ucan((iss, did), aud, caps, exp, prf).await,
         )
     }
-    async fn gen_ucan(
+    async fn _gen_ucan(
         iss: (&JWK, String),
         audience: String,
         attenuation: Vec<Capability>,
diff --git a/src/orbit.rs b/src/orbit.rs
index ece9934d..305c23ef 100644
--- a/src/orbit.rs
+++ b/src/orbit.rs
@@ -284,6 +284,6 @@ mod tests {
 
         let md = Manifest::resolve_dyn(&oid, None).await.unwrap().unwrap();
 
-        let (orbit, dir) = op(md).await.unwrap();
+        let (_, _) = op(md).await.unwrap();
     }
 }
diff --git a/src/p2p/relay.rs b/src/p2p/relay.rs
index ae8aefc7..5c0ae69a 100644
--- a/src/p2p/relay.rs
+++ b/src/p2p/relay.rs
@@ -1,14 +1,11 @@
-use crate::{
-    orbit::AbortOnDrop,
-    p2p::{transport::IntoTransport, IdentifyConfig},
-};
+use crate::p2p::{transport::IntoTransport, IdentifyConfig};
 use anyhow::Result;
 use futures::{
     channel::{mpsc, oneshot},
     future::{select, Either},
     io::{AsyncRead, AsyncWrite},
     sink::SinkExt,
-    stream::{iter, StreamExt},
+    stream::StreamExt,
 };
 use libp2p::{
     autonat::{Behaviour as AutoNat, Config as AutoNatConfig},
@@ -167,7 +164,7 @@ impl Config {
                                 .map_err(|_| anyhow!("failed to return listeners"))?;
                         }
                     },
-                    Either::Right((Some(e), _)) => {
+                    Either::Right((Some(_), _)) => {
                         // process swarm event
                     }
                 }