Track chain state in registry (fixes #60)

This commit moves the last sign state tracking for chains into the
global chain registry.

It also allows configurable locations for the chain state tracking
files, which should probably make it easier to ensure they don't clobber
each other in tests.

This doesn't fully implement the double signing plan in #60 but at this
point I think we're close enough. The remaining item is (optionally)
running a user-specified command on startup to query the current block
height.

.gitignore

@@ -1,6 +1,7 @@
 /target
-**/*.rs.bk
 tmkms.toml
+**/*.rs.bk
+**/*priv_validator_state.json
 
 # Ignore VIM swap files
 *.swp

src/chain/mod.rs

@@ -3,9 +3,11 @@
 mod guard;
 pub mod key;
 mod registry;
+pub mod state;
 
-pub use self::{guard::Guard, registry::REGISTRY};
-use crate::config::chain::ChainConfig;
+pub use self::{guard::Guard, registry::REGISTRY, state::LastSignState};
+use crate::{config::chain::ChainConfig, error::KmsError};
+use std::{path::PathBuf, sync::Mutex};
 pub use tendermint::chain::Id;
 
 /// Information about a particular Tendermint blockchain network
@@ -15,13 +17,25 @@ pub struct Chain {
 
     /// Key format configuration
     pub key_format: key::Format,
+
+    /// State from the last block signed for this chain
+    pub state: Mutex<LastSignState>,
 }
 
-impl<'a> From<&ChainConfig> for Chain {
-    fn from(config: &ChainConfig) -> Chain {
-        Self {
+impl Chain {
+    /// Attempt to create a `Chain` state from the given configuration
+    pub fn from_config(config: &ChainConfig) -> Result<Chain, KmsError> {
+        let state_file = match config.state_file {
+            Some(ref path) => path.to_owned(),
+            None => PathBuf::from(&format!("{}_priv_validator_state.json", config.id)),
+        };
+
+        let last_sign_state = LastSignState::load_state(state_file)?;
+
+        Ok(Self {
             id: config.id,
             key_format: config.key_format.clone(),
-        }
+            state: Mutex::new(last_sign_state),
+        })
     }
 }

src/last_sign_state.rs → src/chain/state.rs

@@ -1,3 +1,4 @@
+use crate::{chain, error::KmsError};
 use abscissa::Error;
 use atomicwrites::{AtomicFile, OverwriteBehavior};
 use serde_json;
@@ -7,8 +8,28 @@ use std::{
     io::{self, prelude::*},
     path::{Path, PathBuf},
 };
-use tendermint::{block, chain};
+use tendermint::block;
+
+/// Check and update the chain position for the given `chain::Id`
+pub fn check_and_update_hrs(
+    chain_id: chain::Id,
+    height: i64,
+    round: i64,
+    step: i8,
+    block_id: Option<block::Id>,
+) -> Result<(), KmsError> {
+    let registry = chain::REGISTRY.get();
+    let chain = registry
+        .chain(chain_id)
+        .unwrap_or_else(|| panic!("can't update state for unregistered chain: {}", chain_id));
+
+    // TODO(tarcieri): better handle `PoisonErrore`?
+    let mut last_sign_state = chain.state.lock().unwrap();
+    last_sign_state.check_and_update_hrs(height, round, step, block_id)?;
+    Ok(())
+}
 
+/// Position of the chain the last time we attempted to sign
 #[derive(Clone, Debug, Default, Serialize, Deserialize)]
 struct LastSignData {
     pub height: i64,
@@ -17,26 +38,38 @@ struct LastSignData {
     pub block_id: Option<block::Id>,
 }
 
+/// State tracking for double signing prevention
 pub struct LastSignState {
     data: LastSignData,
     path: PathBuf,
-    _chain_id: chain::Id,
 }
 
 /// Error type
 #[derive(Debug)]
 pub struct LastSignError(Error<LastSignErrorKind>);
 
+/// Kinds of errors
 #[derive(Copy, Clone, Eq, PartialEq, Debug, Fail)]
 pub enum LastSignErrorKind {
+    /// Height regressed
     #[fail(display = "height regression")]
     HeightRegression,
+
+    /// Step regressed
     #[fail(display = "step regression")]
     StepRegression,
+
+    /// Round regressed
     #[fail(display = "round regression")]
     RoundRegression,
-    #[fail(display = "invalid block id")]
+
+    /// Double sign detected
+    #[fail(display = "double sign detected")]
     DoubleSign,
+
+    /// Error syncing state to disk
+    #[fail(display = "error syncing state to disk")]
+    SyncError,
 }
 
 impl From<Error<LastSignErrorKind>> for LastSignError {
@@ -52,11 +85,14 @@ impl Display for LastSignError {
 }
 
 impl LastSignState {
-    pub fn load_state(path: &Path, chain_id: chain::Id) -> std::io::Result<LastSignState> {
+    /// Load the state from the given path
+    pub fn load_state<P>(path: P) -> std::io::Result<LastSignState>
+    where
+        P: AsRef<Path>,
+    {
         let mut lst = LastSignState {
             data: LastSignData::default(),
-            path: path.to_owned(),
-            _chain_id: chain_id,
+            path: path.as_ref().to_owned(),
         };
 
         match fs::read_to_string(path) {
@@ -75,15 +111,7 @@ impl LastSignState {
         }
     }
 
-    pub fn sync_to_disk(&mut self) -> std::io::Result<()> {
-        let json = serde_json::to_string(&self.data)?;
-
-        AtomicFile::new(&self.path, OverwriteBehavior::AllowOverwrite)
-            .write(|f| f.write_all(json.as_bytes()))?;
-
-        Ok(())
-    }
-
+    /// Check and update the chain's height, round, and step
     pub fn check_and_update_hrs(
         &mut self,
         height: i64,
@@ -98,8 +126,7 @@ impl LastSignState {
                 self.data.height,
                 height
             );
-        }
-        if height == self.data.height {
+        } else if height == self.data.height {
             if round < self.data.round {
                 fail!(
                     LastSignErrorKind::RoundRegression,
@@ -108,8 +135,7 @@ impl LastSignState {
                     self.data.round,
                     round
                 )
-            }
-            if round == self.data.round {
+            } else if round == self.data.round {
                 if step < self.data.step {
                     fail!(
                         LastSignErrorKind::StepRegression,
@@ -121,24 +147,46 @@ impl LastSignState {
                     )
                 }
 
-                if block_id != None && self.data.block_id != None && self.data.block_id != block_id
+                if block_id.is_some()
+                    && self.data.block_id.is_some()
+                    && self.data.block_id != block_id
                 {
                     fail!(
-                    LastSignErrorKind::DoubleSign,
-                    "Attempting to sign a second proposal at height:{} round:{} step:{} old block id:{} new block {}",
-                    height,
-                    round,
-                    step,
-                    self.data.block_id.clone().unwrap(),
-                    block_id.unwrap()
+                        LastSignErrorKind::DoubleSign,
+                        "Attempting to sign a second proposal at height:{} round:{} step:{} old block id:{} new block {}",
+                        height,
+                        round,
+                        step,
+                        self.data.block_id.clone().unwrap(),
+                        block_id.unwrap()
                     )
                 }
             }
         }
+
         self.data.height = height;
         self.data.round = round;
         self.data.step = step;
         self.data.block_id = block_id;
+
+        self.sync_to_disk().map_err(|e| {
+            err!(
+                LastSignErrorKind::SyncError,
+                "error writing state to {}: {}",
+                self.path.display(),
+                e
+            )
+        })?;
+        Ok(())
+    }
+
+    /// Sync the current state to disk
+    fn sync_to_disk(&mut self) -> std::io::Result<()> {
+        let json = serde_json::to_string(&self.data)?;
+
+        AtomicFile::new(&self.path, OverwriteBehavior::AllowOverwrite)
+            .write(|f| f.write_all(json.as_bytes()))?;
+
         Ok(())
     }
 }
@@ -147,7 +195,7 @@ impl LastSignState {
 mod tests {
     use super::*;
     use std::str::FromStr;
-    use tendermint::{block, chain};
+    use tendermint::block;
 
     const EXAMPLE_BLOCK_ID: &str =
         "26C0A41F3243C6BCD7AD2DFF8A8D83A71D29D307B5326C227F734A1A512FE47D";
@@ -167,7 +215,6 @@ mod tests {
                 block_id: None,
             },
             path: EXAMPLE_PATH.into(),
-            _chain_id: "example-chain".parse::<chain::Id>().unwrap(),
         };
         assert_eq!(
             last_sign_state.check_and_update_hrs(2, 0, 0, None).unwrap(),
@@ -185,7 +232,6 @@ mod tests {
                 block_id: Some(block::Id::from_str(EXAMPLE_BLOCK_ID).unwrap()),
             },
             path: EXAMPLE_PATH.into(),
-            _chain_id: "example-chain".parse::<chain::Id>().unwrap(),
         };
         let double_sign_block = block::Id::from_str(EXAMPLE_DOUBLE_SIGN_BLOCK_ID).unwrap();
         let err = last_sign_state.check_and_update_hrs(1, 1, 1, Some(double_sign_block));

src/commands/start.rs

@@ -1,5 +1,5 @@
 use crate::{
-    chain,
+    chain::{self, Chain},
     client::Client,
     config::{KmsConfig, ValidatorConfig},
     keyring::KeyRing,
@@ -47,7 +47,12 @@ impl Callable for StartCommand {
         let config = KmsConfig::get_global();
 
         for chain_config in &config.chain {
-            chain::REGISTRY.register(chain_config.into()).unwrap();
+            chain::REGISTRY
+                .register(Chain::from_config(&chain_config).unwrap_or_else(|e| {
+                    status_err!("error initializing chain '{}': {}", chain_config.id, e);
+                    process::exit(1);
+                }))
+                .unwrap();
         }
 
         KeyRing::load_from_config(&config.providers).unwrap_or_else(|e| {

src/config/chain.rs

@@ -1,6 +1,7 @@
 //! Chain configuration
 
 use crate::chain;
+use std::path::PathBuf;
 
 /// Chain configuration
 #[derive(Clone, Deserialize, Debug)]
@@ -10,4 +11,7 @@ pub struct ChainConfig {
 
     /// Key format configuration
     pub key_format: chain::key::Format,
+
+    /// Path to chain-specific `priv_validator_state.json` file
+    pub state_file: Option<PathBuf>,
 }

src/error.rs

@@ -1,7 +1,6 @@
 // Error types
 
-use crate::last_sign_state;
-use crate::prost;
+use crate::{chain, prost};
 use abscissa::Error;
 use signatory;
 use std::{
@@ -151,8 +150,8 @@ impl From<TmValidationError> for KmsError {
     }
 }
 
-impl From<last_sign_state::LastSignError> for KmsError {
-    fn from(other: last_sign_state::LastSignError) -> Self {
+impl From<chain::state::LastSignError> for KmsError {
+    fn from(other: chain::state::LastSignError) -> Self {
         err!(KmsErrorKind::DoubleSign, other).into()
     }
 }

src/lib.rs

@@ -35,7 +35,6 @@ mod client;
 mod commands;
 mod config;
 mod keyring;
-mod last_sign_state;
 mod rpc;
 mod session;
 mod unix_connection;