Validator key hot swap (#11536)

Issue: #11264

This PR build on:
* #11372
* #11400

and contains actual changes and test for validator key hot swap.

### Summary
- Extend `UpdateableConfig` with validator key.
- Update client's mutable validator key when we detect it changed in the
updateable config.
- Advertise our new validator key through `advertise_tier1_proxies()`.
- Add integration test for the new behaviour:
- We start with 2 validating nodes (`node0`, `node1`) and 1
non-validating node (`node2`). It is important that the non-validating
node tracks all shards, because we do not know which shard it will track
when we switch validator keys.
  - We copy validator key from `node0` to `node2`.
  - We stop `node0`, then we trigger validator key reload for `node2`.
- Now `node2` is a validator, but it figures out as `node0` because it
copied validator key from `node0`.
- We wait for a couple of epochs and we require that both remaining
nodes progress the chain. Both nodes should be synchronised after a few
epochs.


Test with:
```
cargo build -pneard --features test_features,rosetta_rpc &&
cargo build -pgenesis-populate -prestaked -pnear-test-contracts &&
python3 pytest/tests/sanity/validator_switch_key_quick.py
```

#### Extra changes:
- Use `MutableValidatorSigner` alias instead of
`MutableConfigValue<Option<Arc<ValidatorSigner>>>`
- Return `ConfigUpdaterResult` from config updater.
- Remove (de)serialization derives for `UpdateableConfigs`.
-

---------

Co-authored-by: Your Name <[email protected]>

chain/chain/src/chain.rs

@@ -39,7 +39,9 @@ use itertools::Itertools;
 use lru::LruCache;
 use near_async::futures::{AsyncComputationSpawner, AsyncComputationSpawnerExt};
 use near_async::time::{Clock, Duration, Instant};
-use near_chain_configs::{MutableConfigValue, ReshardingConfig, ReshardingHandle};
+use near_chain_configs::{
+    MutableConfigValue, MutableValidatorSigner, ReshardingConfig, ReshardingHandle,
+};
 #[cfg(feature = "new_epoch_sync")]
 use near_chain_primitives::error::epoch_sync::EpochSyncInfoError;
 use near_chain_primitives::error::{BlockKnownError, Error, LogTransientStorageError};
@@ -87,7 +89,6 @@ use near_primitives::unwrap_or_return;
 #[cfg(feature = "new_epoch_sync")]
 use near_primitives::utils::index_to_bytes;
 use near_primitives::utils::MaybeValidated;
-use near_primitives::validator_signer::ValidatorSigner;
 use near_primitives::version::{ProtocolFeature, ProtocolVersion, PROTOCOL_VERSION};
 use near_primitives::views::{
     BlockStatusView, DroppedReason, ExecutionOutcomeWithIdView, ExecutionStatusView,
@@ -404,7 +405,7 @@ impl Chain {
         chain_config: ChainConfig,
         snapshot_callbacks: Option<SnapshotCallbacks>,
         apply_chunks_spawner: Arc<dyn AsyncComputationSpawner>,
-        validator: MutableConfigValue<Option<Arc<ValidatorSigner>>>,
+        validator: MutableValidatorSigner,
     ) -> Result<Chain, Error> {
         // Get runtime initial state and create genesis block out of it.
         let state_roots = get_genesis_state_roots(runtime_adapter.store())?

chain/chunks/src/shards_manager_actor.rs

@@ -98,7 +98,7 @@ use near_chain::byzantine_assert;
 use near_chain::chunks_store::ReadOnlyChunksStore;
 use near_chain::near_chain_primitives::error::Error::DBNotFoundErr;
 use near_chain::types::EpochManagerAdapter;
-use near_chain_configs::MutableConfigValue;
+use near_chain_configs::MutableValidatorSigner;
 pub use near_chunks_primitives::Error;
 use near_epoch_manager::shard_tracker::ShardTracker;
 use near_network::shards_manager::ShardsManagerRequestFromNetwork;
@@ -246,7 +246,7 @@ pub struct ShardsManagerActor {
     /// Contains validator info about this node. This field is mutable and optional. Use with caution!
     /// Lock the value of mutable validator signer for the duration of a request to ensure consistency.
     /// Please note that the locked value should not be stored anywhere or passed through the thread boundary.
-    validator_signer: MutableConfigValue<Option<Arc<ValidatorSigner>>>,
+    validator_signer: MutableValidatorSigner,
     store: ReadOnlyChunksStore,
 
     epoch_manager: Arc<dyn EpochManagerAdapter>,
@@ -296,7 +296,7 @@ pub fn start_shards_manager(
     shard_tracker: ShardTracker,
     network_adapter: Sender<PeerManagerMessageRequest>,
     client_adapter_for_shards_manager: Sender<ShardsManagerResponse>,
-    validator_signer: MutableConfigValue<Option<Arc<ValidatorSigner>>>,
+    validator_signer: MutableValidatorSigner,
     store: Store,
     chunk_request_retry_period: Duration,
 ) -> (actix::Addr<ActixWrapper<ShardsManagerActor>>, actix::ArbiterHandle) {
@@ -334,7 +334,7 @@ pub fn start_shards_manager(
 impl ShardsManagerActor {
     pub fn new(
         clock: time::Clock,
-        validator_signer: MutableConfigValue<Option<Arc<ValidatorSigner>>>,
+        validator_signer: MutableValidatorSigner,
         epoch_manager: Arc<dyn EpochManagerAdapter>,
         shard_tracker: ShardTracker,
         network_adapter: Sender<PeerManagerMessageRequest>,
@@ -2242,6 +2242,7 @@ mod test {
     use assert_matches::assert_matches;
     use near_async::messaging::IntoSender;
     use near_async::time::FakeClock;
+    use near_chain_configs::MutableConfigValue;
     use near_epoch_manager::shard_tracker::TrackedConfig;
     use near_epoch_manager::test_utils::setup_epoch_manager_with_block_and_chunk_producers;
     use near_network::test_utils::MockPeerManagerAdapter;
@@ -2257,9 +2258,7 @@ mod test {
     use crate::logic::persist_chunk;
     use crate::test_utils::*;
 
-    fn mutable_validator_signer(
-        account_id: &AccountId,
-    ) -> MutableConfigValue<Option<Arc<ValidatorSigner>>> {
+    fn mutable_validator_signer(account_id: &AccountId) -> MutableValidatorSigner {
         MutableConfigValue::new(
             Some(Arc::new(EmptyValidatorSigner::new(account_id.clone()))),
             "validator_signer",

chain/client/src/client.rs

@@ -40,7 +40,7 @@ use near_chain::{
     DoomslugThresholdMode, Provenance,
 };
 use near_chain_configs::{
-    ClientConfig, LogSummaryStyle, MutableConfigValue, UpdateableClientConfig,
+    ClientConfig, LogSummaryStyle, MutableValidatorSigner, UpdateableClientConfig,
 };
 use near_chunks::adapter::ShardsManagerRequestFromClient;
 use near_chunks::client::ShardedTransactionPool;
@@ -150,7 +150,7 @@ pub struct Client {
     /// Signer for block producer (if present). This field is mutable and optional. Use with caution!
     /// Lock the value of mutable validator signer for the duration of a request to ensure consistency.
     /// Please note that the locked value should not be stored anywhere or passed through the thread boundary.
-    pub validator_signer: MutableConfigValue<Option<Arc<ValidatorSigner>>>,
+    pub validator_signer: MutableValidatorSigner,
     /// Approvals for which we do not have the block yet
     pub pending_approvals:
         lru::LruCache<ApprovalInner, HashMap<AccountId, (Approval, ApprovalType)>>,
@@ -202,12 +202,24 @@ pub struct Client {
 }
 
 impl Client {
-    pub(crate) fn update_client_config(&self, update_client_config: UpdateableClientConfig) {
-        self.config.expected_shutdown.update(update_client_config.expected_shutdown);
-        self.config.resharding_config.update(update_client_config.resharding_config);
-        self.config
+    pub(crate) fn update_client_config(
+        &self,
+        update_client_config: UpdateableClientConfig,
+    ) -> bool {
+        let mut is_updated = false;
+        is_updated |= self.config.expected_shutdown.update(update_client_config.expected_shutdown);
+        is_updated |= self.config.resharding_config.update(update_client_config.resharding_config);
+        is_updated |= self
+            .config
             .produce_chunk_add_transactions_time_limit
             .update(update_client_config.produce_chunk_add_transactions_time_limit);
+        is_updated
+    }
+
+    /// Updates client's mutable validator signer.
+    /// It will update all validator signers that synchronize with it.
+    pub(crate) fn update_validator_signer(&self, signer: Arc<ValidatorSigner>) -> bool {
+        self.validator_signer.update(Some(signer))
     }
 }
 
@@ -253,7 +265,7 @@ impl Client {
         runtime_adapter: Arc<dyn RuntimeAdapter>,
         network_adapter: PeerManagerAdapter,
         shards_manager_adapter: Sender<ShardsManagerRequestFromClient>,
-        validator_signer: MutableConfigValue<Option<Arc<ValidatorSigner>>>,
+        validator_signer: MutableValidatorSigner,
         enable_doomslug: bool,
         rng_seed: RngSeed,
         snapshot_callbacks: Option<SnapshotCallbacks>,

chain/client/src/client_actor.rs

@@ -41,7 +41,7 @@ use near_chain::{
     byzantine_assert, near_chain_primitives, Block, BlockHeader, BlockProcessingArtifact,
     ChainGenesis, Provenance,
 };
-use near_chain_configs::{ClientConfig, LogSummaryStyle, MutableConfigValue, ReshardingHandle};
+use near_chain_configs::{ClientConfig, LogSummaryStyle, MutableValidatorSigner, ReshardingHandle};
 use near_chain_primitives::error::EpochErrorResultToChainError;
 use near_chunks::adapter::ShardsManagerRequestFromClient;
 use near_chunks::client::ShardsManagerResponse;
@@ -134,7 +134,7 @@ pub fn start_client(
     state_sync_adapter: Arc<RwLock<SyncAdapter>>,
     network_adapter: PeerManagerAdapter,
     shards_manager_adapter: Sender<ShardsManagerRequestFromClient>,
-    validator_signer: MutableConfigValue<Option<Arc<ValidatorSigner>>>,
+    validator_signer: MutableValidatorSigner,
     telemetry_sender: Sender<TelemetryEvent>,
     snapshot_callbacks: Option<SnapshotCallbacks>,
     sender: Option<broadcast::Sender<()>>,
@@ -1166,9 +1166,17 @@ impl ClientActorInner {
     fn check_triggers(&mut self, ctx: &mut dyn DelayedActionRunner<Self>) -> Duration {
         let _span = tracing::debug_span!(target: "client", "check_triggers").entered();
         if let Some(config_updater) = &mut self.config_updater {
-            config_updater.try_update(&|updateable_client_config| {
-                self.client.update_client_config(updateable_client_config)
-            });
+            let update_result = config_updater.try_update(
+                &|updateable_client_config| {
+                    self.client.update_client_config(updateable_client_config)
+                },
+                &|validator_signer| self.client.update_validator_signer(validator_signer),
+            );
+            if update_result.validator_signer_updated {
+                // Request PeerManager to advertise tier1 proxies.
+                // It is needed to advertise that our validator key changed.
+                self.network_adapter.send(PeerManagerMessageRequest::AdvertiseTier1Proxies);
+            }
         }
 
         // Check block height to trigger expected shutdown

chain/client/src/config_updater.rs

@@ -1,5 +1,6 @@
 use near_chain_configs::UpdateableClientConfig;
 use near_dyn_configs::{UpdateableConfigLoaderError, UpdateableConfigs};
+use near_primitives::validator_signer::ValidatorSigner;
 use std::sync::Arc;
 use tokio::sync::broadcast::Receiver;
 
@@ -12,6 +13,14 @@ pub struct ConfigUpdater {
     updateable_configs_error: Option<Arc<UpdateableConfigLoaderError>>,
 }
 
+/// Return type of `ConfigUpdater::try_update()`.
+/// Represents which values have been updated.
+#[derive(Default)]
+pub struct ConfigUpdaterResult {
+    pub client_config_updated: bool,
+    pub validator_signer_updated: bool,
+}
+
 impl ConfigUpdater {
     pub fn new(
         rx_config_update: Receiver<Result<UpdateableConfigs, Arc<UpdateableConfigLoaderError>>>,
@@ -21,21 +30,33 @@ impl ConfigUpdater {
 
     /// Check if any of the configs were updated.
     /// If they did, the receiver (rx_config_update) will contain a clone of the new configs.
-    pub fn try_update(&mut self, update_client_config_fn: &dyn Fn(UpdateableClientConfig)) {
+    pub fn try_update(
+        &mut self,
+        update_client_config_fn: &dyn Fn(UpdateableClientConfig) -> bool,
+        update_validator_signer_fn: &dyn Fn(Arc<ValidatorSigner>) -> bool,
+    ) -> ConfigUpdaterResult {
+        let mut update_result = ConfigUpdaterResult::default();
         while let Ok(maybe_updateable_configs) = self.rx_config_update.try_recv() {
             match maybe_updateable_configs {
                 Ok(updateable_configs) => {
                     if let Some(client_config) = updateable_configs.client_config {
-                        update_client_config_fn(client_config);
+                        update_result.client_config_updated |=
+                            update_client_config_fn(client_config);
                         tracing::info!(target: "config", "Updated ClientConfig");
                     }
+                    if let Some(validator_signer) = updateable_configs.validator_signer {
+                        update_result.validator_signer_updated |=
+                            update_validator_signer_fn(validator_signer);
+                        tracing::info!(target: "config", "Updated validator key");
+                    }
                     self.updateable_configs_error = None;
                 }
                 Err(err) => {
                     self.updateable_configs_error = Some(err.clone());
                 }
             }
         }
+        update_result
     }
 
     /// Prints an error if it's present.

chain/client/src/stateless_validation/partial_witness/partial_witness_actor.rs

@@ -5,7 +5,7 @@ use near_async::messaging::{Actor, CanSend, Handler, Sender};
 use near_async::time::Clock;
 use near_async::{MultiSend, MultiSendMessage, MultiSenderFrom};
 use near_chain::Error;
-use near_chain_configs::MutableConfigValue;
+use near_chain_configs::MutableValidatorSigner;
 use near_epoch_manager::EpochManagerAdapter;
 use near_network::state_witness::{
     ChunkStateWitnessAckMessage, PartialEncodedStateWitnessForwardMessage,
@@ -36,7 +36,7 @@ pub struct PartialWitnessActor {
     /// Validator signer to sign the state witness. This field is mutable and optional. Use with caution!
     /// Lock the value of mutable validator signer for the duration of a request to ensure consistency.
     /// Please note that the locked value should not be stored anywhere or passed through the thread boundary.
-    my_signer: MutableConfigValue<Option<Arc<ValidatorSigner>>>,
+    my_signer: MutableValidatorSigner,
     /// Epoch manager to get the set of chunk validators
     epoch_manager: Arc<dyn EpochManagerAdapter>,
     /// Tracks the parts of the state witness sent from chunk producers to chunk validators.
@@ -107,7 +107,7 @@ impl PartialWitnessActor {
         clock: Clock,
         network_adapter: PeerManagerAdapter,
         client_sender: ClientSenderForPartialWitness,
-        my_signer: MutableConfigValue<Option<Arc<ValidatorSigner>>>,
+        my_signer: MutableValidatorSigner,
         epoch_manager: Arc<dyn EpochManagerAdapter>,
         store: Store,
     ) -> Self {

chain/client/src/view_client_actor.rs

@@ -13,7 +13,7 @@ use near_chain::types::{RuntimeAdapter, Tip};
 use near_chain::{
     get_epoch_block_producers_view, Chain, ChainGenesis, ChainStoreAccess, DoomslugThresholdMode,
 };
-use near_chain_configs::{ClientConfig, MutableConfigValue, ProtocolConfigView};
+use near_chain_configs::{ClientConfig, MutableValidatorSigner, ProtocolConfigView};
 use near_chain_primitives::error::EpochErrorResultToChainError;
 use near_client_primitives::types::{
     Error, GetBlock, GetBlockError, GetBlockProof, GetBlockProofError, GetBlockProofResponse,
@@ -99,7 +99,7 @@ pub struct ViewClientActorInner {
     /// Validator account (if present). This field is mutable and optional. Use with caution!
     /// Lock the value of mutable validator signer for the duration of a request to ensure consistency.
     /// Please note that the locked value should not be stored anywhere or passed through the thread boundary.
-    validator: MutableConfigValue<Option<Arc<ValidatorSigner>>>,
+    validator: MutableValidatorSigner,
     chain: Chain,
     epoch_manager: Arc<dyn EpochManagerAdapter>,
     shard_tracker: ShardTracker,
@@ -128,7 +128,7 @@ impl ViewClientActorInner {
 
     pub fn spawn_actix_actor(
         clock: Clock,
-        validator: MutableConfigValue<Option<Arc<ValidatorSigner>>>,
+        validator: MutableValidatorSigner,
         chain_genesis: ChainGenesis,
         epoch_manager: Arc<dyn EpochManagerAdapter>,
         shard_tracker: ShardTracker,

chain/network/src/config.rs

@@ -10,6 +10,7 @@ use crate::types::ROUTED_MESSAGE_TTL;
 use anyhow::Context;
 use near_async::time;
 use near_chain_configs::MutableConfigValue;
+use near_chain_configs::MutableValidatorSigner;
 use near_crypto::{KeyType, SecretKey};
 use near_primitives::network::PeerId;
 use near_primitives::test_utils::create_test_signer;
@@ -60,7 +61,7 @@ pub struct ValidatorConfig {
     /// Contains signer key for this node. This field is mutable and optional. Use with caution!
     /// Lock the value of mutable validator signer for the duration of a request to ensure consistency.
     /// Please note that the locked value should not be stored anywhere or passed through the thread boundary.
-    pub signer: MutableConfigValue<Option<Arc<ValidatorSigner>>>,
+    pub signer: MutableValidatorSigner,
     pub proxies: ValidatorProxies,
 }
 
@@ -241,7 +242,7 @@ impl NetworkConfig {
     pub fn new(
         cfg: crate::config_json::Config,
         node_key: SecretKey,
-        validator_signer: MutableConfigValue<Option<Arc<ValidatorSigner>>>,
+        validator_signer: MutableValidatorSigner,
         archive: bool,
     ) -> anyhow::Result<Self> {
         if cfg.public_addrs.len() > MAX_PEER_ADDRS {

chain/network/src/peer_manager/peer_manager_actor.rs

@@ -1026,6 +1026,14 @@ impl PeerManagerActor {
                     self.handle_msg_network_requests(msg, ctx),
                 )
             }
+            PeerManagerMessageRequest::AdvertiseTier1Proxies => {
+                let state = self.state.clone();
+                let clock = self.clock.clone();
+                ctx.spawn(wrap_future(async move {
+                    state.tier1_advertise_proxies(&clock).await;
+                }));
+                PeerManagerMessageResponse::AdvertiseTier1Proxies
+            }
             PeerManagerMessageRequest::OutboundTcpConnect(stream) => {
                 let peer_addr = stream.peer_addr;
                 if let Err(err) =

chain/network/src/types.rs

@@ -162,6 +162,10 @@ pub struct SetChainInfo(pub ChainInfo);
 #[rtype(result = "PeerManagerMessageResponse")]
 pub enum PeerManagerMessageRequest {
     NetworkRequests(NetworkRequests),
+    /// Request PeerManager to call `tier1_advertise_proxies()`. Used internally.
+    /// The effect would be accounts data known by this node broadcasted to other tier1 nodes.
+    /// That includes info about validator signer of this node.
+    AdvertiseTier1Proxies,
     /// Request PeerManager to connect to the given peer.
     /// Used in tests and internally by PeerManager.
     /// TODO: replace it with AsyncContext::spawn/run_later for internal use.
@@ -193,6 +197,7 @@ impl PeerManagerMessageRequest {
 #[derive(actix::MessageResponse, Debug)]
 pub enum PeerManagerMessageResponse {
     NetworkResponses(NetworkResponses),
+    AdvertiseTier1Proxies,
     /// TEST-ONLY
     OutboundTcpConnect,
     FetchRoutingTable(RoutingTableInfo),

core/chain-configs/src/lib.rs

@@ -30,7 +30,7 @@ pub use genesis_config::{
 };
 use near_primitives::types::{Balance, BlockHeightDelta, Gas, NumBlocks, NumSeats};
 use num_rational::Rational32;
-pub use updateable_config::{MutableConfigValue, UpdateableClientConfig};
+pub use updateable_config::{MutableConfigValue, MutableValidatorSigner, UpdateableClientConfig};
 
 pub const GENESIS_CONFIG_FILENAME: &str = "genesis.json";
 

core/chain-configs/src/updateable_config.rs

@@ -1,6 +1,7 @@
 #[cfg(feature = "metrics")]
 use near_async::time::Clock;
 use near_primitives::types::BlockHeight;
+use near_primitives::validator_signer::ValidatorSigner;
 use serde::{Deserialize, Serialize, Serializer};
 use std::fmt::Debug;
 use std::sync::{Arc, Mutex};
@@ -108,3 +109,5 @@ pub struct UpdateableClientConfig {
     #[serde(with = "near_async::time::serde_opt_duration_as_std")]
     pub produce_chunk_add_transactions_time_limit: Option<Duration>,
 }
+
+pub type MutableValidatorSigner = MutableConfigValue<Option<Arc<ValidatorSigner>>>;