IF: Use finalizer diffs in instant_finality_extension

libraries/chain/block_header_state.cpp

@@ -22,8 +22,7 @@ digest_type block_header_state::compute_base_digest() const {
       fc::raw::pack( enc, fp_pair.first );
       const finalizer_policy_tracker& tracker = fp_pair.second;
       fc::raw::pack( enc, tracker.state );
-      assert(tracker.policy);
-      fc::raw::pack( enc, *tracker.policy );
+      fc::raw::pack( enc, tracker.policy_diff );
    }
 
    assert(active_proposer_policy);
@@ -136,13 +135,14 @@ void finish_next(const block_header_state& prev,
             if (tracker.state == finalizer_policy_tracker::state_t::pending) {
                // new finalizer_policy becones active
                // -----------------------------------
-               next_header_state.active_finalizer_policy.reset(new finalizer_policy(*tracker.policy));
+               next_header_state.active_finalizer_policy.reset(new finalizer_policy(*prev.active_finalizer_policy));
+               next_header_state.active_finalizer_policy->apply_diff(tracker.policy_diff);
             } else {
                assert(tracker.state == finalizer_policy_tracker::state_t::proposed);
                // block where finalizer_policy was proposed became final. The finalizer policy will
                // become active when next block becomes final.
                // ---------------------------------------------------------------------------------
-               finalizer_policy_tracker t { finalizer_policy_tracker::state_t::pending, tracker.policy };
+               finalizer_policy_tracker t { finalizer_policy_tracker::state_t::pending, tracker.policy_diff };
                next_header_state.finalizer_policies.emplace(next_header_state.block_num(), std::move(t));
             }
             ++it;
@@ -156,19 +156,19 @@ void finish_next(const block_header_state& prev,
       }
    }
 
-   if (if_ext.new_finalizer_policy) {
+   if (if_ext.new_finalizer_policy_diff) {
       // a new `finalizer_policy` was proposed in the previous block, and is present in the previous
       // block's header extensions.
       // Add this new proposal to the `finalizer_policies` multimap which tracks the in-flight proposals,
       // increment the generation number, and log that proposal (debug level).
       // ------------------------------------------------------------------------------------------------
       dlog("New finalizer policy proposed in block ${id}: ${pol}",
-           ("id", prev.block_id)("pol", *if_ext.new_finalizer_policy));
-      next_header_state.finalizer_policy_generation = if_ext.new_finalizer_policy->generation;
+           ("id", prev.block_id)("pol", *if_ext.new_finalizer_policy_diff));
+      next_header_state.finalizer_policy_generation = if_ext.new_finalizer_policy_diff->generation;
       next_header_state.finalizer_policies.emplace(
          next_header_state.block_num(),
          finalizer_policy_tracker{finalizer_policy_tracker::state_t::proposed,
-                                  std::make_shared<finalizer_policy>(std::move(*if_ext.new_finalizer_policy))});
+                                  std::move(*if_ext.new_finalizer_policy_diff)});
 
    } else {
       next_header_state.finalizer_policy_generation = prev.finalizer_policy_generation;
@@ -205,8 +205,12 @@ block_header_state block_header_state::next(block_header_state_input& input) con
 
    // finality extension
    // ------------------
+   std::optional<finalizer_policy_diff> new_finalizer_policy_diff;
+   if (input.new_finalizer_policy) {
+      new_finalizer_policy_diff = calculate_finalizer_policy_diff(*input.new_finalizer_policy);
+   }
    instant_finality_extension new_if_ext { input.most_recent_ancestor_with_qc,
-                                           std::move(input.new_finalizer_policy),
+                                           std::move(new_finalizer_policy_diff),
                                            std::move(input.new_proposer_policy) };
 
    uint16_t if_ext_id = instant_finality_extension::extension_id();

libraries/chain/block_header_state_legacy.cpp

@@ -220,8 +220,10 @@ namespace eosio::chain {
          // set current block_num as qc_claim.last_qc_block_num in the IF extension
          qc_claim_t initial_if_claim { .block_num = block_num,
                                        .is_strong_qc = false };
+         finalizer_policy no_policy;
+         auto new_fin_policy_diff = no_policy.create_diff(*new_finalizer_policy);
          emplace_extension(h.header_extensions, instant_finality_extension::extension_id(),
-                           fc::raw::pack(instant_finality_extension{ initial_if_claim, std::move(new_finalizer_policy), {} }));
+                           fc::raw::pack(instant_finality_extension{ initial_if_claim, std::move(new_fin_policy_diff), {} }));
       } else if (qc_claim) {
          emplace_extension(h.header_extensions, instant_finality_extension::extension_id(),
                            fc::raw::pack(instant_finality_extension{ *qc_claim, {}, {} }));

libraries/chain/block_state.cpp

@@ -72,8 +72,9 @@ block_state_ptr block_state::create_if_genesis_block(const block_state_legacy& b
 
    assert(bsp.block->contains_header_extension(instant_finality_extension::extension_id())); // required by transition mechanism
    instant_finality_extension if_ext = bsp.block->extract_header_extension<instant_finality_extension>();
-   assert(if_ext.new_finalizer_policy); // required by transition mechanism
-   result.active_finalizer_policy = std::make_shared<finalizer_policy>(*if_ext.new_finalizer_policy);
+   assert(if_ext.new_finalizer_policy_diff); // required by transition mechanism
+   result.active_finalizer_policy = std::make_shared<finalizer_policy>();
+   result.active_finalizer_policy->apply_diff(std::move(*if_ext.new_finalizer_policy_diff));
    result.active_proposer_policy = std::make_shared<proposer_policy>();
    result.active_proposer_policy->active_time = bsp.timestamp();
    result.active_proposer_policy->proposer_schedule = bsp.active_schedule;

libraries/chain/include/eosio/chain/block_header_state.hpp

@@ -46,8 +46,8 @@ struct finality_digest_data_v1 {
 // ------------------------------------------------------------------------------------------
 struct finalizer_policy_tracker {
    enum class state_t { proposed = 0, pending };
-   state_t               state;
-   finalizer_policy_ptr  policy;
+   state_t                    state;
+   finalizer_policy_diff      policy_diff;
 };
 
 struct building_block_input {
@@ -129,6 +129,24 @@ struct block_header_state {
 
    const vector<digest_type>& get_new_protocol_feature_activations() const;
    const producer_authority& get_scheduled_producer(block_timestamp_type t) const;
+
+   finalizer_policy_diff calculate_finalizer_policy_diff(const finalizer_policy& new_policy) const {
+      if (finalizer_policies.empty()) {
+         return active_finalizer_policy->create_diff(new_policy);
+      }
+      finalizer_policy fin_policy = *active_finalizer_policy;
+      for (const auto& e : finalizer_policies) {
+         if (e.second.state == finalizer_policy_tracker::state_t::pending) {
+            fin_policy.apply_diff(e.second.policy_diff);
+         }
+      }
+      for (const auto& e : finalizer_policies) {
+         if (e.second.state == finalizer_policy_tracker::state_t::proposed) {
+            fin_policy.apply_diff(e.second.policy_diff);
+         }
+      }
+      return fin_policy.create_diff(new_policy);
+   }
 };
 
 using block_header_state_ptr = std::shared_ptr<block_header_state>;
@@ -137,7 +155,7 @@ using block_header_state_ptr = std::shared_ptr<block_header_state>;
 
 FC_REFLECT_ENUM( eosio::chain::finalizer_policy_tracker::state_t, (proposed)(pending))
 
-FC_REFLECT( eosio::chain::finalizer_policy_tracker, (state)(policy))
+FC_REFLECT( eosio::chain::finalizer_policy_tracker, (state)(policy_diff))
 
 FC_REFLECT( eosio::chain::block_header_state, (block_id)(header)
             (activated_protocol_features)(core)(active_finalizer_policy)

libraries/chain/include/eosio/chain/finality/finalizer_policy.hpp

@@ -2,13 +2,37 @@
 
 #include <eosio/chain/types.hpp>
 #include <eosio/chain/finality/finalizer_authority.hpp>
+#include <fc/container/ordered_diff.hpp>
 
 namespace eosio::chain {
 
+   using finalizers_differ = fc::ordered_diff<finalizer_authority>;
+   using finalizers_diff_t = fc::ordered_diff<finalizer_authority>::diff_result;
+
+   struct finalizer_policy_diff {
+      uint32_t                         generation = 0; ///< sequentially incrementing version number
+      uint64_t                         threshold = 0;  ///< vote weight threshold to finalize blocks
+      finalizers_diff_t                finalizers_diff;
+   };
+
    struct finalizer_policy {
       uint32_t                         generation = 0; ///< sequentially incrementing version number
       uint64_t                         threshold = 0;  ///< vote weight threshold to finalize blocks
-      std::vector<finalizer_authority> finalizers; ///< Instant Finality voter set
+      std::vector<finalizer_authority> finalizers;     ///< Instant Finality voter set
+
+      finalizer_policy_diff create_diff(const finalizer_policy& target) const {
+         return {.generation = target.generation,
+                 .threshold = target.threshold,
+                 .finalizers_diff = finalizers_differ::diff(finalizers, target.finalizers)};
+      }
+
+      template <typename X>
+      requires std::same_as<std::decay_t<X>, finalizer_policy_diff>
+      void apply_diff(X&& diff) {
+         generation = diff.generation;
+         threshold = diff.threshold;
+         finalizers_differ::apply_diff(finalizers, std::move(diff.finalizers_diff));
+      }
 
       // max accumulated weak weight before becoming weak_final
       uint64_t max_weak_sum_before_weak_final() const {
@@ -23,7 +47,10 @@ namespace eosio::chain {
    };
 
    using finalizer_policy_ptr = std::shared_ptr<finalizer_policy>;
+   using finalizer_policy_diff_ptr = std::shared_ptr<finalizer_policy_diff>;
 
 } /// eosio::chain
 
 FC_REFLECT( eosio::chain::finalizer_policy, (generation)(threshold)(finalizers) )
+FC_REFLECT( eosio::chain::finalizers_diff_t, (remove_indexes)(insert_indexes) )
+FC_REFLECT( eosio::chain::finalizer_policy_diff, (generation)(threshold)(finalizers_diff) )

libraries/chain/include/eosio/chain/finality/instant_finality_extension.hpp

@@ -12,10 +12,10 @@ struct instant_finality_extension : fc::reflect_init {
 
    instant_finality_extension() = default;
    instant_finality_extension(qc_claim_t qc_claim,
-                              std::optional<finalizer_policy> new_finalizer_policy,
+                              std::optional<finalizer_policy_diff> new_finalizer_policy_diff,
                               std::shared_ptr<proposer_policy> new_proposer_policy) :
       qc_claim(qc_claim),
-      new_finalizer_policy(std::move(new_finalizer_policy)),
+      new_finalizer_policy_diff(std::move(new_finalizer_policy_diff)),
       new_proposer_policy(std::move(new_proposer_policy))
    {}
 
@@ -25,11 +25,11 @@ struct instant_finality_extension : fc::reflect_init {
       static_assert( extension_id() == 2, "instant_finality_extension extension id must be 2" );
    }
 
-   qc_claim_t                         qc_claim;
-   std::optional<finalizer_policy>    new_finalizer_policy;
-   std::shared_ptr<proposer_policy>   new_proposer_policy;
+   qc_claim_t                              qc_claim;
+   std::optional<finalizer_policy_diff>    new_finalizer_policy_diff;
+   std::shared_ptr<proposer_policy>        new_proposer_policy;
 };
 
 } /// eosio::chain
 
-FC_REFLECT( eosio::chain::instant_finality_extension, (qc_claim)(new_finalizer_policy)(new_proposer_policy) )
+FC_REFLECT( eosio::chain::instant_finality_extension, (qc_claim)(new_finalizer_policy_diff)(new_proposer_policy) )

libraries/libfc/include/fc/container/ordered_diff.hpp

@@ -0,0 +1,99 @@
+#pragma once
+
+#include <vector>
+#include <utility>
+
+namespace fc {
+
+/**
+ * @class ordered_diff
+ * @brief Provides ablity to generate and apply diff of containers of type T
+ *
+ * Example use:
+ *    std::vector<char> source = { 'a', 'b', 'f', 'c', 'd' };
+ *    std::vector<char> target = { 'b', 'f', 'c', 'd', 'e', 'h' };
+ *    ordered_diff<char>::diff_result diff = ordered_diff<char>::diff(source, target);
+ *    ordered_diff<char>::apply_diff(source, std::move(diff));
+ *    assert(source == target);
+ *
+ * @param T type stored in Container, must provide ==
+ */
+template <typename T, template<typename Y, typename...> typename Container = std::vector>
+requires std::equality_comparable<T> && std::random_access_iterator<typename Container<T>::iterator>
+class ordered_diff {
+public:
+   struct diff_result {
+      Container<size_t>                remove_indexes;
+      Container<std::pair<size_t, T>>  insert_indexes;
+   };
+
+   /// Generate diff_result that when `apply_diff(source, diff_result)` to source will generate target
+   static diff_result diff(const Container<T>& source, const Container<T>& target) {
+      size_t s = 0;
+      size_t t = 0;
+
+      diff_result result;
+      while (s < source.size() || t < target.size()) {
+         if (s < source.size() && t < target.size()) {
+            if (source[s] == target[t]) {
+               // nothing to do, skip over
+               ++s;
+               ++t;
+            } else { // not equal
+               if (s == source.size() - 1 && t == target.size() - 1) {
+                  // both at end, insert target and remove source
+                  result.remove_indexes.push_back(s);
+                  result.insert_indexes.emplace_back(t, target[t]);
+                  ++s;
+                  ++t;
+               } else if (s + 1 < source.size() && t + 1 < target.size() && source[s + 1] == target[t + 1]) {
+                  // misalignment, but next value equal, insert and remove
+                  result.remove_indexes.push_back(s);
+                  result.insert_indexes.emplace_back(t, target[t]);
+                  ++s;
+                  ++t;
+               } else if (t + 1 < target.size() && source[s] == target[t + 1]) {
+                  // source equals next target, insert current target
+                  result.insert_indexes.emplace_back(t, target[t]);
+                  ++t;
+               } else { // source[s + 1] == target[t]
+                  // target matches next source, remove current source
+                  result.remove_indexes.push_back(s);
+                  ++s;
+               }
+            }
+         } else if (s < source.size()) {
+            // remove extra in source
+            result.remove_indexes.push_back(s);
+            ++s;
+         } else if (t < target.size()) {
+            // insert extra in target
+            result.insert_indexes.emplace_back(t, target[t]);
+            ++t;
+         }
+      }
+
+      return result;
+   }
+
+   template <typename X>
+   requires std::same_as<std::decay_t<X>, diff_result>
+   static void apply_diff(Container<T>& source, X&& diff) {
+      // Remove from the source based on diff.remove_indexes
+      std::ptrdiff_t offset = 0;
+      for (size_t index : diff.remove_indexes) {
+         source.erase(source.begin() + index + offset);
+         --offset;
+      }
+
+      // Insert into the source based on diff.insert_indexes
+      for (auto& t : diff.insert_indexes) {
+         size_t index = std::get<0>(t);
+         auto&  value = std::get<1>(t);
+         source.insert(source.begin() + index, std::move(value));
+      }
+   }
+
+}; // class ordered_diff
+
+} // namespace fc

libraries/libfc/test/CMakeLists.txt

@@ -19,6 +19,7 @@ add_executable( test_fc
         test_base64.cpp
         test_escape_str.cpp
         test_bls.cpp
+        test_ordered_diff.cpp
         main.cpp
         )
 target_link_libraries( test_fc fc )