Merge remote-tracking branch 'origin/hotstuff_integration' into GH-2334…

…-bls-serialization

.cicd/platforms/ubsan.Dockerfile

@@ -0,0 +1,42 @@
+# syntax=docker/dockerfile:1
+FROM ubuntu:jammy
+ENV TZ="America/New_York"
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && apt-get upgrade -y && \
+    apt-get install -y build-essential      \
+                       cmake                \
+                       git                  \
+                       jq                   \
+                       libcurl4-openssl-dev \
+                       libgmp-dev           \
+                       llvm-11-dev          \
+                       lsb-release          \
+                       ninja-build          \
+                       python3-numpy        \
+                       software-properties-common \
+                       file                 \
+                       wget                 \
+                       zlib1g-dev           \
+                       zstd
+
+RUN yes | bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" llvm.sh 18
+
+#make sure no confusion on what llvm library leap's cmake should pick up on
+RUN rm -rf /usr/lib/llvm-18/lib/cmake
+
+COPY <<-EOF /ubsan.supp
+  vptr:wasm_eosio_validation.hpp
+  vptr:wasm_eosio_injection.hpp
+EOF
+
+ENV LEAP_PLATFORM_HAS_EXTRAS_CMAKE=1
+COPY <<-EOF /extras.cmake
+  set(CMAKE_BUILD_TYPE "RelWithDebInfo" CACHE STRING "" FORCE)
+
+  set(CMAKE_C_COMPILER "clang-18" CACHE STRING "")
+  set(CMAKE_CXX_COMPILER "clang++-18" CACHE STRING "")
+  set(CMAKE_C_FLAGS "-fsanitize=undefined -fno-sanitize-recover=all -fno-omit-frame-pointer" CACHE STRING "")
+  set(CMAKE_CXX_FLAGS "-fsanitize=undefined -fno-sanitize-recover=all -fno-omit-frame-pointer" CACHE STRING "")
+EOF
+
+ENV UBSAN_OPTIONS=print_stacktrace=1,suppressions=/ubsan.supp

.github/workflows/build.yaml

@@ -136,6 +136,7 @@ jobs:
           - cfg: {name: 'ubuntu20',      base: 'ubuntu20', builddir: 'ubuntu20'}
           - cfg: {name: 'ubuntu22',      base: 'ubuntu22', builddir: 'ubuntu22'}
           - cfg: {name: 'asserton',      base: 'asserton', builddir: 'asserton'}
+          - cfg: {name: 'ubsan',         base: 'ubsan',    builddir: 'ubsan'}
           - cfg: {name: 'ubuntu20repro', base: 'ubuntu20', builddir: 'reproducible'}
           - cfg: {name: 'ubuntu22repro', base: 'ubuntu22', builddir: 'reproducible'}
     runs-on: ["self-hosted", "enf-x86-hightier"]
@@ -176,6 +177,7 @@ jobs:
           - cfg: {name: 'ubuntu20',      base: 'ubuntu20', builddir: 'ubuntu20'}
           - cfg: {name: 'ubuntu22',      base: 'ubuntu22', builddir: 'ubuntu22'}
           - cfg: {name: 'asserton',      base: 'asserton', builddir: 'asserton'}
+          - cfg: {name: 'ubsan',         base: 'ubsan',    builddir: 'ubsan'}
           - cfg: {name: 'ubuntu20repro', base: 'ubuntu20', builddir: 'reproducible'}
           - cfg: {name: 'ubuntu22repro', base: 'ubuntu22', builddir: 'reproducible'}
     runs-on: ["self-hosted", "enf-x86-midtier"]
@@ -216,6 +218,7 @@ jobs:
           - cfg: {name: 'ubuntu20',      base: 'ubuntu20', builddir: 'ubuntu20'}
           - cfg: {name: 'ubuntu22',      base: 'ubuntu22', builddir: 'ubuntu22'}
           - cfg: {name: 'asserton',      base: 'asserton', builddir: 'asserton'}
+          - cfg: {name: 'ubsan',         base: 'ubsan',    builddir: 'ubsan'}
           - cfg: {name: 'ubuntu20repro', base: 'ubuntu20', builddir: 'reproducible'}
           - cfg: {name: 'ubuntu22repro', base: 'ubuntu22', builddir: 'reproducible'}
     runs-on: ["self-hosted", "enf-x86-lowtier"]

libraries/chain/fork_database.cpp

@@ -686,7 +686,7 @@ namespace eosio::chain {
 
       // check that fork_dbs are in a consistent state
       if (!legacy_valid && !savanna_valid) {
-         elog( "fork_database is in a bad state when closing; not writing out '${filename}', legacy_valid=${l}, savanna_valid=${s}",
+         wlog( "fork_database is in a bad state when closing; not writing out '${filename}', legacy_valid=${l}, savanna_valid=${s}",
                ("filename", fork_db_file)("l", legacy_valid)("s", savanna_valid) );
          return;
       }

libraries/chain/include/eosio/chain/chainbase_environment.hpp

@@ -0,0 +1,25 @@
+#pragma once
+
+#include <chainbase/environment.hpp>
+
+// reflect chainbase::environment for --print-build-info option
+FC_REFLECT_ENUM(chainbase::environment::os_t,
+                (OS_LINUX) (OS_MACOS) (OS_WINDOWS) (OS_OTHER))
+FC_REFLECT_ENUM(chainbase::environment::arch_t,
+                (ARCH_X86_64) (ARCH_ARM) (ARCH_RISCV) (ARCH_OTHER))
+
+namespace fc {
+
+void to_variant(const chainbase::environment& bi, variant& v) {
+   // the variant conversion ultimately binds a reference to each member, but chainbase::environment is packed making
+   // a reference to an unaligned variable UB. The boost_version is the only offender
+   unsigned aligned_boost_version = bi.boost_version;
+   v = fc::mutable_variant_object()("debug", bi.debug)
+                                   ("os", bi.os)
+                                   ("arch", bi.arch)
+                                   ("boost_version", aligned_boost_version)
+                                   ("compiler", bi.compiler);
+
+}
+
+}

libraries/chain/include/eosio/chain/protocol_feature_manager.hpp

@@ -3,7 +3,7 @@
 #include <eosio/chain/types.hpp>
 #include <iterator>
 
-namespace eosio { namespace chain {
+namespace eosio::chain {
 
 class deep_mind_handler;
 
@@ -403,7 +403,7 @@ class protocol_feature_manager {
 std::optional<builtin_protocol_feature> read_builtin_protocol_feature( const std::filesystem::path& p  );
 protocol_feature_set initialize_protocol_features( const std::filesystem::path& p, bool populate_missing_builtins = true );
 
-} } // namespace eosio::chain
+} // namespace eosio::chain
 
 FC_REFLECT(eosio::chain::protocol_feature_subjective_restrictions,
                (earliest_allowed_activation_time)(preactivation_required)(enabled))

libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/config.hpp

@@ -9,6 +9,10 @@
 
 #include <sys/resource.h>
 
+#ifndef __has_feature
+#define __has_feature(x) 0
+#endif
+
 namespace eosio { namespace chain { namespace eosvmoc {
 
 struct config {
@@ -20,7 +24,11 @@ struct config {
    // libtester disables the limits in all tests, except enforces the limits
    // in the tests in unittests/eosvmoc_limits_tests.cpp.
    std::optional<rlim_t>   cpu_limit {20u};
+#if __has_feature(undefined_behavior_sanitizer) || __has_feature(address_sanitizer)
+   std::optional<rlim_t>   vm_limit; // UBSAN & ASAN can add massive virtual memory usage; don't enforce vm limits when either of them are enabled
+#else
    std::optional<rlim_t>   vm_limit  {512u*1024u*1024u};
+#endif
    std::optional<uint64_t> stack_size_limit {16u*1024u};
    std::optional<size_t>   generated_code_size_limit {16u*1024u*1024u};
 };

libraries/chain/protocol_feature_manager.cpp

@@ -310,12 +310,23 @@ retires a deferred transaction is invalid.
          } )
          (  builtin_protocol_feature_t::instant_finality, builtin_protocol_feature_spec{
             "INSTANT_FINALITY",
-            fc::variant("bc726a24928ea2d71ba294b70c5c9efc515c1542139bcf9e42f8bc174f2e72ff").as<digest_type>(),
+            fc::variant("bd496b9e85ce61dcddeee4576ea185add87844238da992a9ee6df2a2bdb357c2").as<digest_type>(),
             // SHA256 hash of the raw message below within the comment delimiters (do not modify message below).
 /*
 Builtin protocol feature: INSTANT_FINALITY
+Depends on: DISALLOW_EMPTY_PRODUCER_SCHEDULE
+            WTMSIG_BLOCK_SIGNATURES
+            ACTION_RETURN_VALUE
+            BLS_PRIMITIVES2
+
+Once this protocol feature is activated, the first subsequent block including a `set_finalizers`
+host function call will trigger a switch to the Savanna consensus algorithm.
 */
-            {}
+            { builtin_protocol_feature_t::disallow_empty_producer_schedule,
+              builtin_protocol_feature_t::wtmsig_block_signatures,
+              builtin_protocol_feature_t::action_return_value,
+              builtin_protocol_feature_t::bls_primitives
+            }
          } )
    ;
 
@@ -592,8 +603,8 @@ Builtin protocol feature: INSTANT_FINALITY
          }
 
          EOS_THROW(  protocol_feature_validation_exception,
-                     "Not all the builtin dependencies of the builtin protocol feature with codename '${codename}' and digest of ${digest} were satisfied.",
-                     ("missing_dependencies", missing_builtins_with_names)
+                     "Not all the builtin dependencies of the builtin protocol feature with codename '${codename}' and digest of ${digest} were satisfied. Missing dependencies: ${missing_dependencies}",
+                     ("codename", f.builtin_feature_codename)("digest",feature_digest)("missing_dependencies", missing_builtins_with_names)
          );
       }
 
@@ -973,22 +984,23 @@ Builtin protocol feature: INSTANT_FINALITY
          auto file_path = p / filename;
 
          EOS_ASSERT( !std::filesystem::exists( file_path ), plugin_exception,
-                     "Could not save builtin protocol feature with codename '${codename}' because a file at the following path already exists: ${path}",
+                     "Could not save builtin protocol feature with codename '${codename}' because a file at "
+                     "the following path already exists: ${path}",
                      ("codename", builtin_protocol_feature_codename( f.get_codename() ))
-                           ("path", file_path)
+                     ("path", file_path)
          );
 
          if( fc::json::save_to_file( f, file_path ) ) {
             ilog( "Saved default specification for builtin protocol feature '${codename}' (with digest of '${digest}') to: ${path}",
                   ("codename", builtin_protocol_feature_codename(f.get_codename()))
-                        ("digest", feature_digest)
-                        ("path", file_path)
+                  ("digest", feature_digest)
+                  ("path", file_path)
             );
          } else {
             elog( "Error occurred while writing default specification for builtin protocol feature '${codename}' (with digest of '${digest}') to: ${path}",
                   ("codename", builtin_protocol_feature_codename(f.get_codename()))
-                        ("digest", feature_digest)
-                        ("path", file_path)
+                  ("digest", feature_digest)
+                  ("path", file_path)
             );
          }
       };

libraries/chain/snapshot.cpp

@@ -257,10 +257,7 @@ void istream_snapshot_reader::validate() const {
                  ("expected", expected_version)("actual", actual_version));
 
       while (validate_section()) {}
-   } catch( const std::exception& e ) {  \
-      snapshot_exception fce(FC_LOG_MESSAGE( warn, "Binary snapshot validation threw IO exception (${what})",("what",e.what())));
-      throw fce;
-   }
+   } FC_LOG_AND_RETHROW()
 }
 
 bool istream_snapshot_reader::validate_section() const {

libraries/chain/webassembly/runtimes/eos-vm-oc/compile_trampoline.cpp

@@ -122,7 +122,8 @@ void run_compile(wrapped_fd&& response_sock, wrapped_fd&& wasm_code, uint64_t st
 
       if(base_offset + data_segment.data.size() > initial_mem.size())
          initial_mem.resize(base_offset + data_segment.data.size(), 0x00);
-      memcpy(initial_mem.data() + base_offset, data_segment.data.data(), data_segment.data.size());
+      if(data_segment.data.size())
+         memcpy(initial_mem.data() + base_offset, data_segment.data.data(), data_segment.data.size());
    }
 
    result_message.initdata_prologue_size = prologue.end() - prologue_it;

libraries/chain/webassembly/runtimes/eos-vm-oc/memory.cpp

@@ -40,7 +40,7 @@ memory::memory(uint64_t sliced_pages) {
    uintptr_t* const intrinsic_jump_table = reinterpret_cast<uintptr_t* const>(zeropage_base - first_intrinsic_offset);
    const intrinsic_map_t& intrinsics = get_intrinsic_map();
    for(const auto& intrinsic : intrinsics)
-      intrinsic_jump_table[-intrinsic.second.ordinal] = (uintptr_t)intrinsic.second.function_ptr;
+      intrinsic_jump_table[-(int)intrinsic.second.ordinal] = (uintptr_t)intrinsic.second.function_ptr;
 }
 
 memory::~memory() {

libraries/testing/include/eosio/testing/tester.hpp

@@ -401,11 +401,12 @@ namespace eosio { namespace testing {
             return cfg;
          }
 
-         void schedule_protocol_features_wo_preactivation(const vector<digest_type> feature_digests);
-         void preactivate_protocol_features(const vector<digest_type> feature_digests);
+         void schedule_protocol_features_wo_preactivation(const vector<digest_type>& feature_digests);
+         void preactivate_protocol_features(const vector<digest_type>& feature_digests);
          void preactivate_builtin_protocol_features(const std::vector<builtin_protocol_feature_t>& features);
          void preactivate_all_builtin_protocol_features();
          void preactivate_all_but_disable_deferred_trx();
+         void preactivate_savanna_protocol_features();
 
          static genesis_state default_genesis() {
             genesis_state genesis;

libraries/testing/tester.cpp

@@ -1229,21 +1229,34 @@ namespace eosio { namespace testing {
       return tid;
    }
 
-   void base_tester::schedule_protocol_features_wo_preactivation(const vector<digest_type> feature_digests) {
+   void base_tester::schedule_protocol_features_wo_preactivation(const vector<digest_type>& feature_digests) {
       protocol_features_to_be_activated_wo_preactivation.insert(
          protocol_features_to_be_activated_wo_preactivation.end(),
          feature_digests.begin(),
          feature_digests.end()
       );
    }
 
-   void base_tester::preactivate_protocol_features(const vector<digest_type> feature_digests) {
+   void base_tester::preactivate_protocol_features(const vector<digest_type>& feature_digests) {
       for( const auto& feature_digest: feature_digests ) {
          push_action( config::system_account_name, "activate"_n, config::system_account_name,
                       fc::mutable_variant_object()("feature_digest", feature_digest) );
       }
    }
 
+   void base_tester::preactivate_savanna_protocol_features() {
+      const auto& pfm = control->get_protocol_feature_manager();
+      const auto& d = pfm.get_builtin_digest(builtin_protocol_feature_t::instant_finality);
+
+      // dependencies of builtin_protocol_feature_t::instant_finality
+      const auto& deps = pfm.get_builtin_digest(builtin_protocol_feature_t::disallow_empty_producer_schedule);
+      const auto& wtm  = pfm.get_builtin_digest(builtin_protocol_feature_t::wtmsig_block_signatures);
+      const auto& arv  = pfm.get_builtin_digest(builtin_protocol_feature_t::action_return_value);
+      const auto& bls   = pfm.get_builtin_digest(builtin_protocol_feature_t::bls_primitives);
+
+      preactivate_protocol_features( {*deps, *wtm, *arv, *bls, *d} );
+   }
+
    void base_tester::preactivate_builtin_protocol_features(const std::vector<builtin_protocol_feature_t>& builtins) {
       const auto& pfm = control->get_protocol_feature_manager();
       const auto& pfs = pfm.get_protocol_feature_set();

plugins/chain_plugin/chain_plugin.cpp

@@ -15,11 +15,10 @@
 #include <eosio/chain_plugin/trx_finality_status_processing.hpp>
 #include <eosio/chain/permission_link_object.hpp>
 #include <eosio/chain/global_property_object.hpp>
+#include <eosio/chain/chainbase_environment.hpp>
 
 #include <eosio/resource_monitor_plugin/resource_monitor_plugin.hpp>
 
-#include <chainbase/environment.hpp>
-
 #include <boost/signals2/connection.hpp>
 #include <boost/algorithm/string.hpp>
 #include <boost/lexical_cast.hpp>
@@ -28,13 +27,6 @@
 #include <fc/variant.hpp>
 #include <cstdlib>
 
-// reflect chainbase::environment for --print-build-info option
-FC_REFLECT_ENUM( chainbase::environment::os_t,
-                 (OS_LINUX)(OS_MACOS)(OS_WINDOWS)(OS_OTHER) )
-FC_REFLECT_ENUM( chainbase::environment::arch_t,
-                 (ARCH_X86_64)(ARCH_ARM)(ARCH_RISCV)(ARCH_OTHER) )
-FC_REFLECT(chainbase::environment, (debug)(os)(arch)(boost_version)(compiler) )
-
 const std::string deep_mind_logger_name("deep-mind");
 eosio::chain::deep_mind_handler _deep_mind_log;