Merge branch 'auxpow'

Added wallet locks that were missing from the original implementation in
#422 and failing the
name_deterministic_salt.py regtest with lock debugging turned on.

Removed unnecessary check for unsigned > 0, which was an oversight
from when a variable was changed from int to unsigned, fixing
#425.

.cirrus.yml

@@ -112,7 +112,7 @@ task:
     FILE_ENV: "./ci/test/00_setup_env_i686_centos.sh"
 
 task:
-  name: '[previous releases, uses qt5 dev package and some depends packages] [unsigned char] [bionic]'
+  name: '[previous releases, uses qt5 dev package and some depends packages, DEBUG] [unsigned char] [bionic]'
   previous_releases_cache:
     folder: "releases"
   << : *GLOBAL_TASK_TEMPLATE
@@ -161,7 +161,7 @@ task:
     FILE_ENV: "./ci/test/00_setup_env_native_fuzz.sh"
 
 task:
-  name: '[multiprocess] [focal]'
+  name: '[multiprocess, DEBUG] [focal]'
   << : *GLOBAL_TASK_TEMPLATE
   container:
     image: ubuntu:focal

REVIEWERS

@@ -19,6 +19,7 @@
 
 # Maintainers
 #   @fanquake
+#   @hebasto
 #   @jonasschnelli
 #   @laanwj
 #   @marcofalke
@@ -59,11 +60,16 @@
 /src/test/fuzz/                             @practicalswift
 /doc/fuzzing.md                             @practicalswift
 
-# Test framework
+# Tests
+/src/test/net_peer_eviction_tests.cpp       @jonatack
 /test/functional/mempool_updatefromblock.py @hebasto
 /test/functional/feature_asmap.py           @jonatack
 /test/functional/interface_bitcoin_cli.py   @jonatack
-/test/functional/tool_wallet.py             @jonatack
+
+# Backwards compatibility tests
+*_compatibility.py                          @sjors
+/test/functional/wallet_upgradewallet.py    @sjors @achow101
+/test/get_previous_releases.py              @sjors
 
 # Translations
 /src/util/translation.h                     @hebasto
@@ -98,15 +104,19 @@
 # Descriptors
 *descriptor*                                @achow101 @sipa
 
+# External signer
+*external_signer*                           @sjors
+/doc/external-signer.md                     @sjors
+*signer.py                                  @sjors
+
 # Interfaces
 /src/interfaces/                            @ryanofsky
 
 # DB
 /src/txdb.*                                 @jamesob
 /src/dbwrapper.*                            @jamesob
 
-# Scripts/Linter
-*.sh                                        @practicalswift
+# Linter
 /test/lint/                                 @practicalswift
 /test/lint/lint-shell.sh                    @hebasto
 

build_msvc/testconsensus/testconsensus.cpp

@@ -45,7 +45,7 @@ int main()
     stream << vanillaSpendTx;
 
     bitcoinconsensus_error err;
-    auto op0Result = bitcoinconsensus_verify_script_with_amount(pubKeyScript.data(), pubKeyScript.size(), amount, (const unsigned char*)&stream[0], stream.size(), 0, bitcoinconsensus_SCRIPT_FLAGS_VERIFY_ALL, &err);
+    auto op0Result = bitcoinconsensus_verify_script_with_amount(pubKeyScript.data(), pubKeyScript.size(), amount, stream.data(), stream.size(), 0, bitcoinconsensus_SCRIPT_FLAGS_VERIFY_ALL, &err);
     std::cout << "Op0 result: " << op0Result << ", error code " << err << std::endl;
 
     getchar();

ci/test/00_setup_env.sh

@@ -14,6 +14,9 @@ export BASE_ROOT_DIR
 # The depends dir.
 # This folder exists on the ci host and ci guest. Changes are propagated back and forth.
 export DEPENDS_DIR=${DEPENDS_DIR:-$BASE_ROOT_DIR/depends}
+# A folder for the ci system to put temporary files (ccache, datadirs for tests, ...)
+# This folder only exists on the ci host.
+export BASE_SCRATCH_DIR=${BASE_SCRATCH_DIR:-$BASE_ROOT_DIR/ci/scratch}
 
 echo "Setting specific values in env"
 if [ -n "${FILE_ENV}" ]; then
@@ -25,9 +28,6 @@ fi
 echo "Fallback to default values in env (if not yet set)"
 # The number of parallel jobs to pass down to make and test_runner.py
 export MAKEJOBS=${MAKEJOBS:--j4}
-# A folder for the ci system to put temporary files (ccache, datadirs for tests, ...)
-# This folder only exists on the ci host.
-export BASE_SCRATCH_DIR=${BASE_SCRATCH_DIR:-$BASE_ROOT_DIR/ci/scratch}
 # What host to compile for. See also ./depends/README.md
 # Tests that need cross-compilation export the appropriate HOST.
 # Tests that run natively guess the host

ci/test/00_setup_env_mac.sh

@@ -10,8 +10,8 @@ export CONTAINER_NAME=ci_macos_cross
 export DOCKER_NAME_TAG=ubuntu:20.04  # Check that Focal can cross-compile to macos (Focal is used in the gitian build as well)
 export HOST=x86_64-apple-darwin18
 export PACKAGES="cmake imagemagick librsvg2-bin libz-dev libtiff-tools libtinfo5 python3-setuptools xorriso"
-export XCODE_VERSION=11.3.1
-export XCODE_BUILD_ID=11C505
+export XCODE_VERSION=12.1
+export XCODE_BUILD_ID=12A7403
 export RUN_UNIT_TESTS=false
 export RUN_FUNCTIONAL_TESTS=false
 export GOAL="deploy"

ci/test/00_setup_env_mac_host.sh

@@ -7,12 +7,11 @@
 export LC_ALL=C.UTF-8
 
 export HOST=x86_64-apple-darwin18
-export PIP_PACKAGES="zmq"
+export PIP_PACKAGES="zmq lief"
 export GOAL="install"
 export BITCOIN_CONFIG="--with-gui --enable-reduce-exports --enable-external-signer"
 export CI_OS_NAME="macos"
 export NO_DEPENDS=1
 export OSX_SDK=""
 export CCACHE_SIZE=300M
-
 export RUN_SECURITY_TESTS="true"

ci/test/00_setup_env_native_fuzz_with_msan.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2020 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+export DOCKER_NAME_TAG="ubuntu:20.04"
+LIBCXX_DIR="${BASE_SCRATCH_DIR}/msan/build/"
+export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls"
+LIBCXX_FLAGS="-nostdinc++ -stdlib=libc++ -L${LIBCXX_DIR}lib -lc++abi -I${LIBCXX_DIR}include -I${LIBCXX_DIR}include/c++/v1 -lpthread -Wl,-rpath,${LIBCXX_DIR}lib -Wno-unused-command-line-argument"
+export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}"
+
+export CONTAINER_NAME="ci_native_msan"
+export PACKAGES="clang-9 llvm-9 cmake"
+export DEP_OPTS="NO_BDB=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}' boost_cxxflags='-std=c++17 -fvisibility=hidden -fPIC ${MSAN_AND_LIBCXX_FLAGS}' libevent_cflags='${MSAN_FLAGS}' zeromq_cxxflags='-std=c++17 ${MSAN_AND_LIBCXX_FLAGS}'"
+export GOAL="install"
+export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,memory --with-asm=no --prefix=${DEPENDS_DIR}/x86_64-pc-linux-gnu/ CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'"
+export USE_MEMORY_SANITIZER="true"
+export RUN_UNIT_TESTS="false"
+export RUN_FUNCTIONAL_TESTS="false"
+export RUN_FUZZ_TESTS=true
+export CCACHE_SIZE=250M

ci/test/00_setup_env_native_msan.sh

@@ -7,17 +7,17 @@
 export LC_ALL=C.UTF-8
 
 export DOCKER_NAME_TAG="ubuntu:20.04"
-LIBCXX_DIR="${BASE_ROOT_DIR}/ci/scratch/msan/build/"
+LIBCXX_DIR="${BASE_SCRATCH_DIR}/msan/build/"
 export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls"
 LIBCXX_FLAGS="-nostdinc++ -stdlib=libc++ -L${LIBCXX_DIR}lib -lc++abi -I${LIBCXX_DIR}include -I${LIBCXX_DIR}include/c++/v1 -lpthread -Wl,-rpath,${LIBCXX_DIR}lib -Wno-unused-command-line-argument"
 export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}"
 export BDB_PREFIX="${BASE_ROOT_DIR}/db4"
 
 export CONTAINER_NAME="ci_native_msan"
 export PACKAGES="clang-9 llvm-9 cmake"
-export DEP_OPTS="NO_BDB=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}' boost_cxxflags='-std=c++17 -fvisibility=hidden -fPIC ${MSAN_AND_LIBCXX_FLAGS}' zeromq_cxxflags='-std=c++17 ${MSAN_AND_LIBCXX_FLAGS}'"
+export DEP_OPTS="NO_BDB=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}' boost_cxxflags='-std=c++17 -fvisibility=hidden -fPIC ${MSAN_AND_LIBCXX_FLAGS}' libevent_cflags='${MSAN_FLAGS}' sqlite_cflags='${MSAN_FLAGS}' zeromq_cxxflags='-std=c++17 ${MSAN_AND_LIBCXX_FLAGS}'"
 export GOAL="install"
-export BITCOIN_CONFIG="--enable-wallet --with-sanitizers=memory --with-asm=no --prefix=${BASE_ROOT_DIR}/depends/x86_64-pc-linux-gnu/ CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}' BDB_LIBS='-L${BDB_PREFIX}/lib -ldb_cxx-4.8' BDB_CFLAGS='-I${BDB_PREFIX}/include'"
+export BITCOIN_CONFIG="--enable-wallet --with-sanitizers=memory --with-asm=no --prefix=${DEPENDS_DIR}/x86_64-pc-linux-gnu/ CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}' BDB_LIBS='-L${BDB_PREFIX}/lib -ldb_cxx-4.8' BDB_CFLAGS='-I${BDB_PREFIX}/include'"
 export USE_MEMORY_SANITIZER="true"
 export RUN_FUNCTIONAL_TESTS="false"
 export CCACHE_SIZE=250M

ci/test/00_setup_env_native_multiprocess.sh

@@ -8,9 +8,10 @@ export LC_ALL=C.UTF-8
 
 export CONTAINER_NAME=ci_native_multiprocess
 export DOCKER_NAME_TAG=ubuntu:20.04
-export PACKAGES="cmake python3"
-export DEP_OPTS="MULTIPROCESS=1"
+export PACKAGES="cmake python3 python3-pip llvm clang"
+export DEP_OPTS="DEBUG=1 MULTIPROCESS=1"
 export GOAL="install"
-export BITCOIN_CONFIG="--enable-external-signer"
+export BITCOIN_CONFIG="--enable-external-signer --enable-debug CC=clang CXX=clang++"  # Use clang to avoid OOM
 export TEST_RUNNER_ENV="BITCOIND=bitcoin-node"
 export RUN_SECURITY_TESTS="true"
+export PIP_PACKAGES="lief"

ci/test/04_install.sh

@@ -67,6 +67,9 @@ if [[ $DOCKER_NAME_TAG == centos* ]]; then
 elif [ "$CI_USE_APT_INSTALL" != "no" ]; then
   ${CI_RETRY_EXE} DOCKER_EXEC apt-get update
   ${CI_RETRY_EXE} DOCKER_EXEC apt-get install --no-install-recommends --no-upgrade -y $PACKAGES $DOCKER_PACKAGES
+  if [ -n "$PIP_PACKAGES" ]; then
+    ${CI_RETRY_EXE} pip3 install --user $PIP_PACKAGES
+  fi
 fi
 
 if [ "$CI_OS_NAME" == "macos" ]; then
@@ -92,7 +95,7 @@ if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then
   DOCKER_EXEC "update-alternatives --install /usr/bin/clang++ clang++ \$(which clang++-9) 100"
   DOCKER_EXEC "update-alternatives --install /usr/bin/clang clang \$(which clang-9) 100"
   DOCKER_EXEC "mkdir -p ${BASE_SCRATCH_DIR}/msan/build/"
-  DOCKER_EXEC "git clone --depth=1 https://github.com/llvm/llvm-project -b llvmorg-10.0.0 ${BASE_SCRATCH_DIR}/msan/llvm-project"
+  DOCKER_EXEC "git clone --depth=1 https://github.com/llvm/llvm-project -b llvmorg-12.0.0 ${BASE_SCRATCH_DIR}/msan/llvm-project"
   DOCKER_EXEC "cd ${BASE_SCRATCH_DIR}/msan/build/ && cmake -DLLVM_ENABLE_PROJECTS='libcxx;libcxxabi' -DCMAKE_BUILD_TYPE=Release -DLLVM_USE_SANITIZER=Memory -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DLLVM_TARGETS_TO_BUILD=X86 ../llvm-project/llvm/"
   DOCKER_EXEC "cd ${BASE_SCRATCH_DIR}/msan/build/ && make $MAKEJOBS cxx"
 fi

configure.ac

@@ -194,12 +194,6 @@ AC_ARG_ENABLE([fuzz-binary],
     [enable_fuzz_binary=$enableval],
     [enable_fuzz_binary=yes])
 
-AC_ARG_ENABLE([danger_fuzz_link_all],
-    AS_HELP_STRING([--enable-danger-fuzz-link-all],
-    [Danger! Modifies source code. Needs git and gnu sed installed. Link each fuzz target (default no).]),
-    [enable_danger_fuzz_link_all=$enableval],
-    [enable_danger_fuzz_link_all=no])
-
 AC_ARG_WITH([qrencode],
   [AS_HELP_STRING([--with-qrencode],
   [enable QR code support (default is yes if qt is enabled and libqrencode is found)])],
@@ -1326,9 +1320,11 @@ fi
 
 if test x$enable_wallet != xno; then
     dnl Check for libdb_cxx only if wallet enabled
-    BITCOIN_FIND_BDB48
-    if test x$suppress_external_warnings != xno ; then
+    if test "x$use_bdb" != "xno"; then
+      BITCOIN_FIND_BDB48
+      if test x$suppress_external_warnings != xno ; then
         BDB_CPPFLAGS=SUPPRESS_WARNINGS($BDB_CPPFLAGS)
+      fi
     fi
 
     dnl Check for sqlite3
@@ -1788,7 +1784,6 @@ AM_CONDITIONAL([ENABLE_TRACING],[test x$have_sdt = xyes])
 AM_CONDITIONAL([ENABLE_TESTS],[test x$BUILD_TEST = xyes])
 AM_CONDITIONAL([ENABLE_FUZZ],[test x$enable_fuzz = xyes])
 AM_CONDITIONAL([ENABLE_FUZZ_BINARY],[test x$enable_fuzz_binary = xyes])
-AM_CONDITIONAL([ENABLE_FUZZ_LINK_ALL],[test x$enable_danger_fuzz_link_all = xyes])
 AM_CONDITIONAL([ENABLE_QT],[test x$bitcoin_enable_qt = xyes])
 AM_CONDITIONAL([ENABLE_QT_TESTS],[test x$BUILD_TEST_QT = xyes])
 AM_CONDITIONAL([ENABLE_BENCH],[test x$use_bench = xyes])