From 9690265f285021d707f60edfae0965095c8dc31c Mon Sep 17 00:00:00 2001 From: Sjors Provoost Date: Wed, 29 Jan 2025 18:19:22 +0100 Subject: [PATCH] ci: build multiprocess for all depends jobs Except: 1. i686, DEBUG (changed to "no multiprocess") 2. Windows due to lack of support --- .cirrus.yml | 4 ++-- ci/test/00_setup_env_arm.sh | 1 + ..._multiprocess.sh => 00_setup_env_i686_no_multiprocess.sh} | 5 ++--- ci/test/00_setup_env_mac_cross.sh | 1 + ci/test/00_setup_env_native_centos.sh | 3 ++- ci/test/00_setup_env_native_fuzz_with_msan.sh | 2 +- ci/test/00_setup_env_native_msan.sh | 2 +- ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh | 2 +- ci/test/00_setup_env_native_previous_releases.sh | 2 +- ci/test/00_setup_env_native_tsan.sh | 2 +- ci/test/00_setup_env_s390x.sh | 1 + depends/README.md | 2 +- 12 files changed, 15 insertions(+), 12 deletions(-) rename ci/test/{00_setup_env_i686_multiprocess.sh => 00_setup_env_i686_no_multiprocess.sh} (83%) diff --git a/.cirrus.yml b/.cirrus.yml index 9be06943695..0216da43c5e 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -186,13 +186,13 @@ task: FILE_ENV: "./ci/test/00_setup_env_native_fuzz.sh" task: - name: 'multiprocess, i686, DEBUG' + name: 'no multiprocess, i686, DEBUG' << : *GLOBAL_TASK_TEMPLATE persistent_worker: labels: type: medium env: - FILE_ENV: "./ci/test/00_setup_env_i686_multiprocess.sh" + FILE_ENV: "./ci/test/00_setup_env_i686_no_multiprocess.sh" task: name: 'no wallet, libbitcoinkernel' diff --git a/ci/test/00_setup_env_arm.sh b/ci/test/00_setup_env_arm.sh index 67e43aca0d1..5995dab66c1 100755 --- a/ci/test/00_setup_env_arm.sh +++ b/ci/test/00_setup_env_arm.sh @@ -16,6 +16,7 @@ export USE_BUSY_BOX=true export RUN_UNIT_TESTS=true export RUN_FUNCTIONAL_TESTS=false export GOAL="install" +export DEP_OPTS="MULTIPROCESS=1" # -Wno-psabi is to disable ABI warnings: "note: parameter passing for argument of type ... changed in GCC 7.1" # This could be removed once the ABI change warning does not show up by default export BITCOIN_CONFIG="-DREDUCE_EXPORTS=ON -DCMAKE_CXX_FLAGS='-Wno-psabi -Wno-error=maybe-uninitialized'" diff --git a/ci/test/00_setup_env_i686_multiprocess.sh b/ci/test/00_setup_env_i686_no_multiprocess.sh similarity index 83% rename from ci/test/00_setup_env_i686_multiprocess.sh rename to ci/test/00_setup_env_i686_no_multiprocess.sh index e6594864079..e754053d202 100755 --- a/ci/test/00_setup_env_i686_multiprocess.sh +++ b/ci/test/00_setup_env_i686_no_multiprocess.sh @@ -7,11 +7,11 @@ export LC_ALL=C.UTF-8 export HOST=i686-pc-linux-gnu -export CONTAINER_NAME=ci_i686_multiprocess +export CONTAINER_NAME=ci_i686_no_multiprocess export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04" export CI_IMAGE_PLATFORM="linux/amd64" export PACKAGES="llvm clang g++-multilib" -export DEP_OPTS="DEBUG=1 MULTIPROCESS=1" +export DEP_OPTS="DEBUG=1" export GOAL="install" export TEST_RUNNER_EXTRA="--v2transport" export BITCOIN_CONFIG="\ @@ -21,4 +21,3 @@ export BITCOIN_CONFIG="\ -DCMAKE_CXX_FLAGS='-Wno-error=documentation' \ -DAPPEND_CPPFLAGS='-DBOOST_MULTI_INDEX_ENABLE_SAFE_MODE' \ " -export BITCOIND=bitcoin-node # Used in functional tests diff --git a/ci/test/00_setup_env_mac_cross.sh b/ci/test/00_setup_env_mac_cross.sh index 6a1e116a547..c8ee90b8518 100755 --- a/ci/test/00_setup_env_mac_cross.sh +++ b/ci/test/00_setup_env_mac_cross.sh @@ -17,4 +17,5 @@ export XCODE_BUILD_ID=15A240d export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false export GOAL="deploy" +export DEP_OPTS="MULTIPROCESS=1" export BITCOIN_CONFIG="-DBUILD_GUI=ON -DREDUCE_EXPORTS=ON" diff --git a/ci/test/00_setup_env_native_centos.sh b/ci/test/00_setup_env_native_centos.sh index c423d788eb8..1c691827a4d 100755 --- a/ci/test/00_setup_env_native_centos.sh +++ b/ci/test/00_setup_env_native_centos.sh @@ -10,6 +10,7 @@ export CONTAINER_NAME=ci_native_centos export CI_IMAGE_NAME_TAG="quay.io/centos/centos:stream10" export CI_BASE_PACKAGES="gcc-c++ glibc-devel libstdc++-devel ccache make git python3 python3-pip which patch xz procps-ng ksh rsync coreutils bison e2fsprogs cmake" export PIP_PACKAGES="pyzmq" -export DEP_OPTS="DEBUG=1" # Temporarily enable a DEBUG=1 build to check for GCC-bug-117966 regressions. This can be removed once the minimum GCC version is bumped to 12 in the previous releases task, see https://github.com/bitcoin/bitcoin/issues/31436#issuecomment-2530717875 +export DEP_OPTS="DEBUG=1 MULTIPROCESS=1" # Temporarily enable a DEBUG=1 build to check for GCC-bug-117966 regressions. This can be removed once the minimum GCC version is bumped to 12 in the previous releases task, see https://github.com/bitcoin/bitcoin/issues/31436#issuecomment-2530717875 export GOAL="install" export BITCOIN_CONFIG="-DWITH_ZMQ=ON -DBUILD_GUI=ON -DREDUCE_EXPORTS=ON -DCMAKE_BUILD_TYPE=Debug" +export BITCOIND=bitcoin-node # Used in functional tests diff --git a/ci/test/00_setup_env_native_fuzz_with_msan.sh b/ci/test/00_setup_env_native_fuzz_with_msan.sh index cfdbc8c0142..562e3f24bd4 100755 --- a/ci/test/00_setup_env_native_fuzz_with_msan.sh +++ b/ci/test/00_setup_env_native_fuzz_with_msan.sh @@ -15,7 +15,7 @@ export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}" export CONTAINER_NAME="ci_native_fuzz_msan" export PACKAGES="ninja-build" # BDB generates false-positives and will be removed in future -export DEP_OPTS="DEBUG=1 NO_BDB=1 NO_QT=1 CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" +export DEP_OPTS="DEBUG=1 NO_BDB=1 NO_QT=1 MULTIPROCESS=1 CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" export GOAL="install" # Setting CMAKE_{C,CXX}_FLAGS_DEBUG flags to an empty string ensures that the flags set in MSAN_FLAGS remain unaltered. # _FORTIFY_SOURCE is not compatible with MSAN. diff --git a/ci/test/00_setup_env_native_msan.sh b/ci/test/00_setup_env_native_msan.sh index cd4ac999428..58c51125c8f 100755 --- a/ci/test/00_setup_env_native_msan.sh +++ b/ci/test/00_setup_env_native_msan.sh @@ -15,7 +15,7 @@ export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}" export CONTAINER_NAME="ci_native_msan" export PACKAGES="ninja-build" # BDB generates false-positives and will be removed in future -export DEP_OPTS="DEBUG=1 NO_BDB=1 NO_QT=1 CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" +export DEP_OPTS="DEBUG=1 NO_BDB=1 NO_QT=1 MULTIPROCESS=1 CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" export GOAL="install" # Setting CMAKE_{C,CXX}_FLAGS_DEBUG flags to an empty string ensures that the flags set in MSAN_FLAGS remain unaltered. # _FORTIFY_SOURCE is not compatible with MSAN. diff --git a/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh b/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh index 3d5d1b77456..0b9406de39d 100755 --- a/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh +++ b/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh @@ -10,6 +10,6 @@ export CONTAINER_NAME=ci_native_nowallet_libbitcoinkernel export CI_IMAGE_NAME_TAG="docker.io/debian:bookworm" # Use minimum supported python3.10 (or best-effort 3.11) and clang-16, see doc/dependencies.md export PACKAGES="python3-zmq clang-16 llvm-16 libc++abi-16-dev libc++-16-dev" -export DEP_OPTS="NO_WALLET=1 CC=clang-16 CXX='clang++-16 -stdlib=libc++'" +export DEP_OPTS="NO_WALLET=1 MULTIPROCESS=1 CC=clang-16 CXX='clang++-16 -stdlib=libc++'" export GOAL="install" export BITCOIN_CONFIG="-DREDUCE_EXPORTS=ON -DBUILD_UTIL_CHAINSTATE=ON -DBUILD_KERNEL_LIB=ON -DBUILD_SHARED_LIBS=ON" diff --git a/ci/test/00_setup_env_native_previous_releases.sh b/ci/test/00_setup_env_native_previous_releases.sh index 19a33f14dd8..6650ad45db1 100755 --- a/ci/test/00_setup_env_native_previous_releases.sh +++ b/ci/test/00_setup_env_native_previous_releases.sh @@ -10,7 +10,7 @@ export CONTAINER_NAME=ci_native_previous_releases export CI_IMAGE_NAME_TAG="docker.io/ubuntu:22.04" # Use minimum supported python3.10 and gcc-11, see doc/dependencies.md export PACKAGES="gcc-11 g++-11 python3-zmq" -export DEP_OPTS="DEBUG=1 CC=gcc-11 CXX=g++-11" +export DEP_OPTS="DEBUG=1 MULTIPROCESS=1 CC=gcc-11 CXX=g++-11" export TEST_RUNNER_EXTRA="--previous-releases --coverage --extended --exclude feature_dbcrash" # Run extended tests so that coverage does not fail, but exclude the very slow dbcrash export RUN_UNIT_TESTS_SEQUENTIAL="true" export RUN_UNIT_TESTS="false" diff --git a/ci/test/00_setup_env_native_tsan.sh b/ci/test/00_setup_env_native_tsan.sh index d82dda344fe..3e0870a93bf 100755 --- a/ci/test/00_setup_env_native_tsan.sh +++ b/ci/test/00_setup_env_native_tsan.sh @@ -10,7 +10,7 @@ export CONTAINER_NAME=ci_native_tsan export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04" export APT_LLVM_V="19" export PACKAGES="clang-${APT_LLVM_V} llvm-${APT_LLVM_V} libclang-rt-${APT_LLVM_V}-dev libc++abi-${APT_LLVM_V}-dev libc++-${APT_LLVM_V}-dev python3-zmq" -export DEP_OPTS="CC=clang-${APT_LLVM_V} CXX='clang++-${APT_LLVM_V} -stdlib=libc++'" +export DEP_OPTS="CC=clang-${APT_LLVM_V} CXX='clang++-${APT_LLVM_V} -stdlib=libc++' MULTIPROCESS=1" export GOAL="install" export BITCOIN_CONFIG="-DWITH_ZMQ=ON -DSANITIZERS=thread \ -DAPPEND_CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER -DDEBUG_LOCKCONTENTION -D_LIBCPP_REMOVE_TRANSITIVE_INCLUDES'" diff --git a/ci/test/00_setup_env_s390x.sh b/ci/test/00_setup_env_s390x.sh index 0e0ddf33825..6da6f22f861 100755 --- a/ci/test/00_setup_env_s390x.sh +++ b/ci/test/00_setup_env_s390x.sh @@ -14,4 +14,5 @@ export CI_IMAGE_PLATFORM="linux/s390x" export TEST_RUNNER_EXTRA="--exclude rpc_bind,feature_bind_extra" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 export RUN_FUNCTIONAL_TESTS=true export GOAL="install" +export DEP_OPTS="MULTIPROCESS=1" export BITCOIN_CONFIG="-DREDUCE_EXPORTS=ON" diff --git a/depends/README.md b/depends/README.md index 848137f03fa..fd219104e2c 100644 --- a/depends/README.md +++ b/depends/README.md @@ -119,7 +119,7 @@ The following can be set when running make: `make FOO=bar` - `NO_BDB`: Don't download/build/cache BerkeleyDB - `NO_SQLITE`: Don't download/build/cache SQLite - `NO_USDT`: Don't download/build/cache packages needed for enabling USDT tracepoints -- `MULTIPROCESS`: Build libmultiprocess (experimental) +- `MULTIPROCESS`: Build libmultiprocess (experimental, no Windows and OpenBSD) - `DEBUG`: Disable some optimizations and enable more runtime checking - `HOST_ID_SALT`: Optional salt to use when generating host package ids - `BUILD_ID_SALT`: Optional salt to use when generating build package ids