foster/bitcoin-bitcoin-core
0
0
Fork 0
mirror of https://github.com/bitcoin/bitcoin.git synced 2025-02-08 10:31:50 -05:00
Code Issues Activity

Merge bitcoin/bitcoin#27737: ci: compile Clang and compiler-rt in msan jobs

Browse source 
5763b232e6 ci: return to using Ubuntu 22.04 in MSAN jobs (fanquake)
d3cbcbf626 ci: compile clang and compiler-rt in MSAN jobs (fanquake)
796bd1d0d1 ci: use LLVM 16.0.4 in MSAN jobs (fanquake)
883bc9f561 ci: remove extra CC & CXX from MSAN jobs (fanquake)
2d4f4b8f29 ci: standardize custom libc++ usage in MSAN jobs (fanquake)

Pull request description:

  This reworks the MSAN CIs, to first compile Clang and compiler-rt (using GCC 12), and then, compile an MSAN instrumented libc++ using the just-built Clang 16. This fixes the `native_fuzz_with_msan` job, working around https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=1005341, by not using the Debian provided Clang/LLVM.

  Also included are changes to streamline how we use our "custom libc++", according to upstream: https://releases.llvm.org/16.0.0/projects/libcxx/docs/UsingLibcxx.html#using-a-custom-built-libc, as well as other minor cleanups in the CI configs.

  An example job is currently running in the qa-assets repo: https://github.com/bitcoin-core/qa-assets/pull/129 (https://cirrus-ci.com/task/4632561431871488).

ACKs for top commit:
  dergoegge:
    utACK 5763b232e6

Tree-SHA512: 4f2a6e0b796bb1830b8346dd1e55eaa86a79037b8b4f16a336c1e29f4fc460acca2ecba076635459370bcbb4009333cb79d27ef1521c1fb5db7599cd5bdf558c
This commit is contained in:
fanquake 2023-06-02 10:34:40 +01:00
commit 8a972813ba
No known key found for this signature in database
GPG key ID: 2EEB9F5CC09526C1
4 changed files with 40 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
.cirrus.yml
View file

@ -252,11 +252,11 @@ task:
<< : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
task:
name: '[MSan, depends] [lunar]'
name: '[MSan, depends] [jammy]'
<< : *GLOBAL_TASK_TEMPLATE
container:
docker_arguments:
CI_IMAGE_NAME_TAG: ubuntu:lunar
CI_IMAGE_NAME_TAG: ubuntu:jammy
FILE_ENV: "./ci/test/00_setup_env_native_msan.sh"
env:
<< : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV

12
ci/test/00_setup_env_native_fuzz_with_msan.sh
View file

@ -6,18 +6,18 @@
export LC_ALL=C.UTF-8
export CI_IMAGE_NAME_TAG="ubuntu:23.04" # Version 23.04 will reach EOL in Jan 2024, and can be replaced by "ubuntu:24.04" (or anything else that ships the wanted clang version).
LIBCXX_DIR="${BASE_SCRATCH_DIR}/msan/build/"
export CI_IMAGE_NAME_TAG="ubuntu:22.04"
LIBCXX_DIR="${BASE_SCRATCH_DIR}/msan/cxx_build/"
export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls"
LIBCXX_FLAGS="-nostdinc++ -stdlib=libc++ -L${LIBCXX_DIR}lib -lc++abi -I${LIBCXX_DIR}include -I${LIBCXX_DIR}include/c++/v1 -lpthread -Wl,-rpath,${LIBCXX_DIR}lib -Wno-unused-command-line-argument"
LIBCXX_FLAGS="-nostdinc++ -nostdlib++ -isystem ${LIBCXX_DIR}include/c++/v1 -L${LIBCXX_DIR}lib -Wl,-rpath,${LIBCXX_DIR}lib -lc++ -lc++abi -lpthread -Wno-unused-command-line-argument"
export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}"
export CONTAINER_NAME="ci_native_fuzz_msan"
export PACKAGES="clang-16 llvm-16 libclang-rt-16-dev cmake"
export PACKAGES="cmake ninja-build"
# BDB generates false-positives and will be removed in future
export DEP_OPTS="NO_BDB=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'"
export DEP_OPTS="NO_BDB=1 NO_QT=1 CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'"
export GOAL="install"
export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,memory --disable-hardening --with-asm=no CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CPPFLAGS='-DBOOST_MULTI_INDEX_ENABLE_SAFE_MODE' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'"
export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,memory --disable-hardening --with-asm=no CFLAGS='${MSAN_FLAGS}' CPPFLAGS='-DBOOST_MULTI_INDEX_ENABLE_SAFE_MODE' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'"
export USE_MEMORY_SANITIZER="true"
export RUN_UNIT_TESTS="false"
export RUN_FUNCTIONAL_TESTS="false"

12
ci/test/00_setup_env_native_msan.sh
View file

@ -6,18 +6,18 @@
export LC_ALL=C.UTF-8
export CI_IMAGE_NAME_TAG="ubuntu:23.04" # Version 23.04 will reach EOL in Jan 2024, and can be replaced by "ubuntu:24.04" (or anything else that ships the wanted clang version).
LIBCXX_DIR="${BASE_SCRATCH_DIR}/msan/build/"
export CI_IMAGE_NAME_TAG="ubuntu:22.04"
LIBCXX_DIR="${BASE_SCRATCH_DIR}/msan/cxx_build/"
export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls"
LIBCXX_FLAGS="-nostdinc++ -stdlib=libc++ -L${LIBCXX_DIR}lib -lc++abi -I${LIBCXX_DIR}include -I${LIBCXX_DIR}include/c++/v1 -lpthread -Wl,-rpath,${LIBCXX_DIR}lib -Wno-unused-command-line-argument"
LIBCXX_FLAGS="-nostdinc++ -nostdlib++ -isystem ${LIBCXX_DIR}include/c++/v1 -L${LIBCXX_DIR}lib -Wl,-rpath,${LIBCXX_DIR}lib -lc++ -lc++abi -lpthread -Wno-unused-command-line-argument"
export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}"
export CONTAINER_NAME="ci_native_msan"
export PACKAGES="clang-16 llvm-16 libclang-rt-16-dev cmake"
export PACKAGES="cmake ninja-build"
# BDB generates false-positives and will be removed in future
export DEP_OPTS="NO_BDB=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'"
export DEP_OPTS="NO_BDB=1 NO_QT=1 CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'"
export GOAL="install"
export BITCOIN_CONFIG="--with-sanitizers=memory --disable-hardening --with-asm=no CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'"
export BITCOIN_CONFIG="--with-sanitizers=memory --disable-hardening --with-asm=no CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'"
export USE_MEMORY_SANITIZER="true"
export RUN_FUNCTIONAL_TESTS="false"
export CCACHE_SIZE=250M

31
ci/test/01_base_install.sh
View file

@ -42,11 +42,32 @@ if [ -n "$PIP_PACKAGES" ]; then
fi
if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then
update-alternatives --install /usr/bin/clang++ clang++ "$(which clang++-16)" 100
update-alternatives --install /usr/bin/clang clang "$(which clang-16)" 100
git clone --depth=1 https://github.com/llvm/llvm-project -b llvmorg-16.0.1 "${BASE_SCRATCH_DIR}"/msan/llvm-project
cmake -B "${BASE_SCRATCH_DIR}"/msan/build/ -DLLVM_ENABLE_RUNTIMES='libcxx;libcxxabi' -DCMAKE_BUILD_TYPE=Release -DLLVM_USE_SANITIZER=MemoryWithOrigins -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DLLVM_TARGETS_TO_BUILD=X86 -DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF -DLIBCXX_ENABLE_DEBUG_MODE=ON -DLIBCXX_ENABLE_ASSERTIONS=ON -S "${BASE_SCRATCH_DIR}"/msan/llvm-project/runtimes
make -C "${BASE_SCRATCH_DIR}"/msan/build/ "$MAKEJOBS"
git clone --depth=1 https://github.com/llvm/llvm-project -b llvmorg-16.0.4 "${BASE_SCRATCH_DIR}"/msan/llvm-project
cmake -G Ninja -B "${BASE_SCRATCH_DIR}"/msan/clang_build/ -DLLVM_ENABLE_PROJECTS="clang" \
-DCMAKE_BUILD_TYPE=Release \
-DLLVM_TARGETS_TO_BUILD=X86 \
-DLLVM_ENABLE_RUNTIMES="compiler-rt;libcxx;libcxxabi;libunwind" \
-S "${BASE_SCRATCH_DIR}"/msan/llvm-project/llvm
ninja -C "${BASE_SCRATCH_DIR}"/msan/clang_build/ "$MAKEJOBS"
ninja -C "${BASE_SCRATCH_DIR}"/msan/clang_build/ install-runtimes
update-alternatives --install /usr/bin/clang++ clang++ "${BASE_SCRATCH_DIR}"/msan/clang_build/bin/clang++ 100
update-alternatives --install /usr/bin/clang clang "${BASE_SCRATCH_DIR}"/msan/clang_build/bin/clang 100
cmake -G Ninja -B "${BASE_SCRATCH_DIR}"/msan/cxx_build/ -DLLVM_ENABLE_RUNTIMES='libcxx;libcxxabi' \
-DCMAKE_BUILD_TYPE=Release \
-DLLVM_USE_SANITIZER=MemoryWithOrigins \
-DCMAKE_C_COMPILER=clang \
-DCMAKE_CXX_COMPILER=clang++ \
-DLLVM_TARGETS_TO_BUILD=X86 \
-DLLVM_ENABLE_PER_TARGET_RUNTIME_DIR=OFF \
-DLIBCXX_ENABLE_DEBUG_MODE=ON \
-DLIBCXX_ENABLE_ASSERTIONS=ON \
-S "${BASE_SCRATCH_DIR}"/msan/llvm-project/runtimes
ninja -C "${BASE_SCRATCH_DIR}"/msan/cxx_build/ "$MAKEJOBS"
fi
if [[ "${RUN_TIDY}" == "true" ]]; then