foster/bitcoin-core
0
0
Fork 0
mirror of https://github.com/bitcoin/bitcoin.git synced 2025-03-05 14:06:27 -05:00
Code Issues Activity

Merge bitcoin/bitcoin#27033: ci: Cache stuff in volumes, not host folders

Browse source 
fa8e92c022 doc: Update ci docs (721217.xyz)
5fffff54e9 ci: Cache stuff in volumes, not host folders (MarcoFalke)

Pull request description:

  Storing cached stuff in host system folders may lead to unexpected issues when the ci-built stuff is used for a non-ci build or a ci task leaks into another ci task.

ACKs for top commit:
  john-moffett:
    ACK fa8e92c022

Tree-SHA512: 8b0c9019452fbe507a272c1037c3dce3c178c21f85ab1096ed3372ad9d4b3c7aa27d89e5bf80c9a6260ea652e0268be0cbe61d6a4fcb3add569fa38076d32287
This commit is contained in:
merge-script 2023-02-13 11:53:50 +01:00
commit 141115a060
No known key found for this signature in database
GPG key ID: CE2B75697E69A548
3 changed files with 11 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
ci/README.md
View file

@ -8,8 +8,7 @@ Be aware that the tests will be built and run in-place, so please run at your ow
If the repository is not a fresh git clone, you might have to clean files from previous builds or test runs first. If the repository is not a fresh git clone, you might have to clean files from previous builds or test runs first.
The ci needs to perform various sysadmin tasks such as installing packages or writing to the user's home directory. The ci needs to perform various sysadmin tasks such as installing packages or writing to the user's home directory.
While most of the actions are done inside a docker container, this is not possible for all. Thus, cache directories, While it should be fine to run
such as the depends cache, previous release binaries, or ccache, are mounted as read-write into the docker container. While it should be fine to run
the ci system locally on you development box, the ci scripts can generally be assumed to have received less review and the ci system locally on you development box, the ci scripts can generally be assumed to have received less review and
testing compared to other parts of the codebase. If you want to keep the work tree clean, you might want to run the ci testing compared to other parts of the codebase. If you want to keep the work tree clean, you might want to run the ci
system in a virtual machine with a Linux operating system of your choice. system in a virtual machine with a Linux operating system of your choice.

7
ci/test/00_setup_env.sh
View file

@ -8,11 +8,10 @@ export LC_ALL=C.UTF-8
# The root dir. # The root dir.
# The ci system copies this folder. # The ci system copies this folder.
# This is where the depends build is done.
BASE_ROOT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../../ >/dev/null 2>&1 && pwd ) BASE_ROOT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../../ >/dev/null 2>&1 && pwd )
export BASE_ROOT_DIR export BASE_ROOT_DIR
# The depends dir. # The depends dir.
# This folder exists on the ci host and ci guest. Changes are propagated back and forth. # This folder exists only on the ci guest, and on the ci host as a volume.
export DEPENDS_DIR=${DEPENDS_DIR:-$BASE_ROOT_DIR/depends} export DEPENDS_DIR=${DEPENDS_DIR:-$BASE_ROOT_DIR/depends}
# A folder for the ci system to put temporary files (ccache, datadirs for tests, ...) # A folder for the ci system to put temporary files (ccache, datadirs for tests, ...)
# This folder only exists on the ci host. # This folder only exists on the ci host.
@ -58,12 +57,14 @@ export CCACHE_SIZE=${CCACHE_SIZE:-100M}
export CCACHE_TEMPDIR=${CCACHE_TEMPDIR:-/tmp/.ccache-temp} export CCACHE_TEMPDIR=${CCACHE_TEMPDIR:-/tmp/.ccache-temp}
export CCACHE_COMPRESS=${CCACHE_COMPRESS:-1} export CCACHE_COMPRESS=${CCACHE_COMPRESS:-1}
# The cache dir. # The cache dir.
# This folder exists on the ci host and ci guest. Changes are propagated back and forth. # This folder exists only on the ci guest, and on the ci host as a volume.
export CCACHE_DIR=${CCACHE_DIR:-$BASE_SCRATCH_DIR/.ccache} export CCACHE_DIR=${CCACHE_DIR:-$BASE_SCRATCH_DIR/.ccache}
# Folder where the build result is put (bin and lib). # Folder where the build result is put (bin and lib).
export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_SCRATCH_DIR/out/$HOST} export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_SCRATCH_DIR/out/$HOST}
# Folder where the build is done (dist and out-of-tree build). # Folder where the build is done (dist and out-of-tree build).
export BASE_BUILD_DIR=${BASE_BUILD_DIR:-$BASE_SCRATCH_DIR/build} export BASE_BUILD_DIR=${BASE_BUILD_DIR:-$BASE_SCRATCH_DIR/build}
# The folder for previous release binaries.
# This folder exists only on the ci guest, and on the ci host as a volume.
export PREVIOUS_RELEASES_DIR=${PREVIOUS_RELEASES_DIR:-$BASE_ROOT_DIR/releases/$HOST} export PREVIOUS_RELEASES_DIR=${PREVIOUS_RELEASES_DIR:-$BASE_ROOT_DIR/releases/$HOST}
export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks} export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks}
export CI_BASE_PACKAGES=${CI_BASE_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3 rsync git procps bison} export CI_BASE_PACKAGES=${CI_BASE_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3 rsync git procps bison}

9
ci/test/04_install.sh
View file

@ -39,6 +39,9 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then
--build-arg "FILE_ENV=${FILE_ENV}" \ --build-arg "FILE_ENV=${FILE_ENV}" \
--tag="${CONTAINER_NAME}" \ --tag="${CONTAINER_NAME}" \
"${BASE_ROOT_DIR}" "${BASE_ROOT_DIR}"
docker volume create "${CONTAINER_NAME}_ccache" || true
docker volume create "${CONTAINER_NAME}_depends" || true
docker volume create "${CONTAINER_NAME}_previous_releases" || true
if [ -n "${RESTART_CI_DOCKER_BEFORE_RUN}" ] ; then if [ -n "${RESTART_CI_DOCKER_BEFORE_RUN}" ] ; then
echo "Restart docker before run to stop and clear all containers started with --rm" echo "Restart docker before run to stop and clear all containers started with --rm"
@ -48,9 +51,9 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then
# shellcheck disable=SC2086 # shellcheck disable=SC2086
CI_CONTAINER_ID=$(docker run $CI_CONTAINER_CAP --rm --interactive --detach --tty \ CI_CONTAINER_ID=$(docker run $CI_CONTAINER_CAP --rm --interactive --detach --tty \
--mount type=bind,src=$BASE_ROOT_DIR,dst=/ro_base,readonly \ --mount type=bind,src=$BASE_ROOT_DIR,dst=/ro_base,readonly \
--mount type=bind,src=$CCACHE_DIR,dst=$CCACHE_DIR \ --mount "type=volume,src=${CONTAINER_NAME}_ccache,dst=$CCACHE_DIR" \
--mount type=bind,src=$DEPENDS_DIR,dst=$DEPENDS_DIR \ --mount "type=volume,src=${CONTAINER_NAME}_depends,dst=$DEPENDS_DIR" \
--mount type=bind,src=$PREVIOUS_RELEASES_DIR,dst=$PREVIOUS_RELEASES_DIR \ --mount "type=volume,src=${CONTAINER_NAME}_previous_releases,dst=$PREVIOUS_RELEASES_DIR" \
-w $BASE_ROOT_DIR \ -w $BASE_ROOT_DIR \
--env-file /tmp/env \ --env-file /tmp/env \
--name $CONTAINER_NAME \ --name $CONTAINER_NAME \