From 395c1137f630dc495ffb2752a23bc1dfd470ee53 Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Wed, 22 Apr 2020 13:00:42 -0400 Subject: [PATCH 1/9] gitian: Limit sourced script to just assignments Previously, the sourced script would create the source tarball. Now, it only assigns variables and the source-ing script has more flexibility in determining what to do with these variables. See later commit showing how this flexibility is useful in our Guix builds. --- contrib/gitian-descriptors/assign_DISTNAME | 12 ++++++++++++ contrib/gitian-descriptors/gitian-linux.yml | 12 +++++++++--- contrib/gitian-descriptors/gitian-osx.yml | 12 +++++++++--- contrib/gitian-descriptors/gitian-win.yml | 12 +++++++++--- contrib/gitian-descriptors/make_git_archive | 20 -------------------- 5 files changed, 39 insertions(+), 29 deletions(-) create mode 100755 contrib/gitian-descriptors/assign_DISTNAME delete mode 100755 contrib/gitian-descriptors/make_git_archive diff --git a/contrib/gitian-descriptors/assign_DISTNAME b/contrib/gitian-descriptors/assign_DISTNAME new file mode 100755 index 0000000000..a1ab3f7cbd --- /dev/null +++ b/contrib/gitian-descriptors/assign_DISTNAME @@ -0,0 +1,12 @@ +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# A helper script to be sourced into the gitian descriptors + +if RECENT_TAG="$(git describe --exact-match HEAD)"; then + VERSION="${RECENT_TAG#v}" +else + VERSION="$(git rev-parse --short HEAD)" +fi +DISTNAME="bitcoin-${VERSION}" diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml index f421372e10..0ed1e16f7e 100644 --- a/contrib/gitian-descriptors/gitian-linux.yml +++ b/contrib/gitian-descriptors/gitian-linux.yml @@ -140,9 +140,15 @@ script: | create_per-host_faketime_wrappers "${REFERENCE_DATETIME}" export PATH=${WRAP_DIR}:${PATH} - # Create the git archive, and define DISTNAME and GIT_ARCHIVE variables. - # shellcheck source=contrib/gitian-descriptors/make_git_archive - source contrib/gitian-descriptors/make_git_archive + # Define DISTNAME variable. + # shellcheck source=contrib/gitian-descriptors/assign_DISTNAME + source contrib/gitian-descriptors/assign_DISTNAME + + GIT_ARCHIVE="${OUTDIR}/src/${DISTNAME}.tar.gz" + + # Create the source tarball + mkdir -p "$(dirname "$GIT_ARCHIVE")" + git archive --output="$GIT_ARCHIVE" HEAD ORIGPATH="$PATH" # Extract the git archive into a dir for each host and build diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml index 82f8f194fc..bbae7201e5 100644 --- a/contrib/gitian-descriptors/gitian-osx.yml +++ b/contrib/gitian-descriptors/gitian-osx.yml @@ -103,9 +103,15 @@ script: | create_per-host_faketime_wrappers "${REFERENCE_DATETIME}" export PATH=${WRAP_DIR}:${PATH} - # Create the git archive, and define DISTNAME and GIT_ARCHIVE variables. - # shellcheck source=contrib/gitian-descriptors/make_git_archive - source contrib/gitian-descriptors/make_git_archive + # Define DISTNAME variable. + # shellcheck source=contrib/gitian-descriptors/assign_DISTNAME + source contrib/gitian-descriptors/assign_DISTNAME + + GIT_ARCHIVE="${OUTDIR}/src/${DISTNAME}.tar.gz" + + # Create the source tarball + mkdir -p "$(dirname "$GIT_ARCHIVE")" + git archive --output="$GIT_ARCHIVE" HEAD ORIGPATH="$PATH" # Extract the git archive into a dir for each host and build diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml index 54ad68a2a3..adad7c32d1 100644 --- a/contrib/gitian-descriptors/gitian-win.yml +++ b/contrib/gitian-descriptors/gitian-win.yml @@ -108,9 +108,15 @@ script: | create_per-host_compiler_wrapper "${REFERENCE_DATETIME}" export PATH=${WRAP_DIR}:${PATH} - # Create the git archive, and define DISTNAME and GIT_ARCHIVE variables. - # shellcheck source=contrib/gitian-descriptors/make_git_archive - source contrib/gitian-descriptors/make_git_archive + # Define DISTNAME variable. + # shellcheck source=contrib/gitian-descriptors/assign_DISTNAME + source contrib/gitian-descriptors/assign_DISTNAME + + GIT_ARCHIVE="${OUTDIR}/src/${DISTNAME}.tar.gz" + + # Create the source tarball + mkdir -p "$(dirname "$GIT_ARCHIVE")" + git archive --output="$GIT_ARCHIVE" HEAD ORIGPATH="$PATH" # Extract the git archive into a dir for each host and build diff --git a/contrib/gitian-descriptors/make_git_archive b/contrib/gitian-descriptors/make_git_archive deleted file mode 100755 index d922c94c60..0000000000 --- a/contrib/gitian-descriptors/make_git_archive +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright (c) 2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# A helper script to be sourced into the gitian descriptors - -mkdir -p ${OUTDIR}/src -RECENT_TAG=$(git describe --abbrev=0 HEAD) -if [ $RECENT_TAG = $(git describe HEAD) ]; then - if [[ $RECENT_TAG == v* ]]; then - VERSION=${RECENT_TAG:1} - else - VERSION=$RECENT_TAG - fi -else - VERSION=$(git rev-parse --short HEAD) -fi -DISTNAME=bitcoin-${VERSION} -GIT_ARCHIVE="${OUTDIR}/src/${DISTNAME}.tar.gz" -git archive --output=$GIT_ARCHIVE HEAD From f5a6ac4f48b18f93050d77bcb23f9cf45ec34647 Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Wed, 22 Apr 2020 13:05:37 -0400 Subject: [PATCH 2/9] guix: Make source tarball using git-archive --- contrib/guix/libexec/build.sh | 40 +++++++++++++++++++++-------------- contrib/guix/manifest.scm | 3 +++ 2 files changed, 27 insertions(+), 16 deletions(-) diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index 550b1b8f40..864db48915 100644 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -141,19 +141,17 @@ make -C depends --jobs="$MAX_JOBS" HOST="$HOST" \ # Source Tarball Building # ########################### -# Create the source tarball and move it to "${OUTDIR}/src" if not already there -if [ -z "$(find "${OUTDIR}/src" -name 'bitcoin-*.tar.gz')" ]; then - ./autogen.sh - env CONFIG_SITE="${BASEPREFIX}/${HOST}/share/config.site" ./configure --prefix=/ - make dist GZIP_ENV='-9n' ${V:+V=1} - mkdir -p "${OUTDIR}/src" - mv "$(find "${PWD}" -name 'bitcoin-*.tar.gz')" "${OUTDIR}/src/" -fi +# Define DISTNAME variable. +# shellcheck source=contrib/gitian-descriptors/assign_DISTNAME +source contrib/gitian-descriptors/assign_DISTNAME -# Determine the full path to our source tarball -SOURCEDIST="$(find "${OUTDIR}/src" -name 'bitcoin-*.tar.gz')" -# Determine our distribution name (e.g. bitcoin-0.18.0) -DISTNAME="$(basename "$SOURCEDIST" '.tar.gz')" +GIT_ARCHIVE="${OUTDIR}/src/${DISTNAME}.tar.gz" + +# Create the source tarball if not already there +if [ ! -e "$GIT_ARCHIVE" ]; then + mkdir -p "$(dirname "$GIT_ARCHIVE")" + git archive --output="$GIT_ARCHIVE" HEAD +fi ########################### # Binary Tarball Building # @@ -187,7 +185,9 @@ export PATH="${BASEPREFIX}/${HOST}/native/bin:${PATH}" cd "$DISTSRC" # Extract the source tarball - tar --strip-components=1 -xf "${SOURCEDIST}" + tar -xf "${GIT_ARCHIVE}" + + ./autogen.sh # Configure this DISTSRC for $HOST # shellcheck disable=SC2086 @@ -234,7 +234,15 @@ export PATH="${BASEPREFIX}/${HOST}/native/bin:${PATH}" case "$HOST" in *mingw*) - cp -f --target-directory="$OUTDIR" ./*-setup-unsigned.exe + # This step not only moves the unsigned NSIS executable to + # "${OUTDIR}", but also renames it + # + # from: + # bitcoin-@PACKAGE_VERSION@-win64-setup-unsigned.exe + # to: + # ${DISTNAME}-win64-setup-unsigned.exe + # + cp -f ./bitcoin-*-win64-setup-unsigned.exe "${OUTDIR}/${DISTNAME}-win64-setup-unsigned.exe" ;; esac ( @@ -264,7 +272,7 @@ export PATH="${BASEPREFIX}/${HOST}/native/bin:${PATH}" cp "${DISTSRC}/doc/README_windows.txt" "${DISTNAME}/readme.txt" ;; *linux*) - cp "${DISTSRC}/doc/README.md" "${DISTNAME}/" + cp "${DISTSRC}/README.md" "${DISTNAME}/" ;; esac @@ -307,7 +315,7 @@ case "$HOST" in ( cd ./windeploy mkdir unsigned - cp --target-directory=unsigned/ "$OUTDIR"/bitcoin-*-setup-unsigned.exe + cp --target-directory=unsigned/ "${OUTDIR}/${DISTNAME}-win64-setup-unsigned.exe" find . -print0 \ | sort --zero-terminated \ | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ diff --git a/contrib/guix/manifest.scm b/contrib/guix/manifest.scm index 86c1a8d27f..5e011ea184 100644 --- a/contrib/guix/manifest.scm +++ b/contrib/guix/manifest.scm @@ -17,6 +17,7 @@ (gnu packages pkg-config) (gnu packages python) (gnu packages shells) + (gnu packages version-control) (guix build-system gnu) (guix build-system trivial) (guix gexp) @@ -181,6 +182,8 @@ chain for " target " development.")) ;; Scripting perl python-3.7 + ;; Git + git ;; Native gcc 9 toolchain targeting glibc 2.27 (make-gcc-toolchain gcc-9 glibc-2.27)) (let ((target (getenv "HOST"))) From 14701604d0904bc5bbf1c67de08f8ee6d3215523 Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Wed, 22 Apr 2020 13:06:18 -0400 Subject: [PATCH 3/9] guix: Expose GIT_COMMON_DIR in container as readonly When using worktrees or submodules, you'll see a `.git' plain text file at the root of your working tree instead of the usual `.git' directory. This plain text file will point to the real GIT_DIR, under the GIT_COMMON_DIR. From experimentation, the full GIT_COMMON_DIR is required to exist for operations such as git-archive(1), so we expose it as readonly inside the container. --- contrib/guix/guix-build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/guix/guix-build.sh b/contrib/guix/guix-build.sh index e20b2a048d..11d2c8b867 100755 --- a/contrib/guix/guix-build.sh +++ b/contrib/guix/guix-build.sh @@ -105,6 +105,7 @@ for host in ${HOSTS=x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv --pure \ --no-cwd \ --share="$PWD"=/bitcoin \ + --expose="$(git rev-parse --git-common-dir)" \ ${SOURCES_PATH:+--share="$SOURCES_PATH"} \ ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \ -- env HOST="$host" \ From fa791da02f9684e3fd554b687fb692ae6a23d65a Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Mon, 27 Apr 2020 16:05:57 -0400 Subject: [PATCH 4/9] nsis: Specify OutFile path only once Previously, we would specify the makensis output file path twice: 1. At the top of Makefile.am as BITCOIN_WIN_INSTALLER, and 2. In share/setup.nsi.ini This commit uses the -X flag of makensis to eliminate the need for the second instance mentioned above, referring makensis directly to the value of BITCOIN_WIN_INSTALLER --- Makefile.am | 2 +- share/setup.nsi.in | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile.am b/Makefile.am index c35f5080aa..1c39f1940c 100644 --- a/Makefile.am +++ b/Makefile.am @@ -80,7 +80,7 @@ $(BITCOIN_WIN_INSTALLER): all-recursive STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIN_CLI_BIN) $(top_builddir)/release STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIN_TX_BIN) $(top_builddir)/release STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIN_WALLET_BIN) $(top_builddir)/release - @test -f $(MAKENSIS) && $(MAKENSIS) -V2 $(top_builddir)/share/setup.nsi || \ + @test -f $(MAKENSIS) && $(MAKENSIS) -V2 $(top_builddir)/share/setup.nsi -X'OutFile "$@"' || \ echo error: could not build $@ @echo built $@ diff --git a/share/setup.nsi.in b/share/setup.nsi.in index 4b2903a7c9..5431909bb2 100644 --- a/share/setup.nsi.in +++ b/share/setup.nsi.in @@ -51,7 +51,6 @@ Var StartMenuGroup !insertmacro MUI_LANGUAGE English # Installer attributes -OutFile @abs_top_srcdir@/@PACKAGE_TARNAME@-@PACKAGE_VERSION@-win64-setup-unsigned.exe InstallDir $PROGRAMFILES64\Bitcoin CRCCheck on XPStyle on From d256f91cb1b0d6ff5170106b99b0266cbe51f5a2 Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Mon, 27 Apr 2020 16:13:57 -0400 Subject: [PATCH 5/9] rel-builds: Directly deploy win installer to OUTDIR --- contrib/gitian-descriptors/gitian-win.yml | 3 +-- contrib/guix/libexec/build.sh | 15 +-------------- 2 files changed, 2 insertions(+), 16 deletions(-) diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml index adad7c32d1..d05b6d426d 100644 --- a/contrib/gitian-descriptors/gitian-win.yml +++ b/contrib/gitian-descriptors/gitian-win.yml @@ -133,9 +133,8 @@ script: | make ${MAKEOPTS} make ${MAKEOPTS} -C src check-security make ${MAKEOPTS} -C src check-symbols - make deploy + make deploy BITCOIN_WIN_INSTALLER="${OUTDIR}/${DISTNAME}-win64-setup-unsigned.exe" make install DESTDIR=${INSTALLPATH} - cp -f ./bitcoin-*-win64-setup-unsigned.exe ${OUTDIR}/${DISTNAME}-win64-setup-unsigned.exe cd installed mv ${DISTNAME}/bin/*.dll ${DISTNAME}/lib/ find . -name "lib*.la" -delete diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index 864db48915..01f4518c73 100644 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -220,7 +220,7 @@ export PATH="${BASEPREFIX}/${HOST}/native/bin:${PATH}" # Make the os-specific installers case "$HOST" in *mingw*) - make deploy ${V:+V=1} + make deploy ${V:+V=1} BITCOIN_WIN_INSTALLER="${OUTDIR}/${DISTNAME}-win64-setup-unsigned.exe" ;; esac @@ -232,19 +232,6 @@ export PATH="${BASEPREFIX}/${HOST}/native/bin:${PATH}" # Install built Bitcoin Core to $INSTALLPATH make install DESTDIR="${INSTALLPATH}" ${V:+V=1} - case "$HOST" in - *mingw*) - # This step not only moves the unsigned NSIS executable to - # "${OUTDIR}", but also renames it - # - # from: - # bitcoin-@PACKAGE_VERSION@-win64-setup-unsigned.exe - # to: - # ${DISTNAME}-win64-setup-unsigned.exe - # - cp -f ./bitcoin-*-win64-setup-unsigned.exe "${OUTDIR}/${DISTNAME}-win64-setup-unsigned.exe" - ;; - esac ( cd installed From a4f6ffa71e335d4b2a6bf525b7f416968f9cd9f7 Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Tue, 28 Apr 2020 12:51:15 -0400 Subject: [PATCH 6/9] lint: Also enable source statements for non-gitian --- test/lint/lint-shell.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/lint/lint-shell.sh b/test/lint/lint-shell.sh index 2bb76ec286..563e076b35 100755 --- a/test/lint/lint-shell.sh +++ b/test/lint/lint-shell.sh @@ -35,8 +35,9 @@ if ! command -v shellcheck > /dev/null; then exit $EXIT_CODE fi +SHELLCHECK_CMD=(shellcheck --external-sources --check-sourced) EXCLUDE="--exclude=$(IFS=','; echo "${disabled[*]}")" -if ! shellcheck "$EXCLUDE" $(git ls-files -- '*.sh' | grep -vE 'src/(leveldb|secp256k1|univalue)/'); then +if ! "${SHELLCHECK_CMD[@]}" "$EXCLUDE" $(git ls-files -- '*.sh' | grep -vE 'src/(leveldb|secp256k1|univalue)/'); then EXIT_CODE=1 fi @@ -46,14 +47,13 @@ if ! command -v yq > /dev/null; then fi EXCLUDE_GITIAN=${EXCLUDE}",$(IFS=','; echo "${disabled_gitian[*]}")" -SHELLCHECK_CMD="shellcheck --external-sources --check-sourced $EXCLUDE_GITIAN" for descriptor in $(git ls-files -- 'contrib/gitian-descriptors/*.yml') do script=$(basename "$descriptor") # Use #!/bin/bash as gitian-builder/bin/gbuild does to complete a script. echo "#!/bin/bash" > $script yq -r .script "$descriptor" >> $script - if ! $SHELLCHECK_CMD $script; then + if ! "${SHELLCHECK_CMD[@]}" "$EXCLUDE_GITIAN" $script; then EXIT_CODE=1 fi rm $script From 1f2c39a30e0f82046c7aecddfda3eb99cb536816 Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Fri, 1 May 2020 12:33:47 -0400 Subject: [PATCH 7/9] guix: Remove logical cores requirement Thanks MarcoFalke for pushing this to its limits and testing :-) --- contrib/guix/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/contrib/guix/README.md b/contrib/guix/README.md index 9f99b36f88..8ce8cb97a0 100644 --- a/contrib/guix/README.md +++ b/contrib/guix/README.md @@ -13,7 +13,6 @@ We achieve bootstrappability by using Guix as a functional package manager. Conservatively, a x86_64 machine with: -- 2 or more logical cores - 4GB of free disk space on the partition that /gnu/store will reside in - 24GB of free disk space on the partition that the Bitcoin Core git repository resides in From 27e63e01cce368d67092de8f0c736927d6f6aa69 Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Fri, 1 May 2020 14:27:57 -0400 Subject: [PATCH 8/9] build: Accomodate makensis v2.x Apparently the -X flag doesn't work as expected in makensis v2.x For example: makensis -V2 share/setup.nsi -X'OutFile "test.exe"' Will output: OutFile expects 1 parameters, got 0. Usage: OutFile install_output.exe So let's instead construct the file using POSIX-compliant commands and shell constructs --- Makefile.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.am b/Makefile.am index 1c39f1940c..43790f1c23 100644 --- a/Makefile.am +++ b/Makefile.am @@ -80,7 +80,7 @@ $(BITCOIN_WIN_INSTALLER): all-recursive STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIN_CLI_BIN) $(top_builddir)/release STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIN_TX_BIN) $(top_builddir)/release STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIN_WALLET_BIN) $(top_builddir)/release - @test -f $(MAKENSIS) && $(MAKENSIS) -V2 $(top_builddir)/share/setup.nsi -X'OutFile "$@"' || \ + @test -f $(MAKENSIS) && echo 'OutFile "$@"' | cat $(top_builddir)/share/setup.nsi - | $(MAKENSIS) -V2 - || \ echo error: could not build $@ @echo built $@ From bfe1ba2f5b36056e0c41edf8206b93d3d83098df Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Mon, 4 May 2020 13:00:08 -0400 Subject: [PATCH 9/9] rel-builds: Specify core.abbrev for git-rev-parse Chose 12 because the kernel uses it: https://public-inbox.org/git/CA+55aFy0_pwtFOYS1Tmnxipw9ZkRNCQHmoYyegO00pjMiZQfbg@mail.gmail.com/raw And also because it's a nice number. --- contrib/gitian-descriptors/assign_DISTNAME | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/gitian-descriptors/assign_DISTNAME b/contrib/gitian-descriptors/assign_DISTNAME index a1ab3f7cbd..a2ca768aaa 100755 --- a/contrib/gitian-descriptors/assign_DISTNAME +++ b/contrib/gitian-descriptors/assign_DISTNAME @@ -7,6 +7,6 @@ if RECENT_TAG="$(git describe --exact-match HEAD)"; then VERSION="${RECENT_TAG#v}" else - VERSION="$(git rev-parse --short HEAD)" + VERSION="$(git rev-parse --short=12 HEAD)" fi DISTNAME="bitcoin-${VERSION}"