Merge #18741: guix: Make source tarball using git-archive

bfe1ba2f5b rel-builds: Specify core.abbrev for git-rev-parse (Carl Dong)
27e63e01cc build: Accomodate makensis v2.x (Carl Dong)
1f2c39a30e guix: Remove logical cores requirement (Carl Dong)
a4f6ffa71e lint: Also enable source statements for non-gitian (Carl Dong)
d256f91cb1 rel-builds: Directly deploy win installer to OUTDIR (Carl Dong)
fa791da02f nsis: Specify OutFile path only once (Carl Dong)
14701604d0 guix: Expose GIT_COMMON_DIR in container as readonly (Carl Dong)
f5a6ac4f48 guix: Make source tarball using git-archive (Carl Dong)
395c1137f6 gitian: Limit sourced script to just assignments (Carl Dong)

Pull request description:

  Based on: #18556
  Related: https://github.com/bitcoin/bitcoin/pull/17595#discussion_r399728721

ACKs for top commit:
  fanquake:
    ACK bfe1ba2f5b - I agree with Carl, and am going to merge this. I'd like for Linux Guix builds to be working again, and we can rebase #18818.

Tree-SHA512: c87ada7e3de17ca0b692a91029b86573442ded5780fc081c214773f6b374a0cdbeaf6f6898c36669c2e247ee32aa7f82defb1180f8decac52c65f0c140f18674

Makefile.am

@@ -80,7 +80,7 @@ $(BITCOIN_WIN_INSTALLER): all-recursive
 	STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIN_CLI_BIN) $(top_builddir)/release
 	STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIN_TX_BIN) $(top_builddir)/release
 	STRIPPROG="$(STRIP)" $(INSTALL_STRIP_PROGRAM) $(BITCOIN_WALLET_BIN) $(top_builddir)/release
-	@test -f $(MAKENSIS) && $(MAKENSIS) -V2 $(top_builddir)/share/setup.nsi || \
+	@test -f $(MAKENSIS) && echo 'OutFile "$@"' | cat $(top_builddir)/share/setup.nsi - | $(MAKENSIS) -V2 - || \
 	  echo error: could not build $@
 	@echo built $@
 

contrib/gitian-descriptors/assign_DISTNAME

@@ -0,0 +1,12 @@
+# Copyright (c) 2020 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+#
+# A helper script to be sourced into the gitian descriptors
+
+if RECENT_TAG="$(git describe --exact-match HEAD)"; then
+    VERSION="${RECENT_TAG#v}"
+else
+    VERSION="$(git rev-parse --short=12 HEAD)"
+fi
+DISTNAME="bitcoin-${VERSION}"

contrib/gitian-descriptors/gitian-linux.yml

@@ -140,9 +140,15 @@ script: |
   create_per-host_faketime_wrappers "${REFERENCE_DATETIME}"
   export PATH=${WRAP_DIR}:${PATH}
 
-  # Create the git archive, and define DISTNAME and GIT_ARCHIVE variables.
+  # Define DISTNAME variable.
-  # shellcheck source=contrib/gitian-descriptors/make_git_archive
+  # shellcheck source=contrib/gitian-descriptors/assign_DISTNAME
-  source contrib/gitian-descriptors/make_git_archive
+  source contrib/gitian-descriptors/assign_DISTNAME
+
+  GIT_ARCHIVE="${OUTDIR}/src/${DISTNAME}.tar.gz"
+
+  # Create the source tarball
+  mkdir -p "$(dirname "$GIT_ARCHIVE")"
+  git archive --output="$GIT_ARCHIVE" HEAD
 
   ORIGPATH="$PATH"
   # Extract the git archive into a dir for each host and build

contrib/gitian-descriptors/gitian-osx.yml

@@ -103,9 +103,15 @@ script: |
   create_per-host_faketime_wrappers "${REFERENCE_DATETIME}"
   export PATH=${WRAP_DIR}:${PATH}
 
-  # Create the git archive, and define DISTNAME and GIT_ARCHIVE variables.
+  # Define DISTNAME variable.
-  # shellcheck source=contrib/gitian-descriptors/make_git_archive
+  # shellcheck source=contrib/gitian-descriptors/assign_DISTNAME
-  source contrib/gitian-descriptors/make_git_archive
+  source contrib/gitian-descriptors/assign_DISTNAME
+
+  GIT_ARCHIVE="${OUTDIR}/src/${DISTNAME}.tar.gz"
+
+  # Create the source tarball
+  mkdir -p "$(dirname "$GIT_ARCHIVE")"
+  git archive --output="$GIT_ARCHIVE" HEAD
 
   ORIGPATH="$PATH"
   # Extract the git archive into a dir for each host and build

contrib/gitian-descriptors/gitian-win.yml

@@ -108,9 +108,15 @@ script: |
   create_per-host_compiler_wrapper "${REFERENCE_DATETIME}"
   export PATH=${WRAP_DIR}:${PATH}
 
-  # Create the git archive, and define DISTNAME and GIT_ARCHIVE variables.
+  # Define DISTNAME variable.
-  # shellcheck source=contrib/gitian-descriptors/make_git_archive
+  # shellcheck source=contrib/gitian-descriptors/assign_DISTNAME
-  source contrib/gitian-descriptors/make_git_archive
+  source contrib/gitian-descriptors/assign_DISTNAME
+
+  GIT_ARCHIVE="${OUTDIR}/src/${DISTNAME}.tar.gz"
+
+  # Create the source tarball
+  mkdir -p "$(dirname "$GIT_ARCHIVE")"
+  git archive --output="$GIT_ARCHIVE" HEAD
 
   ORIGPATH="$PATH"
   # Extract the git archive into a dir for each host and build
@@ -127,9 +133,8 @@ script: |
     make ${MAKEOPTS}
     make ${MAKEOPTS} -C src check-security
     make ${MAKEOPTS} -C src check-symbols
-    make deploy
+    make deploy BITCOIN_WIN_INSTALLER="${OUTDIR}/${DISTNAME}-win64-setup-unsigned.exe"
     make install DESTDIR=${INSTALLPATH}
-    cp -f ./bitcoin-*-win64-setup-unsigned.exe ${OUTDIR}/${DISTNAME}-win64-setup-unsigned.exe
     cd installed
     mv ${DISTNAME}/bin/*.dll ${DISTNAME}/lib/
     find . -name "lib*.la" -delete

contrib/gitian-descriptors/make_git_archive

@@ -1,20 +0,0 @@
-# Copyright (c) 2020 The Bitcoin Core developers
-# Distributed under the MIT software license, see the accompanying
-# file COPYING or http://www.opensource.org/licenses/mit-license.php.
-#
-# A helper script to be sourced into the gitian descriptors
-
-mkdir -p ${OUTDIR}/src
-RECENT_TAG=$(git describe --abbrev=0 HEAD)
-if [ $RECENT_TAG = $(git describe HEAD) ]; then
-  if [[ $RECENT_TAG == v* ]]; then
-    VERSION=${RECENT_TAG:1}
-  else
-    VERSION=$RECENT_TAG
-  fi
-else
-  VERSION=$(git rev-parse --short HEAD)
-fi
-DISTNAME=bitcoin-${VERSION}
-GIT_ARCHIVE="${OUTDIR}/src/${DISTNAME}.tar.gz"
-git archive --output=$GIT_ARCHIVE HEAD

contrib/guix/README.md

@@ -13,7 +13,6 @@ We achieve bootstrappability by using Guix as a functional package manager.
 
 Conservatively, a x86_64 machine with:
 
-- 2 or more logical cores
 - 4GB of free disk space on the partition that /gnu/store will reside in
 - 24GB of free disk space on the partition that the Bitcoin Core git repository
   resides in

contrib/guix/guix-build.sh

@@ -105,6 +105,7 @@ for host in ${HOSTS=x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu riscv
                                  --pure \
                                  --no-cwd \
                                  --share="$PWD"=/bitcoin \
+                                 --expose="$(git rev-parse --git-common-dir)" \
                                  ${SOURCES_PATH:+--share="$SOURCES_PATH"} \
                                  ${ADDITIONAL_GUIX_ENVIRONMENT_FLAGS} \
                                  -- env HOST="$host" \

contrib/guix/libexec/build.sh

@@ -141,19 +141,17 @@ make -C depends --jobs="$MAX_JOBS" HOST="$HOST" \
 # Source Tarball Building #
 ###########################
 
-# Create the source tarball and move it to "${OUTDIR}/src" if not already there
+# Define DISTNAME variable.
-if [ -z "$(find "${OUTDIR}/src" -name 'bitcoin-*.tar.gz')" ]; then
+# shellcheck source=contrib/gitian-descriptors/assign_DISTNAME
-    ./autogen.sh
+source contrib/gitian-descriptors/assign_DISTNAME
-    env CONFIG_SITE="${BASEPREFIX}/${HOST}/share/config.site" ./configure --prefix=/
-    make dist GZIP_ENV='-9n' ${V:+V=1}
-    mkdir -p "${OUTDIR}/src"
-    mv "$(find "${PWD}" -name 'bitcoin-*.tar.gz')" "${OUTDIR}/src/"
-fi
 
-# Determine the full path to our source tarball
+GIT_ARCHIVE="${OUTDIR}/src/${DISTNAME}.tar.gz"
-SOURCEDIST="$(find "${OUTDIR}/src" -name 'bitcoin-*.tar.gz')"
+
-# Determine our distribution name (e.g. bitcoin-0.18.0)
+# Create the source tarball if not already there
-DISTNAME="$(basename "$SOURCEDIST" '.tar.gz')"
+if [ ! -e "$GIT_ARCHIVE" ]; then
+    mkdir -p "$(dirname "$GIT_ARCHIVE")"
+    git archive --output="$GIT_ARCHIVE" HEAD
+fi
 
 ###########################
 # Binary Tarball Building #
@@ -187,7 +185,9 @@ export PATH="${BASEPREFIX}/${HOST}/native/bin:${PATH}"
     cd "$DISTSRC"
 
     # Extract the source tarball
-    tar --strip-components=1 -xf "${SOURCEDIST}"
+    tar -xf "${GIT_ARCHIVE}"
+
+    ./autogen.sh
 
     # Configure this DISTSRC for $HOST
     # shellcheck disable=SC2086
@@ -220,7 +220,7 @@ export PATH="${BASEPREFIX}/${HOST}/native/bin:${PATH}"
     # Make the os-specific installers
     case "$HOST" in
         *mingw*)
-            make deploy ${V:+V=1}
+            make deploy ${V:+V=1} BITCOIN_WIN_INSTALLER="${OUTDIR}/${DISTNAME}-win64-setup-unsigned.exe"
             ;;
     esac
 
@@ -232,11 +232,6 @@ export PATH="${BASEPREFIX}/${HOST}/native/bin:${PATH}"
     # Install built Bitcoin Core to $INSTALLPATH
     make install DESTDIR="${INSTALLPATH}" ${V:+V=1}
 
-    case "$HOST" in
-        *mingw*)
-            cp -f --target-directory="$OUTDIR" ./*-setup-unsigned.exe
-            ;;
-    esac
     (
         cd installed
 
@@ -264,7 +259,7 @@ export PATH="${BASEPREFIX}/${HOST}/native/bin:${PATH}"
                 cp "${DISTSRC}/doc/README_windows.txt" "${DISTNAME}/readme.txt"
                 ;;
             *linux*)
-                cp "${DISTSRC}/doc/README.md" "${DISTNAME}/"
+                cp "${DISTSRC}/README.md" "${DISTNAME}/"
                 ;;
         esac
 
@@ -307,7 +302,7 @@ case "$HOST" in
         (
             cd ./windeploy
             mkdir unsigned
-            cp --target-directory=unsigned/ "$OUTDIR"/bitcoin-*-setup-unsigned.exe
+            cp --target-directory=unsigned/ "${OUTDIR}/${DISTNAME}-win64-setup-unsigned.exe"
             find . -print0 \
                 | sort --zero-terminated \
                 | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \

contrib/guix/manifest.scm

@@ -17,6 +17,7 @@
              (gnu packages pkg-config)
              (gnu packages python)
              (gnu packages shells)
+             (gnu packages version-control)
              (guix build-system gnu)
              (guix build-system trivial)
              (guix gexp)
@@ -181,6 +182,8 @@ chain for " target " development."))
         ;; Scripting
         perl
         python-3.7
+        ;; Git
+        git
         ;; Native gcc 9 toolchain targeting glibc 2.27
         (make-gcc-toolchain gcc-9 glibc-2.27))
   (let ((target (getenv "HOST")))

share/setup.nsi.in

@@ -51,7 +51,6 @@ Var StartMenuGroup
 !insertmacro MUI_LANGUAGE English
 
 # Installer attributes
-OutFile @abs_top_srcdir@/@PACKAGE_TARNAME@-@PACKAGE_VERSION@-win64-setup-unsigned.exe
 InstallDir $PROGRAMFILES64\Bitcoin
 CRCCheck on
 XPStyle on

test/lint/lint-shell.sh

@@ -35,8 +35,9 @@ if ! command -v shellcheck > /dev/null; then
     exit $EXIT_CODE
 fi
 
+SHELLCHECK_CMD=(shellcheck --external-sources --check-sourced)
 EXCLUDE="--exclude=$(IFS=','; echo "${disabled[*]}")"
-if ! shellcheck "$EXCLUDE" $(git ls-files -- '*.sh' | grep -vE 'src/(leveldb|secp256k1|univalue)/'); then
+if ! "${SHELLCHECK_CMD[@]}" "$EXCLUDE" $(git ls-files -- '*.sh' | grep -vE 'src/(leveldb|secp256k1|univalue)/'); then
     EXIT_CODE=1
 fi
 
@@ -46,14 +47,13 @@ if ! command -v yq > /dev/null; then
 fi
 
 EXCLUDE_GITIAN=${EXCLUDE}",$(IFS=','; echo "${disabled_gitian[*]}")"
-SHELLCHECK_CMD="shellcheck --external-sources --check-sourced $EXCLUDE_GITIAN"
 for descriptor in $(git ls-files -- 'contrib/gitian-descriptors/*.yml')
 do
     script=$(basename "$descriptor")
     # Use #!/bin/bash as gitian-builder/bin/gbuild does to complete a script.
     echo "#!/bin/bash" > $script
     yq -r .script "$descriptor" >> $script
-    if ! $SHELLCHECK_CMD $script; then
+    if ! "${SHELLCHECK_CMD[@]}" "$EXCLUDE_GITIAN" $script; then
         EXIT_CODE=1
     fi
     rm $script