From 30daf76a97c57a5f74c8dad1da282dcc0ff8b3fb Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Wed, 17 Mar 2021 13:49:04 -0400 Subject: [PATCH 01/10] guix: Add guix-attest script --- contrib/guix/guix-attest | 170 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 170 insertions(+) create mode 100755 contrib/guix/guix-attest diff --git a/contrib/guix/guix-attest b/contrib/guix/guix-attest new file mode 100755 index 0000000000..d7604dd75f --- /dev/null +++ b/contrib/guix/guix-attest @@ -0,0 +1,170 @@ +#!/usr/bin/env bash +export LC_ALL=C +set -e -o pipefail + +# Source the common prelude, which: +# 1. Checks if we're at the top directory of the Bitcoin Core repository +# 2. Defines a few common functions and variables +# +# shellcheck source=libexec/prelude.bash +source "$(dirname "${BASH_SOURCE[0]}")/libexec/prelude.bash" + + +################### +## Sanity Checks ## +################### + +################ +# Required non-builtin commands should be invokable +################ + +check_tools cat env basename mkdir xargs find gpg + +################ +# Required env vars should be non-empty +################ + +cmd_usage() { +cat < \\ + SIGNER=GPG_KEY_NAME[=SIGNER_NAME] \\ + ./contrib/guix/guix-attest + +Example w/o overriding signing name: + + env GUIX_SIGS_REPO=/home/achow101/guix.sigs \\ + SIGNER=achow101 \\ + ./contrib/guix/guix-attest + +Example overriding signing name: + + env GUIX_SIGS_REPO=/home/dongcarl/guix.sigs \\ + SIGNER=0x96AB007F1A7ED999=dongcarl \\ + ./contrib/guix/guix-attest + +EOF +} + +if [ -z "$GUIX_SIGS_REPO" ] || [ -z "$SIGNER" ]; then + cmd_usage + exit 1 +fi + +################ +# GUIX_SIGS_REPO should exist as a directory +################ + +if [ ! -d "$GUIX_SIGS_REPO" ]; then +cat << EOF +ERR: The specified GUIX_SIGS_REPO is not an existent directory: + + '$GUIX_SIGS_REPO' + +Hint: Please clone the guix.sigs repository and point to it with the + GUIX_SIGS_REPO environment variable. + +EOF +cmd_usage +exit 1 +fi + +################ +# The key specified in SIGNER should be usable +################ + +IFS='=' read -r gpg_key_name signer_name <<< "$SIGNER" +if [ -z "${signer_name}" ]; then + signer_name="$gpg_key_name" +fi + +if ! gpg --dry-run --list-secret-keys "${gpg_key_name}" >/dev/null 2>&1; then + echo "ERR: GPG can't seem to find any key named '${gpg_key_name}'" + exit 1 +fi + +################ +# We should be able to find at least one output +################ + +echo "Looking for build output directories in ${OUTDIR_BASE}" + +shopt -s nullglob +OUTDIRS=( "${OUTDIR_BASE}"/* ) # This expands to an array of directories... +shopt -u nullglob + +if (( ${#OUTDIRS[@]} )); then + echo "Found build output directories:" + for outdir in "${OUTDIRS[@]}"; do + echo " '$outdir'" + done + echo +else + echo "ERR: Could not find any build output directories in ${OUTDIR_BASE}" + exit 1 +fi + + +############## +## Attest ## +############## + +# Usage: out_name $outdir +# +# HOST: The output directory being attested +# +out_name() { + basename "$1" +} + +# Usage: out_sig_dir $outdir +# +# outdir: The output directory being attested +# +out_sig_dir() { + echo "$GUIX_SIGS_REPO/$VERSION/$(out_name "$1")/$signer_name" +} + +# Accumulate a list of signature directories that already exist... +outdirs_already_attested_to=() + +echo "Attesting to build outputs for version: '${VERSION}'" +echo "" + +# MAIN LOGIC: Loop through each output for VERSION and attest to output in +# GUIX_SIGS_REPO as SIGNER, if attestation does not exist +for outdir in "${OUTDIRS[@]}"; do + outname="$(out_name "$outdir")" + outsigdir="$(out_sig_dir "$outdir")" + if [ -e "$outsigdir" ]; then + echo "${outname}: SKIPPING: Signature directory already exists in the specified guix.sigs repository" + outdirs_already_attested_to+=("$outdir") + else + mkdir -p "$outsigdir" + echo "${outname}: Hashing build outputs to produce SHA256SUMS" + ( + cd "$outdir" + find . -type f -printf '%P\0' | env LC_ALL=C sort -z | xargs -r0 sha256sum >> "$outsigdir"/SHA256SUMS + ) + echo "${outname}: Signing SHA256SUMS to produce SHA256SUMS.asc" + gpg --detach-sign --local-user "$gpg_key_name" --output "$outsigdir"/SHA256SUMS.asc "$outsigdir"/SHA256SUMS + echo "" + fi +done + +if (( ${#outdirs_already_attested_to[@]} )); then +# ...so that we can print them out nicely in a warning message +cat << EOF + +WARN: Signature directories from '$signer_name' already exist in the specified + guix.sigs repository for the following output directories and were + skipped: + +EOF +for outdir in "${outdirs_already_attested_to[@]}"; do + echo " '${outdir}'" + echo " Corresponds to: '$(out_sig_dir "$outdir")'" + echo "" +done +fi From 5926432ba68ba154df6c8eaa74adb18cc0123167 Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Wed, 17 Mar 2021 13:49:17 -0400 Subject: [PATCH 02/10] guix: Add guix-verify script --- contrib/guix/guix-verify | 113 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100755 contrib/guix/guix-verify diff --git a/contrib/guix/guix-verify b/contrib/guix/guix-verify new file mode 100755 index 0000000000..629050956c --- /dev/null +++ b/contrib/guix/guix-verify @@ -0,0 +1,113 @@ +#!/usr/bin/env bash +export LC_ALL=C +set -e -o pipefail + +# Source the common prelude, which: +# 1. Checks if we're at the top directory of the Bitcoin Core repository +# 2. Defines a few common functions and variables +# +# shellcheck source=libexec/prelude.bash +source "$(dirname "${BASH_SOURCE[0]}")/libexec/prelude.bash" + + +################### +## Sanity Checks ## +################### + +################ +# Required non-builtin commands should be invokable +################ + +check_tools cat diff gpg + +################ +# Required env vars should be non-empty +################ + +cmd_usage() { +cat < ./contrib/guix/guix-verify + +EOF +} + +if [ -z "$GUIX_SIGS_REPO" ]; then + cmd_usage + exit 1 +fi + +################ +# GUIX_SIGS_REPO should exist as a directory +################ + +if [ ! -d "$GUIX_SIGS_REPO" ]; then +cat << EOF +ERR: The specified GUIX_SIGS_REPO is not an existent directory: + + '$GUIX_SIGS_REPO' + +Hint: Please clone the guix.sigs repository and point to it with the + GUIX_SIGS_REPO environment variable. + +EOF +cmd_usage +exit 1 +fi + +################ +# We should be able to find at least one output +################ + +OUTSIGDIR_BASE="${GUIX_SIGS_REPO}/${VERSION}" +echo "Looking for output signature directories in '${OUTSIGDIR_BASE}'" + +shopt -s nullglob +OUTSIGDIRS=( "$OUTSIGDIR_BASE"/* ) # This expands to an array of directories... +shopt -u nullglob + +if (( ${#OUTSIGDIRS[@]} )); then + echo "Found output signature directories:" + for outsigdir in "${OUTSIGDIRS[@]}"; do + echo " '$outsigdir'" + done + echo +else + echo "ERR: Could not find any output signature directories in ${OUTSIGDIR_BASE}" + exit 1 +fi + + +############## +## Verify ## +############## + +# MAIN LOGIC: Loop through each output for VERSION and check that the SHA256SUMS +# and SHA256SUMS.asc file match between signers, using the first +# available signer as the arbitrary comparison base. +for outsigdir in "${OUTSIGDIRS[@]}"; do + echo "BEGIN: Checking output signatures for $(basename "$outsigdir")" + echo "" + signer_dirs=( "$outsigdir"/* ) # This expands to an array of directories... + compare_signer_dir="${signer_dirs[0]}" # ...we just want the first one + for current_signer_dir in "${signer_dirs[@]}"; do + if ! gpg --quiet --batch --verify "$current_signer_dir"/SHA256SUMS.asc "$current_signer_dir"/SHA256SUMS; then + echo "ERR: Failed to verify GPG signature in '${current_signer_dir}/SHA256SUMS.asc'" + echo "" + echo "Hint: Either the signature is invalid or the public key is missing" + echo "" + elif ! diff --report-identical "$compare_signer_dir"/SHA256SUMS "$current_signer_dir"/SHA256SUMS; then + echo "ERR: The SHA256SUMS attestation in these two directories differ:" + echo " '${compare_signer_dir}'" + echo " '${current_signer_dir}'" + echo "" + else + echo "Verified: '${current_signer_dir}'" + echo "" + fi + done + echo "DONE: Checking output signatures for $(basename "$outsigdir")" + echo "" + echo "" +done From b5fd89c4c89136007429688601ce4fa497f5f09e Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Mon, 12 Apr 2021 12:21:00 -0400 Subject: [PATCH 03/10] guix-attest: Only use cross-platform flags for find+xargs --- contrib/guix/guix-attest | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/contrib/guix/guix-attest b/contrib/guix/guix-attest index d7604dd75f..fbb4b73c71 100755 --- a/contrib/guix/guix-attest +++ b/contrib/guix/guix-attest @@ -145,7 +145,13 @@ for outdir in "${OUTDIRS[@]}"; do echo "${outname}: Hashing build outputs to produce SHA256SUMS" ( cd "$outdir" - find . -type f -printf '%P\0' | env LC_ALL=C sort -z | xargs -r0 sha256sum >> "$outsigdir"/SHA256SUMS + files="$(find . -type f)" + if [ -n "$files" ]; then + cut -c3- <<< "$files" | env LC_ALL=C sort | xargs sha256sum >> "$outsigdir"/SHA256SUMS + else + echo "ERR: ${outname}: No outputs found in '${outdir}'" + exit 1 + fi ) echo "${outname}: Signing SHA256SUMS to produce SHA256SUMS.asc" gpg --detach-sign --local-user "$gpg_key_name" --output "$outsigdir"/SHA256SUMS.asc "$outsigdir"/SHA256SUMS From 0e1c2e448c25568f276e4f022128870c76ca216b Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Mon, 12 Apr 2021 12:29:01 -0400 Subject: [PATCH 04/10] guix-attest: Use ascii-armor signatures --- contrib/guix/guix-attest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/guix/guix-attest b/contrib/guix/guix-attest index fbb4b73c71..78c6a83fe6 100755 --- a/contrib/guix/guix-attest +++ b/contrib/guix/guix-attest @@ -154,7 +154,7 @@ for outdir in "${OUTDIRS[@]}"; do fi ) echo "${outname}: Signing SHA256SUMS to produce SHA256SUMS.asc" - gpg --detach-sign --local-user "$gpg_key_name" --output "$outsigdir"/SHA256SUMS.asc "$outsigdir"/SHA256SUMS + gpg --detach-sign --local-user "$gpg_key_name" --armor --output "$outsigdir"/SHA256SUMS.asc "$outsigdir"/SHA256SUMS echo "" fi done From c83c4fa5b78aef33bba36b3a0d273422297bd630 Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Mon, 12 Apr 2021 12:29:34 -0400 Subject: [PATCH 05/10] guix-attest: Allow skipping GPG signing with NO_SIGN --- contrib/guix/guix-attest | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/contrib/guix/guix-attest b/contrib/guix/guix-attest index 78c6a83fe6..6aa6ce4716 100755 --- a/contrib/guix/guix-attest +++ b/contrib/guix/guix-attest @@ -18,7 +18,10 @@ source "$(dirname "${BASH_SOURCE[0]}")/libexec/prelude.bash" # Required non-builtin commands should be invokable ################ -check_tools cat env basename mkdir xargs find gpg +check_tools cat env basename mkdir xargs find +if [ -z "$NO_SIGN" ]; then + check_tools gpg +fi ################ # Required env vars should be non-empty @@ -30,6 +33,7 @@ Synopsis: env GUIX_SIGS_REPO= \\ SIGNER=GPG_KEY_NAME[=SIGNER_NAME] \\ + [ NO_SIGN=1 ] ./contrib/guix/guix-attest Example w/o overriding signing name: @@ -44,6 +48,13 @@ Example overriding signing name: SIGNER=0x96AB007F1A7ED999=dongcarl \\ ./contrib/guix/guix-attest +Example w/o signing, just creating SHA256SUMS: + + env GUIX_SIGS_REPO=/home/achow101/guix.sigs \\ + SIGNER=achow101 \\ + NO_SIGN=1 \\ + ./contrib/guix/guix-attest + EOF } @@ -79,7 +90,7 @@ if [ -z "${signer_name}" ]; then signer_name="$gpg_key_name" fi -if ! gpg --dry-run --list-secret-keys "${gpg_key_name}" >/dev/null 2>&1; then +if [ -z "$NO_SIGN" ] && ! gpg --dry-run --list-secret-keys "${gpg_key_name}" >/dev/null 2>&1; then echo "ERR: GPG can't seem to find any key named '${gpg_key_name}'" exit 1 fi @@ -153,8 +164,12 @@ for outdir in "${OUTDIRS[@]}"; do exit 1 fi ) - echo "${outname}: Signing SHA256SUMS to produce SHA256SUMS.asc" - gpg --detach-sign --local-user "$gpg_key_name" --armor --output "$outsigdir"/SHA256SUMS.asc "$outsigdir"/SHA256SUMS + if [ -z "$NO_SIGN" ]; then + echo "${outname}: Signing SHA256SUMS to produce SHA256SUMS.asc" + gpg --detach-sign --local-user "$gpg_key_name" --armor --output "$outsigdir"/SHA256SUMS.asc "$outsigdir"/SHA256SUMS + else + echo "${outname}: Not signing SHA256SUMS as \$NO_SIGN is not empty" + fi echo "" fi done From 022abc85fc7e711a900fed8e5071919a151c0a63 Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Tue, 20 Apr 2021 10:32:54 -0400 Subject: [PATCH 06/10] guix: Minor quoting fix in libexec/build.sh --- contrib/guix/libexec/build.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index 4239c3d475..21062751ab 100644 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -293,7 +293,8 @@ mkdir -p "$DISTSRC" # version symbols for Linux distro back-compatibility. make -C src --jobs=1 check-symbols ${V:+V=1} - mkdir -p ${OUTDIR} + mkdir -p "$OUTDIR" + # Make the os-specific installers case "$HOST" in *mingw*) From f9e2960c018103be756a7f8a506816b49d662514 Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Tue, 20 Apr 2021 15:44:38 -0400 Subject: [PATCH 07/10] guix: Construct $OUTDIR in ${DISTSRC}/output While files are being output to $OUTDIR, it will be under ${DISTSRC}/output, and only when everything is done, will ${DISTSRC}/output be moved to the actual $OUTDIR. This makes it so that a Ctrl-C in the middle of a build is less likely to result in a partially-constructed $OUTDIR. In fact, if I understand correctly, if $OUTDIR and $DISTSRC reside on the same filesystem, the move (rename) is likely atomic. Also, since the "working $OUTDIR" is under ${DISTSRC}/output, it will be cleaned properly by the guix-clean script. --- contrib/guix/libexec/build.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index 21062751ab..1bd4fee884 100644 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -33,6 +33,9 @@ Required environment variables as seen inside the container: OUTDIR: ${OUTDIR:?not set} EOF +ACTUAL_OUTDIR="${OUTDIR}" +OUTDIR="${DISTSRC}/output" + ##################### # Environment Setup # ##################### @@ -429,3 +432,5 @@ mkdir -p "$DISTSRC" ;; esac ) # $DISTSRC + +mv --no-target-directory "$OUTDIR" "$ACTUAL_OUTDIR" From d522d8006b891eccd7901faf391f9c041ddf8e38 Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Tue, 20 Apr 2021 15:53:08 -0400 Subject: [PATCH 08/10] guix: Attest to inputs in inputs.SHA256SUMS At build/codesigning-time, hash build inputs and output the digest to ${OUTDIR}/inputs.SHA256SUMS, which gets included in the final SHA256SUMS constructed by guix-attest. Example final SHA256SUMS: ee832d2a35b7701bff581dea05a536118b118e3ad0a587a2855b6ee8cd6fba20 inputs/bitcoin-78199266af7b.tar.gz ca765e70a0c12866dd63c0be228b675278a26329e5f8f5b5c52fd09200fedf21 bitcoin-78199266af7b-powerpc64le-linux-gnu-debug.tar.gz dae95327d7f2c324e2728c4b73627be6cb2c0d2f2e5bea940d1d5e6463939327 bitcoin-78199266af7b-powerpc64le-linux-gnu.tar.gz --- contrib/guix/guix-attest | 11 +++++++++-- contrib/guix/libexec/build.sh | 15 +++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/contrib/guix/guix-attest b/contrib/guix/guix-attest index 6aa6ce4716..5093dcb69d 100755 --- a/contrib/guix/guix-attest +++ b/contrib/guix/guix-attest @@ -153,10 +153,17 @@ for outdir in "${OUTDIRS[@]}"; do outdirs_already_attested_to+=("$outdir") else mkdir -p "$outsigdir" - echo "${outname}: Hashing build outputs to produce SHA256SUMS" + ( cd "$outdir" - files="$(find . -type f)" + + if [ -e inputs.SHA256SUMS ]; then + echo "${outname}: Including existent input SHA256SUMS" + cat inputs.SHA256SUMS >> "$outsigdir"/SHA256SUMS + fi + + echo "${outname}: Hashing build outputs to produce SHA256SUMS" + files="$(find -L . -type f ! -iname '*.SHA256SUMS')" if [ -n "$files" ]; then cut -c3- <<< "$files" | env LC_ALL=C sort | xargs sha256sum >> "$outsigdir"/SHA256SUMS else diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index 1bd4fee884..ce61cd52c7 100644 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -231,6 +231,21 @@ if [ ! -e "$GIT_ARCHIVE" ]; then git archive --prefix="${DISTNAME}/" --output="$GIT_ARCHIVE" HEAD fi +# tmpdir="$(mktemp -d)" +# ( +# cd "$tmpdir" +# mkdir -p inputs +# ln -sf --target-directory=inputs "$GIT_ARCHIVE" + +# mkdir -p "$OUTDIR" +# find -L inputs -type f -print0 | xargs -0 sha256sum > "${OUTDIR}/inputs.SHA256SUMS" +# ) + +mkdir -p "$OUTDIR" +cat << EOF > "$OUTDIR"/inputs.SHA256SUMS +$(sha256sum "$GIT_ARCHIVE" | cut -d' ' -f1) inputs/$(basename "$GIT_ARCHIVE") +EOF + ########################### # Binary Tarball Building # ########################### From feda2c8e3180cb983c35976d4440cea23a155b7f Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Mon, 3 May 2021 13:12:15 -0400 Subject: [PATCH 09/10] guix: Skip attesting to dist-archive We already attest to the relevant dist-archive in inputs.SHA256SUMS, which is recorded at build-time. We use a SKIPATTEST.TAG file to indicate output directories which do not require attestation (much like the CACHEDIR.TAG specification). Generally, it's better to have build scripts declare properties of directories instead of introducing name-based special cases in attest scripts since build scripts have a more detailed context of what is going on. --- contrib/guix/guix-attest | 4 ++++ contrib/guix/libexec/build.sh | 1 + 2 files changed, 5 insertions(+) diff --git a/contrib/guix/guix-attest b/contrib/guix/guix-attest index 5093dcb69d..8b4746caf9 100755 --- a/contrib/guix/guix-attest +++ b/contrib/guix/guix-attest @@ -146,6 +146,10 @@ echo "" # MAIN LOGIC: Loop through each output for VERSION and attest to output in # GUIX_SIGS_REPO as SIGNER, if attestation does not exist for outdir in "${OUTDIRS[@]}"; do + if [ -e "${outdir}/SKIPATTEST.TAG" ]; then + echo "${outname}: SKIPPING: Output directory marked with SKIPATTEST.TAG file" + continue + fi outname="$(out_name "$outdir")" outsigdir="$(out_sig_dir "$outdir")" if [ -e "$outsigdir" ]; then diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index ce61cd52c7..fccd5d1b08 100644 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -228,6 +228,7 @@ GIT_ARCHIVE="${DIST_ARCHIVE_BASE}/${DISTNAME}.tar.gz" # Create the source tarball if not already there if [ ! -e "$GIT_ARCHIVE" ]; then mkdir -p "$(dirname "$GIT_ARCHIVE")" + touch "${DIST_ARCHIVE_BASE}"/SKIPATTEST.TAG git archive --prefix="${DISTNAME}/" --output="$GIT_ARCHIVE" HEAD fi From d420e5c1c015f58d07aca4d6a805086488f74d03 Mon Sep 17 00:00:00 2001 From: Carl Dong Date: Mon, 3 May 2021 15:33:43 -0400 Subject: [PATCH 10/10] guix-attest: Avoid incomplete sigdirs with ERR traps Sometimes GPG connects to the wrong agent... or you don't have your smartcard handy... --- contrib/guix/guix-attest | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/contrib/guix/guix-attest b/contrib/guix/guix-attest index 8b4746caf9..081d1c0465 100755 --- a/contrib/guix/guix-attest +++ b/contrib/guix/guix-attest @@ -156,6 +156,9 @@ for outdir in "${OUTDIRS[@]}"; do echo "${outname}: SKIPPING: Signature directory already exists in the specified guix.sigs repository" outdirs_already_attested_to+=("$outdir") else + # Clean up incomplete sigdir if something fails (likely gpg) + trap 'rm -rf "$outsigdir"' ERR + mkdir -p "$outsigdir" ( @@ -182,6 +185,8 @@ for outdir in "${OUTDIRS[@]}"; do echo "${outname}: Not signing SHA256SUMS as \$NO_SIGN is not empty" fi echo "" + + trap - ERR # Reset ERR trap fi done