From dff7ed5732c01b3ba47ef9e9ddb687d073d7739d Mon Sep 17 00:00:00 2001
From: James O'Beirne <james.obeirne@pm.me>
Date: Tue, 17 Jan 2023 11:52:05 -0500
Subject: [PATCH 1/2] test: add an easy way to run linters locally

Adds a Dockerfile configuration
that allows straightforward running of linters with compatible versions
locally. This removes a ton of annoyance when trying to appease CI,
because many of the linter versions are quite old and difficult to
maintain locally.

I realize that people may not be thrilled to more ancillary tooling to
the repo, but I think this makes a lot of sense given the linter
versions listed in this container configuration are dictated by this
repo (within the CI configuration), so having these things live in
two separate places is a recipe for version mismatches.

Eventually we can likely just use this container on CI directly to avoid
any chance of inconsistencies between local dev experience and CI.
---
 ci/lint/04_install.sh        | 39 +++++++++++++++++++-----------------
 ci/lint/Dockerfile           | 29 +++++++++++++++++++++++++++
 ci/lint/docker-entrypoint.sh | 12 +++++++++++
 test/lint/README.md          | 18 +++++++++++++++++
 4 files changed, 80 insertions(+), 18 deletions(-)
 create mode 100644 ci/lint/Dockerfile
 create mode 100755 ci/lint/docker-entrypoint.sh

diff --git a/ci/lint/04_install.sh b/ci/lint/04_install.sh
index 376737d01a4..1e046162ee6 100755
--- a/ci/lint/04_install.sh
+++ b/ci/lint/04_install.sh
@@ -9,23 +9,25 @@ export LC_ALL=C
 ${CI_RETRY_EXE} apt-get update
 ${CI_RETRY_EXE} apt-get install -y curl git gawk jq xz-utils
 
-PYTHON_PATH=/tmp/python
-if [ ! -d "${PYTHON_PATH}/bin" ]; then
-  (
-    git clone https://github.com/pyenv/pyenv.git
-    cd pyenv/plugins/python-build || exit 1
-    ./install.sh
-  )
-  # For dependencies see https://github.com/pyenv/pyenv/wiki#suggested-build-environment
-  ${CI_RETRY_EXE} apt-get install -y build-essential libssl-dev zlib1g-dev \
-    libbz2-dev libreadline-dev libsqlite3-dev curl llvm \
-    libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev \
-    clang
-  env CC=clang python-build "$(cat "${BASE_ROOT_DIR}/.python-version")" "${PYTHON_PATH}"
+if [ -z "${SKIP_PYTHON_INSTALL}" ]; then
+    PYTHON_PATH=/tmp/python
+    if [ ! -d "${PYTHON_PATH}/bin" ]; then
+      (
+        git clone https://github.com/pyenv/pyenv.git
+        cd pyenv/plugins/python-build || exit 1
+        ./install.sh
+      )
+      # For dependencies see https://github.com/pyenv/pyenv/wiki#suggested-build-environment
+      ${CI_RETRY_EXE} apt-get install -y build-essential libssl-dev zlib1g-dev \
+        libbz2-dev libreadline-dev libsqlite3-dev curl llvm \
+        libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev \
+        clang
+      env CC=clang python-build "$(cat "${BASE_ROOT_DIR}/.python-version")" "${PYTHON_PATH}"
+    fi
+    export PATH="${PYTHON_PATH}/bin:${PATH}"
+    command -v python3
+    python3 --version
 fi
-export PATH="${PYTHON_PATH}/bin:${PATH}"
-command -v python3
-python3 --version
 
 ${CI_RETRY_EXE} pip3 install codespell==2.2.1
 ${CI_RETRY_EXE} pip3 install flake8==5.0.4
@@ -34,5 +36,6 @@ ${CI_RETRY_EXE} pip3 install pyzmq==24.0.1
 ${CI_RETRY_EXE} pip3 install vulture==2.6
 
 SHELLCHECK_VERSION=v0.8.0
-curl -sL "https://github.com/koalaman/shellcheck/releases/download/${SHELLCHECK_VERSION}/shellcheck-${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" | tar --xz -xf - --directory /tmp/
-export PATH="/tmp/shellcheck-${SHELLCHECK_VERSION}:${PATH}"
+curl -sL "https://github.com/koalaman/shellcheck/releases/download/${SHELLCHECK_VERSION}/shellcheck-${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" | \
+    tar --xz -xf - --directory /tmp/
+mv "/tmp/shellcheck-${SHELLCHECK_VERSION}/shellcheck" /usr/bin/
diff --git a/ci/lint/Dockerfile b/ci/lint/Dockerfile
new file mode 100644
index 00000000000..03c20c72860
--- /dev/null
+++ b/ci/lint/Dockerfile
@@ -0,0 +1,29 @@
+# See test/lint/README.md for usage.
+#
+# This container basically has to live in this directory in order to pull in the CI
+# install scripts. If it lived in the root directory, it would have to pull in the
+# entire repo as docker context during build; if it lived elsewhere, it wouldn't be
+# able to make back-references to pull in the install scripts. So here it lives.
+
+FROM python:3.7-buster
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV LC_ALL=C.UTF-8
+
+# This is used by the 04_install.sh script; we can't read the Python version from
+# .python-version for the same reasons as above, and it's more efficient to pull a
+# preexisting Python image than it is to build from source.
+ENV SKIP_PYTHON_INSTALL=1
+
+# Must be built from ./ci/lint/ for these paths to work.
+COPY ./docker-entrypoint.sh /entrypoint.sh
+COPY ./04_install.sh /install.sh
+
+RUN /install.sh && \
+  echo 'alias lint="./ci/lint/06_script.sh"' >> ~/.bashrc && \
+  chmod 755 /entrypoint.sh && \
+  rm -rf /var/lib/apt/lists/*
+
+
+WORKDIR /bitcoin
+ENTRYPOINT ["/entrypoint.sh"]
diff --git a/ci/lint/docker-entrypoint.sh b/ci/lint/docker-entrypoint.sh
new file mode 100755
index 00000000000..6179dfe68b0
--- /dev/null
+++ b/ci/lint/docker-entrypoint.sh
@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+export LC_ALL=C
+
+# Fixes permission issues when there is a container UID/GID mismatch with the owner
+# of the mounted bitcoin src dir.
+git config --global --add safe.directory /bitcoin
+
+if [ -z "$1" ]; then
+  bash -ic "./ci/lint/06_script.sh"
+else
+  exec "$@"
+fi
diff --git a/test/lint/README.md b/test/lint/README.md
index 8d592c3282b..704922d7abe 100644
--- a/test/lint/README.md
+++ b/test/lint/README.md
@@ -1,5 +1,23 @@
 This folder contains lint scripts.
 
+Running locally
+===============
+
+To run linters locally with the same versions as the CI environment, use the included
+Dockerfile:
+
+```sh
+cd ./ci/lint
+docker build -t bitcoin-linter .
+
+cd /root/of/bitcoin/repo
+docker run --rm -v $(pwd):/bitcoin -it bitcoin-linter
+```
+
+After building the container once, you can simply run the last command any time you
+want to lint.
+
+
 check-doc.py
 ============
 Check for missing documentation of command line options.

From b68e5a7feff3e93027e75da0cd9a590fef99aac1 Mon Sep 17 00:00:00 2001
From: James O'Beirne <james.obeirne@pm.me>
Date: Wed, 18 Jan 2023 09:48:14 -0500
Subject: [PATCH 2/2] lint: specify the right commit range when running locally

When running lints on Cirrus, a special envvar is set ($CIRRUS_PR);
emulate this when running linters locally by setting $LOCAL_BRANCH
to any value.
---
 ci/lint/06_script.sh         | 6 +++++-
 ci/lint/docker-entrypoint.sh | 2 +-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/ci/lint/06_script.sh b/ci/lint/06_script.sh
index c14d7473d37..fa28f6126c5 100755
--- a/ci/lint/06_script.sh
+++ b/ci/lint/06_script.sh
@@ -6,7 +6,11 @@
 
 export LC_ALL=C
 
-if [ -n "$CIRRUS_PR" ]; then
+if [ -n "$LOCAL_BRANCH" ]; then
+  # To faithfully recreate CI linting locally, specify all commits on the current
+  # branch.
+  COMMIT_RANGE="$(git merge-base HEAD master)..HEAD"
+elif [ -n "$CIRRUS_PR" ]; then
   COMMIT_RANGE="HEAD~..HEAD"
   echo
   git log --no-merges --oneline "$COMMIT_RANGE"
diff --git a/ci/lint/docker-entrypoint.sh b/ci/lint/docker-entrypoint.sh
index 6179dfe68b0..3fdbbb0761c 100755
--- a/ci/lint/docker-entrypoint.sh
+++ b/ci/lint/docker-entrypoint.sh
@@ -6,7 +6,7 @@ export LC_ALL=C
 git config --global --add safe.directory /bitcoin
 
 if [ -z "$1" ]; then
-  bash -ic "./ci/lint/06_script.sh"
+  LOCAL_BRANCH=1 bash -ic "./ci/lint/06_script.sh"
 else
   exec "$@"
 fi