Merge abf6ad42bd into 85f96b01b7

9d2d9f7ce2 rpc: Include assumeutxo as a failure reason of rescanblockchain (Fabian Jahr)
595edee169 test, assumeutxo: import descriptors during background sync (Alfonso Roman Zubeldia)
d73ae603d4 rpc: Improve importdescriptor RPC error messages (Fabian Jahr)
27f99b6d63 validation: Don't assume m_chain_tx_count in GuessVerificationProgress (Fabian Jahr)
42d5d53363 interfaces: Add helper function for wallet on pruning (Fabian Jahr)

Pull request description:

  A test that is added as part of #30455 uncovered this issue: The `GuessVerificationProgress` function is used during during descriptor import and relies on `m_chain_tx_count`. In #29370 an [`Assume` was added](0fd915ee6b) expecting the `m_chaint_tx_count` to be set. However, as the test uncovered, `GuessVerificationProgress` is called with background sync blocks that have `m_chaint_tx_count = 0` when they have not been downloaded and processed yet.

  The simple fix is to remove the `Assume`. Users should not be thrown off by the `Internal bug detected` error. The behavior of `importdescriptor` is kept consistent with the behavior for blocks missing due to pruning.

  The test by alfonsoromanz is cherry-picked here to show that the [CI errors](https://cirrus-ci.com/task/5110045812195328?logs=ci#L2535) should be fixed by this change.

  This PR also improves error messages returned by the `importdescriptors` and `rescanblockchain` RPCs. The error message now changes depending on the situation of the node, i.e. if pruning is happening or an assumutxo backgroundsync is active.

ACKs for top commit:
  achow101:
    ACK 9d2d9f7ce2
  mzumsande:
    Code Review ACK 9d2d9f7ce2
  furszy:
    Code review ACK 9d2d9f7ce2

Tree-SHA512: b841a9b371e5eb8eb3bfebca35645ff2fdded7a3e5e06308d46a33a51ca42cc4c258028c9958fbbb6cda9bb990e07ab8d8504dd9ec6705ef78afe0435912b365

contrib/completions/bash/bitcoin-cli.bash

@@ -1,5 +1,10 @@
-# bash programmable completion for bitcoin-cli(1)
-# Copyright (c) 2012-2022 The Bitcoin Core developers
+# Dynamic bash programmable completion for bitcoin-cli(1)
+#     DO NOT EDIT THIS FILE BY HAND -- THIS WILL FAIL THE FUNCTIONAL TEST tool_cli_completion
+# This file is auto-generated by the functional test tool_cli_completion.
+# If you want to modify this file, modify test/functional/tool_cli_completion.py and re-autogenerate
+# this file via the --overwrite test flag.
+
+# Copyright (c) 2012-2024 The Bitcoin Core developers
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
@@ -30,8 +35,8 @@ _bitcoin_cli() {
 
     if ((cword > 5)); then
         case ${words[cword-5]} in
-            sendtoaddress)
-                COMPREPLY=( $( compgen -W "true false" -- "$cur" ) )
+            descriptorprocesspsbt)
+                COMPREPLY=( $( compgen -W "false true" -- "$cur" ) )
                 return 0
                 ;;
         esac
@@ -39,12 +44,12 @@ _bitcoin_cli() {
 
     if ((cword > 4)); then
         case ${words[cword-4]} in
-            importaddress|listtransactions|setban)
-                COMPREPLY=( $( compgen -W "true false" -- "$cur" ) )
+            createpsbt|createrawtransaction|descriptorprocesspsbt|setban)
+                COMPREPLY=( $( compgen -W "false true" -- "$cur" ) )
                 return 0
                 ;;
-            signrawtransactionwithkey|signrawtransactionwithwallet)
-                COMPREPLY=( $( compgen -W "ALL NONE SINGLE ALL|ANYONECANPAY NONE|ANYONECANPAY SINGLE|ANYONECANPAY" -- "$cur" ) )
+            signrawtransactionwithkey)
+                COMPREPLY=( $( compgen -W "ALL ALL|ANYONECANPAY NONE NONE|ANYONECANPAY SINGLE SINGLE|ANYONECANPAY" -- "$cur" ) )
                 return 0
                 ;;
         esac
@@ -52,11 +57,12 @@ _bitcoin_cli() {
 
     if ((cword > 3)); then
         case ${words[cword-3]} in
-            addmultisigaddress)
+            addnode|converttopsbt|getdescriptoractivity|gettxout|gettxoutsetinfo)
+                COMPREPLY=( $( compgen -W "false true" -- "$cur" ) )
                 return 0
                 ;;
-            getbalance|gettxout|importaddress|importpubkey|importprivkey|listreceivedbyaddress|listsinceblock)
-                COMPREPLY=( $( compgen -W "true false" -- "$cur" ) )
+            descriptorprocesspsbt)
+                COMPREPLY=( $( compgen -W "ALL ALL|ANYONECANPAY NONE NONE|ANYONECANPAY SINGLE SINGLE|ANYONECANPAY" -- "$cur" ) )
                 return 0
                 ;;
         esac
@@ -64,31 +70,32 @@ _bitcoin_cli() {
 
     if ((cword > 2)); then
         case ${words[cword-2]} in
+            converttopsbt|decoderawtransaction|finalizepsbt|getblockheader|getmempoolancestors|getmempooldescendants|getrawmempool)
+                COMPREPLY=( $( compgen -W "false true" -- "$cur" ) )
+                return 0
+                ;;
             addnode)
-                COMPREPLY=( $( compgen -W "add remove onetry" -- "$cur" ) )
+                COMPREPLY=( $( compgen -W "add onetry remove" -- "$cur" ) )
                 return 0
                 ;;
             setban)
                 COMPREPLY=( $( compgen -W "add remove" -- "$cur" ) )
                 return 0
                 ;;
-            fundrawtransaction|getblock|getblockheader|getmempoolancestors|getmempooldescendants|getrawtransaction|gettransaction|listreceivedbyaddress|sendrawtransaction)
-                COMPREPLY=( $( compgen -W "true false" -- "$cur" ) )
+            estimatesmartfee)
+                COMPREPLY=( $( compgen -W "CONSERVATIVE ECONOMICAL UNSET" -- "$cur" ) )
                 return 0
                 ;;
         esac
     fi
 
     case "$prev" in
-        backupwallet|dumpwallet|importwallet)
+        dumptxoutset|importmempool|loadtxoutset)
             _filedir
             return 0
             ;;
-        getaddednodeinfo|getrawmempool|lockunspent)
-            COMPREPLY=( $( compgen -W "true false" -- "$cur" ) )
-            return 0
-            ;;
-        getbalance|getnewaddress|listtransactions|sendmany)
+        getrawmempool|setnetworkactive)
+            COMPREPLY=( $( compgen -W "false true" -- "$cur" ) )
             return 0
             ;;
     esac

doc/release-notes-31600.md

@@ -0,0 +1,11 @@
+Updated RPCs
+---
+- the `getblocktemplate` RPC `curtime` (BIP22) and `mintime` (BIP23) fields now
+  account for the timewarp fix proposed in BIP94 on all networks. This ensures
+  that, in the event a timewarp fix softfork activates on mainnet, un-upgraded
+  miners will not accidentally violate the timewarp rule. (#31376, #31600)
+
+As a reminder, it's important that any software which uses the `getblocktemplate`
+RPC takes these values into account (either `curtime` or `mintime` is fine).
+Relying only on a clock can lead to invalid blocks under some circumstances,
+especially once a timewarp fix is deployed.

src/bitcoin-chainstate.cpp

@@ -106,6 +106,7 @@ int main(int argc, char* argv[])
     };
     auto notifications = std::make_unique<KernelNotifications>();
 
+    kernel::CacheSizes cache_sizes{DEFAULT_KERNEL_CACHE};
 
     // SETUP: Chainstate
     auto chainparams = CChainParams::Main();
@@ -119,11 +120,14 @@ int main(int argc, char* argv[])
         .chainparams = chainman_opts.chainparams,
         .blocks_dir = abs_datadir / "blocks",
         .notifications = chainman_opts.notifications,
+        .block_tree_db_params = DBParams{
+            .path = abs_datadir / "blocks" / "index",
+            .cache_bytes = cache_sizes.block_tree_db,
+        },
     };
     util::SignalInterrupt interrupt;
     ChainstateManager chainman{interrupt, chainman_opts, blockman_opts};
 
-    kernel::CacheSizes cache_sizes{DEFAULT_KERNEL_CACHE};
     node::ChainstateLoadOptions options;
     auto [status, error] = node::LoadChainstate(chainman, cache_sizes, options);
     if (status != node::ChainstateLoadStatus::SUCCESS) {

src/init.cpp

@@ -1057,6 +1057,10 @@ bool AppInitParameterInteraction(const ArgsManager& args)
             .chainparams = chainman_opts_dummy.chainparams,
             .blocks_dir = args.GetBlocksDirPath(),
             .notifications = chainman_opts_dummy.notifications,
+            .block_tree_db_params = DBParams{
+                .path = args.GetDataDirNet() / "blocks" / "index",
+                .cache_bytes = 0,
+            },
         };
         auto blockman_result{ApplyArgsManOptions(args, blockman_opts_dummy)};
         if (!blockman_result) {
@@ -1203,18 +1207,33 @@ static ChainstateLoadResult InitAndLoadChainstate(
         .signals = node.validation_signals.get(),
     };
     Assert(ApplyArgsManOptions(args, chainman_opts)); // no error can happen, already checked in AppInitParameterInteraction
+
     BlockManager::Options blockman_opts{
         .chainparams = chainman_opts.chainparams,
         .blocks_dir = args.GetBlocksDirPath(),
         .notifications = chainman_opts.notifications,
+        .block_tree_db_params = DBParams{
+            .path = args.GetDataDirNet() / "blocks" / "index",
+            .cache_bytes = cache_sizes.block_tree_db,
+            .wipe_data = do_reindex,
+        },
     };
     Assert(ApplyArgsManOptions(args, blockman_opts)); // no error can happen, already checked in AppInitParameterInteraction
+
+    // Creating the chainstate manager internally creates a BlockManager, opens
+    // the blocks tree db, and wipes existing block files in case of a reindex.
+    // The coinsdb is opened at a later point on LoadChainstate.
     try {
         node.chainman = std::make_unique<ChainstateManager>(*Assert(node.shutdown_signal), chainman_opts, blockman_opts);
+    } catch (dbwrapper_error& e) {
+        LogError("%s", e.what());
+        return {ChainstateLoadStatus::FAILURE, _("Error opening block database")};
     } catch (std::exception& e) {
         return {ChainstateLoadStatus::FAILURE_FATAL, Untranslated(strprintf("Failed to initialize ChainstateManager: %s", e.what()))};
     }
     ChainstateManager& chainman = *node.chainman;
+    if (chainman.m_interrupt) return {ChainstateLoadStatus::INTERRUPTED, {}};
+
     // This is defined and set here instead of inline in validation.h to avoid a hard
     // dependency between validation and index/base, since the latter is not in
     // libbitcoinkernel.
@@ -1237,7 +1256,6 @@ static ChainstateLoadResult InitAndLoadChainstate(
     };
     node::ChainstateLoadOptions options;
     options.mempool = Assert(node.mempool.get());
-    options.wipe_block_tree_db = do_reindex;
     options.wipe_chainstate_db = do_reindex || do_reindex_chainstate;
     options.prune = chainman.m_blockman.IsPruneMode();
     options.check_blocks = args.GetIntArg("-checkblocks", DEFAULT_CHECKBLOCKS);

src/interfaces/chain.h

@@ -289,6 +289,9 @@ public:
     //! Check if any block has been pruned.
     virtual bool havePruned() = 0;
 
+    //! Get the current prune height.
+    virtual std::optional<int> getPruneHeight() = 0;
+
     //! Check if the node is ready to broadcast transactions.
     virtual bool isReadyToBroadcast() = 0;
 

src/kernel/blockmanager_opts.h

@@ -5,6 +5,7 @@
 #ifndef BITCOIN_KERNEL_BLOCKMANAGER_OPTS_H
 #define BITCOIN_KERNEL_BLOCKMANAGER_OPTS_H
 
+#include <dbwrapper.h>
 #include <kernel/notifications_interface.h>
 #include <util/fs.h>
 
@@ -27,6 +28,7 @@ struct BlockManagerOpts {
     bool fast_prune{false};
     const fs::path blocks_dir;
     Notifications& notifications;
+    DBParams block_tree_db_params;
 };
 
 } // namespace kernel

src/kernel/chainstatemanager_opts.h

@@ -42,7 +42,6 @@ struct ChainstateManagerOpts {
     std::optional<uint256> assumed_valid_block{};
     //! If the tip is older than this, the node is considered to be in initial block download.
     std::chrono::seconds max_tip_age{DEFAULT_MAX_TIP_AGE};
-    DBOptions block_tree_db{};
     DBOptions coins_db{};
     CoinsViewOptions coins_view{};
     Notifications& notifications;

src/node/blockmanager_args.cpp

@@ -6,6 +6,7 @@
 
 #include <common/args.h>
 #include <node/blockstorage.h>
+#include <node/database_args.h>
 #include <tinyformat.h>
 #include <util/result.h>
 #include <util/translation.h>
@@ -34,6 +35,8 @@ util::Result<void> ApplyArgsManOptions(const ArgsManager& args, BlockManager::Op
 
     if (auto value{args.GetBoolArg("-fastprune")}) opts.fast_prune = *value;
 
+    ReadDatabaseArgs(args, opts.block_tree_db_params.options);
+
     return {};
 }
 } // namespace node

src/node/blockstorage.cpp

@@ -36,6 +36,7 @@
 #include <util/translation.h>
 #include <validation.h>
 
+#include <cstddef>
 #include <map>
 #include <ranges>
 #include <unordered_map>
@@ -1169,7 +1170,19 @@ BlockManager::BlockManager(const util::SignalInterrupt& interrupt, Options opts)
       m_opts{std::move(opts)},
       m_block_file_seq{FlatFileSeq{m_opts.blocks_dir, "blk", m_opts.fast_prune ? 0x4000 /* 16kB */ : BLOCKFILE_CHUNK_SIZE}},
       m_undo_file_seq{FlatFileSeq{m_opts.blocks_dir, "rev", UNDOFILE_CHUNK_SIZE}},
-      m_interrupt{interrupt} {}
+      m_interrupt{interrupt}
+{
+    m_block_tree_db = std::make_unique<BlockTreeDB>(m_opts.block_tree_db_params);
+
+    if (m_opts.block_tree_db_params.wipe_data) {
+        m_block_tree_db->WriteReindexing(true);
+        m_blockfiles_indexed = false;
+        // If we're reindexing in prune mode, wipe away unusable block files and all undo data files
+        if (m_prune_mode) {
+            CleanupBlockRevFiles();
+        }
+    }
+}
 
 class ImportingNow
 {

src/node/chainstate.cpp

@@ -23,10 +23,7 @@
 #include <validation.h>
 
 #include <algorithm>
-#include <atomic>
 #include <cassert>
-#include <limits>
-#include <memory>
 #include <vector>
 
 using kernel::CacheSizes;
@@ -36,34 +33,8 @@ namespace node {
 // to ChainstateManager::InitializeChainstate().
 static ChainstateLoadResult CompleteChainstateInitialization(
     ChainstateManager& chainman,
-    const CacheSizes& cache_sizes,
     const ChainstateLoadOptions& options) EXCLUSIVE_LOCKS_REQUIRED(::cs_main)
 {
-    auto& pblocktree{chainman.m_blockman.m_block_tree_db};
-    // new BlockTreeDB tries to delete the existing file, which
-    // fails if it's still open from the previous loop. Close it first:
-    pblocktree.reset();
-    try {
-        pblocktree = std::make_unique<BlockTreeDB>(DBParams{
-            .path = chainman.m_options.datadir / "blocks" / "index",
-            .cache_bytes = cache_sizes.block_tree_db,
-            .memory_only = options.block_tree_db_in_memory,
-            .wipe_data = options.wipe_block_tree_db,
-            .options = chainman.m_options.block_tree_db});
-    } catch (dbwrapper_error& err) {
-        LogError("%s\n", err.what());
-        return {ChainstateLoadStatus::FAILURE, _("Error opening block database")};
-    }
-
-    if (options.wipe_block_tree_db) {
-        pblocktree->WriteReindexing(true);
-        chainman.m_blockman.m_blockfiles_indexed = false;
-        //If we're reindexing in prune mode, wipe away unusable block files and all undo data files
-        if (options.prune) {
-            chainman.m_blockman.CleanupBlockRevFiles();
-        }
-    }
-
     if (chainman.m_interrupt) return {ChainstateLoadStatus::INTERRUPTED, {}};
 
     // LoadBlockIndex will load m_have_pruned if we've ever removed a
@@ -155,14 +126,12 @@ static ChainstateLoadResult CompleteChainstateInitialization(
         }
     }
 
-    if (!options.wipe_block_tree_db) {
-        auto chainstates{chainman.GetAll()};
-        if (std::any_of(chainstates.begin(), chainstates.end(),
-                        [](const Chainstate* cs) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { return cs->NeedsRedownload(); })) {
-            return {ChainstateLoadStatus::FAILURE, strprintf(_("Witness data for blocks after height %d requires validation. Please restart with -reindex."),
-                                                             chainman.GetConsensus().SegwitHeight)};
-        };
-    }
+    auto chainstates{chainman.GetAll()};
+    if (std::any_of(chainstates.begin(), chainstates.end(),
+                    [](const Chainstate* cs) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { return cs->NeedsRedownload(); })) {
+        return {ChainstateLoadStatus::FAILURE, strprintf(_("Witness data for blocks after height %d requires validation. Please restart with -reindex."),
+                                                         chainman.GetConsensus().SegwitHeight)};
+    };
 
     // Now that chainstates are loaded and we're able to flush to
     // disk, rebalance the coins caches to desired levels based
@@ -208,7 +177,7 @@ ChainstateLoadResult LoadChainstate(ChainstateManager& chainman, const CacheSize
         }
     }
 
-    auto [init_status, init_error] = CompleteChainstateInitialization(chainman, cache_sizes, options);
+    auto [init_status, init_error] = CompleteChainstateInitialization(chainman, options);
     if (init_status != ChainstateLoadStatus::SUCCESS) {
         return {init_status, init_error};
     }
@@ -244,7 +213,7 @@ ChainstateLoadResult LoadChainstate(ChainstateManager& chainman, const CacheSize
         // for the fully validated chainstate.
         chainman.ActiveChainstate().ClearBlockIndexCandidates();
 
-        auto [init_status, init_error] = CompleteChainstateInitialization(chainman, cache_sizes, options);
+        auto [init_status, init_error] = CompleteChainstateInitialization(chainman, options);
         if (init_status != ChainstateLoadStatus::SUCCESS) {
             return {init_status, init_error};
         }

src/node/chainstate.h

@@ -22,12 +22,7 @@ namespace node {
 
 struct ChainstateLoadOptions {
     CTxMemPool* mempool{nullptr};
-    bool block_tree_db_in_memory{false};
     bool coins_db_in_memory{false};
-    // Whether to wipe the block tree database when loading it. If set, this
-    // will also set a reindexing flag so any existing block data files will be
-    // scanned and added to the database.
-    bool wipe_block_tree_db{false};
     // Whether to wipe the chainstate database when loading it. If set, this
     // will cause the chainstate database to be rebuilt starting from genesis.
     bool wipe_chainstate_db{false};

src/node/chainstatemanager_args.cpp

@@ -49,7 +49,6 @@ util::Result<void> ApplyArgsManOptions(const ArgsManager& args, ChainstateManage
 
     if (auto value{args.GetIntArg("-maxtipage")}) opts.max_tip_age = std::chrono::seconds{*value};
 
-    ReadDatabaseArgs(args, opts.block_tree_db);
     ReadDatabaseArgs(args, opts.coins_db);
     ReadCoinsViewArgs(args, opts.coins_view);
 

src/node/interfaces.cpp

@@ -46,6 +46,7 @@
 #include <policy/settings.h>
 #include <primitives/block.h>
 #include <primitives/transaction.h>
+#include <rpc/blockchain.h>
 #include <rpc/protocol.h>
 #include <rpc/server.h>
 #include <support/allocators/secure.h>
@@ -770,6 +771,11 @@ public:
         LOCK(::cs_main);
         return chainman().m_blockman.m_have_pruned;
     }
+    std::optional<int> getPruneHeight() override
+    {
+        LOCK(chainman().GetMutex());
+        return GetPruneHeight(chainman().m_blockman, chainman().ActiveChain());
+    }
     bool isReadyToBroadcast() override { return !chainman().m_blockman.LoadingBlocks() && !isInitialBlockDownload(); }
     bool isInitialBlockDownload() override
     {

src/node/miner.cpp

@@ -28,16 +28,25 @@
 #include <utility>
 
 namespace node {
+
+int64_t GetMinimumTime(const CBlockIndex* pindexPrev, const int64_t difficulty_adjustment_interval)
+{
+    int64_t min_time{pindexPrev->GetMedianTimePast() + 1};
+    // Height of block to be mined.
+    const int height{pindexPrev->nHeight + 1};
+    // Account for BIP94 timewarp rule on all networks. This makes future
+    // activation safer.
+    if (height % difficulty_adjustment_interval == 0) {
+        min_time = std::max<int64_t>(min_time, pindexPrev->GetBlockTime() - MAX_TIMEWARP);
+    }
+    return min_time;
+}
+
 int64_t UpdateTime(CBlockHeader* pblock, const Consensus::Params& consensusParams, const CBlockIndex* pindexPrev)
 {
     int64_t nOldTime = pblock->nTime;
-    int64_t nNewTime{std::max<int64_t>(pindexPrev->GetMedianTimePast() + 1, TicksSinceEpoch<std::chrono::seconds>(NodeClock::now()))};
-
-    // Height of block to be mined.
-    const int height{pindexPrev->nHeight + 1};
-    if (height % consensusParams.DifficultyAdjustmentInterval() == 0) {
-        nNewTime = std::max<int64_t>(nNewTime, pindexPrev->GetBlockTime() - MAX_TIMEWARP);
-    }
+    int64_t nNewTime{std::max<int64_t>(GetMinimumTime(pindexPrev, consensusParams.DifficultyAdjustmentInterval()),
+                                       TicksSinceEpoch<std::chrono::seconds>(NodeClock::now()))};
 
     if (nOldTime < nNewTime) {
         pblock->nTime = nNewTime;

src/node/miner.h

@@ -211,6 +211,13 @@ private:
     void SortForBlock(const CTxMemPool::setEntries& package, std::vector<CTxMemPool::txiter>& sortedEntries);
 };
 
+/**
+ * Get the minimum time a miner should use in the next block. This always
+ * accounts for the BIP94 timewarp rule, so does not necessarily reflect the
+ * consensus limit.
+ */
+int64_t GetMinimumTime(const CBlockIndex* pindexPrev, const int64_t difficulty_adjustment_interval);
+
 int64_t UpdateTime(CBlockHeader* pblock, const Consensus::Params& consensusParams, const CBlockIndex* pindexPrev);
 
 /** Update an old GenerateCoinbaseCommitment from CreateNewBlock after the block txs have changed */

src/rpc/mining.cpp

@@ -49,6 +49,7 @@
 using interfaces::BlockTemplate;
 using interfaces::Mining;
 using node::BlockAssembler;
+using node::GetMinimumTime;
 using node::NodeContext;
 using node::RegenerateCommitments;
 using node::UpdateTime;
@@ -674,7 +675,7 @@ static RPCHelpMan getblocktemplate()
                 {RPCResult::Type::NUM, "coinbasevalue", "maximum allowable input to coinbase transaction, including the generation award and transaction fees (in satoshis)"},
                 {RPCResult::Type::STR, "longpollid", "an id to include with a request to longpoll on an update to this template"},
                 {RPCResult::Type::STR, "target", "The hash target"},
-                {RPCResult::Type::NUM_TIME, "mintime", "The minimum timestamp appropriate for the next block time, expressed in " + UNIX_EPOCH_TIME},
+                {RPCResult::Type::NUM_TIME, "mintime", "The minimum timestamp appropriate for the next block time, expressed in " + UNIX_EPOCH_TIME + ". Adjusted for the proposed BIP94 timewarp rule."},
                 {RPCResult::Type::ARR, "mutable", "list of ways the block template may be changed",
                 {
                     {RPCResult::Type::STR, "value", "A way the block template may be changed, e.g. 'time', 'transactions', 'prevblock'"},
@@ -683,7 +684,7 @@ static RPCHelpMan getblocktemplate()
                 {RPCResult::Type::NUM, "sigoplimit", "limit of sigops in blocks"},
                 {RPCResult::Type::NUM, "sizelimit", "limit of block size"},
                 {RPCResult::Type::NUM, "weightlimit", /*optional=*/true, "limit of block weight"},
-                {RPCResult::Type::NUM_TIME, "curtime", "current timestamp in " + UNIX_EPOCH_TIME},
+                {RPCResult::Type::NUM_TIME, "curtime", "current timestamp in " + UNIX_EPOCH_TIME + ". Adjusted for the proposed BIP94 timewarp rule."},
                 {RPCResult::Type::STR, "bits", "compressed target of next block"},
                 {RPCResult::Type::NUM, "height", "The height of the next block"},
                 {RPCResult::Type::STR_HEX, "signet_challenge", /*optional=*/true, "Only on signet"},
@@ -977,7 +978,7 @@ static RPCHelpMan getblocktemplate()
     result.pushKV("coinbasevalue", (int64_t)block.vtx[0]->vout[0].nValue);
     result.pushKV("longpollid", tip.GetHex() + ToString(nTransactionsUpdatedLast));
     result.pushKV("target", hashTarget.GetHex());
-    result.pushKV("mintime", (int64_t)pindexPrev->GetMedianTimePast()+1);
+    result.pushKV("mintime", GetMinimumTime(pindexPrev, consensusParams.DifficultyAdjustmentInterval()));
     result.pushKV("mutable", std::move(aMutable));
     result.pushKV("noncerange", "00000000ffffffff");
     int64_t nSigOpLimit = MAX_BLOCK_SIGOPS_COST;

src/rpc/node.cpp

@@ -269,6 +269,32 @@ static RPCHelpMan logging()
     };
 }
 
+static RPCHelpMan format()
+{
+    return RPCHelpMan{"format",
+        "\nFormat data we have about an RPC command in the format specified\n",
+        {
+            {"command", RPCArg::Type::STR, RPCArg::Optional::NO, "Command to query"},
+            {"output", RPCArg::Type::STR, RPCArg::Optional::NO, "Output format. Accepted values: args_cli"},
+        },
+        RPCResult{RPCResult::Type::STR, "data", "Formated data about command"},
+        RPCExamples{""},
+        [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue
+{
+    const std::string command = request.params[0].get_str();
+    JSONRPCRequest jreq(request);
+    jreq.mode = JSONRPCRequest::GET_HELP;
+
+    try {
+        tableRPC.execute(command, jreq);
+    } catch(const UniValue& e) {
+        return e["message"];
+    }
+    return NullUniValue;
+},
+    };
+}
+
 static RPCHelpMan echo(const std::string& name)
 {
     return RPCHelpMan{name,
@@ -408,6 +434,7 @@ void RegisterNodeRPCCommands(CRPCTable& t)
         {"util", &getindexinfo},
         {"hidden", &setmocktime},
         {"hidden", &mockscheduler},
+        {"hidden", &format},
         {"hidden", &echo},
         {"hidden", &echojson},
         {"hidden", &echoipc},

src/rpc/server.cpp

@@ -3,6 +3,7 @@
 // Distributed under the MIT software license, see the accompanying
 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
+#include <rpc/request.h>
 #include <bitcoin-build-config.h> // IWYU pragma: keep
 
 #include <rpc/server.h>
@@ -481,7 +482,7 @@ static bool ExecuteCommands(const std::vector<const CRPCCommand*>& commands, con
     return false;
 }
 
-UniValue CRPCTable::execute(const JSONRPCRequest &request) const
+UniValue CRPCTable::execute(const std::string method, const JSONRPCRequest &request) const
 {
     // Return immediately if in warmup
     {
@@ -491,7 +492,7 @@ UniValue CRPCTable::execute(const JSONRPCRequest &request) const
     }
 
     // Find method
-    auto it = mapCommands.find(request.strMethod);
+    auto it = mapCommands.find(method);
     if (it != mapCommands.end()) {
         UniValue result;
         if (ExecuteCommands(it->second, request, result)) {
@@ -501,6 +502,11 @@ UniValue CRPCTable::execute(const JSONRPCRequest &request) const
     throw JSONRPCError(RPC_METHOD_NOT_FOUND, "Method not found");
 }
 
+UniValue CRPCTable::execute(const JSONRPCRequest &request) const
+{
+    return this->execute(request.strMethod, request);
+}
+
 static bool ExecuteCommand(const CRPCCommand& command, const JSONRPCRequest& request, UniValue& result, bool last_handler)
 {
     try {

src/rpc/server.h

@@ -132,13 +132,22 @@ public:
     std::string help(const std::string& name, const JSONRPCRequest& helpreq) const;
 
     /**
-     * Execute a method.
+     * Execute the request.strMethod.
      * @param request The JSONRPCRequest to execute
      * @returns Result of the call.
      * @throws an exception (UniValue) when an error happens.
      */
     UniValue execute(const JSONRPCRequest &request) const;
 
+    /**
+     * Execute a method.
+     * @param method The method to execute
+     * @param request The JSONRPCRequest to execute
+     * @returns Result of the call.
+     * @throws an exception (UniValue) when an error happens.
+     */
+    UniValue execute(const std::string method, const JSONRPCRequest &request) const;
+
     /**
     * Returns a list of registered commands
     * @returns List of registered commands.

src/rpc/util.cpp

@@ -666,7 +666,11 @@ UniValue RPCHelpMan::HandleRequest(const JSONRPCRequest& request) const
      * the user is asking for help information, and throw help when appropriate.
      */
     if (request.mode == JSONRPCRequest::GET_HELP || !IsValidNumArgs(request.params.size())) {
-        throw std::runtime_error(ToString());
+        std::string help_format = "default";
+        if (request.strMethod == "format") {
+            help_format = request.params[1].get_str();
+        }
+        throw std::runtime_error(ToString(help_format));
     }
     UniValue arg_mismatch{UniValue::VOBJ};
     for (size_t i{0}; i < m_args.size(); ++i) {
@@ -853,6 +857,34 @@ std::string RPCHelpMan::ToString() const
     return ret;
 }
 
+std::string RPCHelpMan::ToStringArgsCli() const
+{
+    std::string res;
+    for (const auto& arg : m_args) {
+        const bool is_file = ToLower(arg.m_description).find("file") != std::string::npos;
+        res += arg.m_names + ":" + (is_file ? "file" : arg.ToTypeString()) + ",";
+    }
+
+    if (res.size() > 0) {
+        res.pop_back();
+    }
+
+    return res;
+}
+
+std::string RPCHelpMan::ToString(const std::string& format) const
+{
+    if (format == "default") {
+        return this->ToString();
+    }
+
+    if (format == "args_cli") {
+        return this->ToStringArgsCli();
+    }
+
+    throw std::runtime_error("unrecogonized help format");
+}
+
 UniValue RPCHelpMan::GetArgMap() const
 {
     UniValue arr{UniValue::VARR};
@@ -952,6 +984,32 @@ bool RPCArg::IsOptional() const
     }
 }
 
+std::string RPCArg::ToTypeString() const
+{
+    switch (m_type) {
+    case Type::STR_HEX:
+    case Type::STR:
+        return "string";
+    case Type::NUM:
+        return "numeric";
+    case Type::AMOUNT:
+        return "numeric or string";
+    case Type::RANGE:
+        return "numeric or array";
+    case Type::BOOL:
+        return "boolean";
+    case Type::OBJ:
+    case Type::OBJ_NAMED_PARAMS:
+    case Type::OBJ_USER_KEYS:
+        return "json object";
+    case Type::ARR:
+        return"json array";
+    } // no default case, so the compiler can warn about missing cases
+
+    //gcc and msvc might complain we don't return anything even if we handle all cases
+    throw std::runtime_error("unknown argument type");
+}
+
 std::string RPCArg::ToDescriptionString(bool is_named_arg) const
 {
     std::string ret;
@@ -959,39 +1017,7 @@ std::string RPCArg::ToDescriptionString(bool is_named_arg) const
     if (m_opts.type_str.size() != 0) {
         ret += m_opts.type_str.at(1);
     } else {
-        switch (m_type) {
-        case Type::STR_HEX:
-        case Type::STR: {
-            ret += "string";
-            break;
-        }
-        case Type::NUM: {
-            ret += "numeric";
-            break;
-        }
-        case Type::AMOUNT: {
-            ret += "numeric or string";
-            break;
-        }
-        case Type::RANGE: {
-            ret += "numeric or array";
-            break;
-        }
-        case Type::BOOL: {
-            ret += "boolean";
-            break;
-        }
-        case Type::OBJ:
-        case Type::OBJ_NAMED_PARAMS:
-        case Type::OBJ_USER_KEYS: {
-            ret += "json object";
-            break;
-        }
-        case Type::ARR: {
-            ret += "json array";
-            break;
-        }
-        } // no default case, so the compiler can warn about missing cases
+        ret += this->ToTypeString();
     }
     if (m_fallback.index() == 1) {
         ret += ", optional, default=" + std::get<RPCArg::DefaultHint>(m_fallback);

src/rpc/util.h

@@ -282,6 +282,10 @@ struct RPCArg {
      * Set oneline to get the oneline representation (less whitespace)
      */
     std::string ToStringObj(bool oneline) const;
+    /**
+     * Return the type as a string
+     */
+    std::string ToTypeString() const;
     /**
      * Return the description string, including the argument type and whether
      * the argument is required.
@@ -483,6 +487,8 @@ public:
         }
     }
     std::string ToString() const;
+    std::string ToStringArgsCli() const;
+    std::string ToString(const std::string& format) const;
     /** Return the named args that need to be converted from string to another JSON type */
     UniValue GetArgMap() const;
     /** If the supplied number of args is neither too small nor too high */

src/test/blockmanager_tests.cpp

@@ -33,6 +33,10 @@ BOOST_AUTO_TEST_CASE(blockmanager_find_block_pos)
         .chainparams = *params,
         .blocks_dir = m_args.GetBlocksDirPath(),
         .notifications = notifications,
+        .block_tree_db_params = DBParams{
+            .path = m_args.GetDataDirNet() / "blocks" / "index",
+            .cache_bytes = 0,
+        },
     };
     BlockManager blockman{*Assert(m_node.shutdown_signal), blockman_opts};
     // simulate adding a genesis block normally
@@ -140,6 +144,10 @@ BOOST_AUTO_TEST_CASE(blockmanager_flush_block_file)
         .chainparams = Params(),
         .blocks_dir = m_args.GetBlocksDirPath(),
         .notifications = notifications,
+        .block_tree_db_params = DBParams{
+            .path = m_args.GetDataDirNet() / "blocks" / "index",
+            .cache_bytes = 0,
+        },
     };
     BlockManager blockman{*Assert(m_node.shutdown_signal), blockman_opts};
 

src/test/fuzz/rpc.cpp

@@ -111,6 +111,7 @@ const std::vector<std::string> RPC_COMMANDS_SAFE_FOR_FUZZING{
     "estimaterawfee",
     "estimatesmartfee",
     "finalizepsbt",
+    "format",
     "generate",
     "generateblock",
     "getaddednodeinfo",

src/test/util/setup_common.cpp

@@ -62,7 +62,6 @@
 #include <stdexcept>
 
 using namespace util::hex_literals;
-using kernel::BlockTreeDB;
 using node::ApplyArgsManOptions;
 using node::BlockAssembler;
 using node::BlockManager;
@@ -252,14 +251,14 @@ ChainTestingSetup::ChainTestingSetup(const ChainType chainType, TestOpts opts)
             .chainparams = chainman_opts.chainparams,
             .blocks_dir = m_args.GetBlocksDirPath(),
             .notifications = chainman_opts.notifications,
+            .block_tree_db_params = DBParams{
+                .path = m_args.GetDataDirNet() / "blocks" / "index",
+                .cache_bytes = m_kernel_cache_sizes.block_tree_db,
+                .memory_only = opts.block_tree_db_in_memory,
+                .wipe_data = m_args.GetBoolArg("-reindex", false),
+            },
         };
         m_node.chainman = std::make_unique<ChainstateManager>(*Assert(m_node.shutdown_signal), chainman_opts, blockman_opts);
-        LOCK(m_node.chainman->GetMutex());
-        m_node.chainman->m_blockman.m_block_tree_db = std::make_unique<BlockTreeDB>(DBParams{
-            .path = m_args.GetDataDirNet() / "blocks" / "index",
-            .cache_bytes = m_kernel_cache_sizes.block_tree_db,
-            .memory_only = true,
-        });
     };
     m_make_chainman();
 }
@@ -285,9 +284,7 @@ void ChainTestingSetup::LoadVerifyActivateChainstate()
     auto& chainman{*Assert(m_node.chainman)};
     node::ChainstateLoadOptions options;
     options.mempool = Assert(m_node.mempool.get());
-    options.block_tree_db_in_memory = m_block_tree_db_in_memory;
     options.coins_db_in_memory = m_coins_db_in_memory;
-    options.wipe_block_tree_db = m_args.GetBoolArg("-reindex", false);
     options.wipe_chainstate_db = m_args.GetBoolArg("-reindex", false) || m_args.GetBoolArg("-reindex-chainstate", false);
     options.prune = chainman.m_blockman.IsPruneMode();
     options.check_blocks = m_args.GetIntArg("-checkblocks", DEFAULT_CHECKBLOCKS);

src/test/validation_chainstatemanager_tests.cpp

@@ -393,6 +393,11 @@ struct SnapshotTestSetup : TestChain100Setup {
                 .chainparams = chainman_opts.chainparams,
                 .blocks_dir = m_args.GetBlocksDirPath(),
                 .notifications = chainman_opts.notifications,
+                .block_tree_db_params = DBParams{
+                    .path = chainman.m_options.datadir / "blocks" / "index",
+                    .cache_bytes = m_kernel_cache_sizes.block_tree_db,
+                    .memory_only = m_block_tree_db_in_memory,
+                },
             };
             // For robustness, ensure the old manager is destroyed before creating a
             // new one.

src/validation.cpp

@@ -5623,9 +5623,8 @@ double ChainstateManager::GuessVerificationProgress(const CBlockIndex* pindex) c
         return 0.0;
     }
 
-    if (!Assume(pindex->m_chain_tx_count > 0)) {
-        LogWarning("Internal bug detected: block %d has unset m_chain_tx_count (%s %s). Please report this issue here: %s\n",
-                   pindex->nHeight, CLIENT_NAME, FormatFullVersion(), CLIENT_BUGREPORT);
+    if (pindex->m_chain_tx_count == 0) {
+        LogDebug(BCLog::VALIDATION, "Block %d has unset m_chain_tx_count. Unable to estimate verification progress.\n", pindex->nHeight);
         return 0.0;
     }
 

src/wallet/rpc/backup.cpp

@@ -1745,20 +1745,27 @@ RPCHelpMan importdescriptors()
                 if (scanned_time <= GetImportTimestamp(request, now) || results.at(i).exists("error")) {
                     response.push_back(results.at(i));
                 } else {
+                    std::string error_msg{strprintf("Rescan failed for descriptor with timestamp %d. There "
+                            "was an error reading a block from time %d, which is after or within %d seconds "
+                            "of key creation, and could contain transactions pertaining to the desc. As a "
+                            "result, transactions and coins using this desc may not appear in the wallet.",
+                            GetImportTimestamp(request, now), scanned_time - TIMESTAMP_WINDOW - 1, TIMESTAMP_WINDOW)};
+                    if (pwallet->chain().havePruned()) {
+                        error_msg += strprintf(" This error could be caused by pruning or data corruption "
+                                "(see bitcoind log for details) and could be dealt with by downloading and "
+                                "rescanning the relevant blocks (see -reindex option and rescanblockchain RPC).");
+                    } else if (pwallet->chain().hasAssumedValidChain()) {
+                        error_msg += strprintf(" This error is likely caused by an in-progress assumeutxo "
+                                "background sync. Check logs or getchainstates RPC for assumeutxo background "
+                                "sync progress and try again later.");
+                    } else {
+                        error_msg += strprintf(" This error could potentially caused by data corruption. If "
+                                "the issue persists you may want to reindex (see -reindex option).");
+                    }
+
                     UniValue result = UniValue(UniValue::VOBJ);
                     result.pushKV("success", UniValue(false));
-                    result.pushKV(
-                        "error",
-                        JSONRPCError(
-                            RPC_MISC_ERROR,
-                            strprintf("Rescan failed for descriptor with timestamp %d. There was an error reading a "
-                                      "block from time %d, which is after or within %d seconds of key creation, and "
-                                      "could contain transactions pertaining to the desc. As a result, transactions "
-                                      "and coins using this desc may not appear in the wallet. This error could be "
-                                      "caused by pruning or data corruption (see bitcoind log for details) and could "
-                                      "be dealt with by downloading and rescanning the relevant blocks (see -reindex "
-                                      "option and rescanblockchain RPC).",
-                                GetImportTimestamp(request, now), scanned_time - TIMESTAMP_WINDOW - 1, TIMESTAMP_WINDOW)));
+                    result.pushKV("error", JSONRPCError(RPC_MISC_ERROR, error_msg));
                     response.push_back(std::move(result));
                 }
             }

src/wallet/rpc/transactions.cpp

@@ -6,6 +6,7 @@
 #include <key_io.h>
 #include <policy/rbf.h>
 #include <rpc/util.h>
+#include <rpc/blockchain.h>
 #include <util/vector.h>
 #include <wallet/receive.h>
 #include <wallet/rpc/util.h>
@@ -909,9 +910,15 @@ RPCHelpMan rescanblockchain()
             }
         }
 
-        // We can't rescan beyond non-pruned blocks, stop and throw an error
+        // We can't rescan unavailable blocks, stop and throw an error
         if (!pwallet->chain().hasBlocks(pwallet->GetLastBlockHash(), start_height, stop_height)) {
-            throw JSONRPCError(RPC_MISC_ERROR, "Can't rescan beyond pruned data. Use RPC call getblockchaininfo to determine your pruned height.");
+            if (pwallet->chain().havePruned() && pwallet->chain().getPruneHeight() >= start_height) {
+                throw JSONRPCError(RPC_MISC_ERROR, "Can't rescan beyond pruned data. Use RPC call getblockchaininfo to determine your pruned height.");
+            }
+            if (pwallet->chain().hasAssumedValidChain()) {
+                throw JSONRPCError(RPC_MISC_ERROR, "Failed to rescan unavailable blocks likely due to an in-progress assumeutxo background sync. Check logs or getchainstates RPC for assumeutxo background sync progress and try again later.");
+            }
+            throw JSONRPCError(RPC_MISC_ERROR, "Failed to rescan unavailable blocks, potentially caused by data corruption. If the issue persists you may want to reindex (see -reindex option).");
         }
 
         CHECK_NONFATAL(pwallet->chain().findAncestorByHeight(pwallet->GetLastBlockHash(), start_height, FoundBlock().hash(start_block)));

test/functional/data/completion/bitcoin-cli.footer.bash-completion

@@ -0,0 +1,47 @@
+
+    case "$cur" in
+        -conf=*)
+            cur="${cur#*=}"
+            _filedir
+            return 0
+            ;;
+        -datadir=*)
+            cur="${cur#*=}"
+            _filedir -d
+            return 0
+            ;;
+        -*=*)	# prevent nonsense completions
+            return 0
+            ;;
+        *)
+            local helpopts commands
+
+            # only parse -help if senseful
+            if [[ -z "$cur" || "$cur" =~ ^- ]]; then
+                helpopts=$($bitcoin_cli -help 2>&1 | awk '$1 ~ /^-/ { sub(/=.*/, "="); print $1 }' )
+            fi
+
+            # only parse help if senseful
+            if [[ -z "$cur" || "$cur" =~ ^[a-z] ]]; then
+                commands=$(_bitcoin_rpc help 2>/dev/null | awk '$1 ~ /^[a-z]/ { print $1; }')
+            fi
+
+            COMPREPLY=( $( compgen -W "$helpopts $commands" -- "$cur" ) )
+
+            # Prevent space if an argument is desired
+            if [[ $COMPREPLY == *= ]]; then
+                compopt -o nospace
+            fi
+            return 0
+            ;;
+    esac
+} &&
+complete -F _bitcoin_cli bitcoin-cli
+
+# Local variables:
+# mode: shell-script
+# sh-basic-offset: 4
+# sh-indent-comment: t
+# indent-tabs-mode: nil
+# End:
+# ex: ts=4 sw=4 et filetype=sh

test/functional/data/completion/bitcoin-cli.header.bash-completion

@@ -0,0 +1,29 @@
+# Copyright (c) 2012-2024 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+# call $bitcoin-cli for RPC
+_bitcoin_rpc() {
+    # determine already specified args necessary for RPC
+    local rpcargs=()
+    for i in ${COMP_LINE}; do
+        case "$i" in
+            -conf=*|-datadir=*|-regtest|-rpc*|-testnet|-testnet4)
+                rpcargs=( "${rpcargs[@]}" "$i" )
+                ;;
+        esac
+    done
+    $bitcoin_cli "${rpcargs[@]}" "$@"
+}
+
+_bitcoin_cli() {
+    local cur prev words=() cword
+    local bitcoin_cli
+
+    # save and use original argument to invoke bitcoin-cli for -help, help and RPC
+    # as bitcoin-cli might not be in $PATH
+    bitcoin_cli="$1"
+
+    COMPREPLY=()
+    _get_comp_words_by_ref -n = cur prev words cword
+

test/functional/mining_basic.py

@@ -153,6 +153,8 @@ class MiningTest(BitcoinTestFramework):
         # The template will have an adjusted timestamp, which we then modify
         tmpl = node.getblocktemplate(NORMAL_GBT_REQUEST_PARAMS)
         assert_greater_than_or_equal(tmpl['curtime'], t + MAX_FUTURE_BLOCK_TIME - MAX_TIMEWARP)
+        # mintime and curtime should match
+        assert_equal(tmpl['mintime'], tmpl['curtime'])
 
         block = CBlock()
         block.nVersion = tmpl["version"]

test/functional/rpc_blockchain.py

@@ -549,6 +549,7 @@ class BlockchainTest(BitcoinTestFramework):
         # The chain has probably already been restored by the time reconsiderblock returns,
         # but poll anyway.
         self.wait_until(lambda: node.waitfornewblock(timeout=100)['hash'] == current_hash)
+        assert_raises_rpc_error(-1, "Negative timeout", node.waitfornewblock, -1)
 
     def _test_waitforblockheight(self):
         self.log.info("Test waitforblockheight")

test/functional/test_runner.py

@@ -193,6 +193,7 @@ BASE_SCRIPTS = [
     'feature_bind_extra.py',
     'mempool_resurrect.py',
     'wallet_txn_doublespend.py --mineblock',
+    'tool_cli_bash_completion.py',
     'tool_wallet.py --legacy-wallet',
     'tool_wallet.py --legacy-wallet --bdbro',
     'tool_wallet.py --legacy-wallet --bdbro --swap-bdb-endian',

test/functional/tool_cli_bash_completion.py

@@ -0,0 +1,282 @@
+#!/usr/bin/env python3
+
+from os import path
+from collections import defaultdict
+
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.util import assert_equal
+
+
+# bash cli completion file header
+COMPLETION_HEADER = """# Dynamic bash programmable completion for bitcoin-cli(1)
+#     DO NOT EDIT THIS FILE BY HAND -- THIS WILL FAIL THE FUNCTIONAL TEST tool_cli_completion
+# This file is auto-generated by the functional test tool_cli_completion.
+# If you want to modify this file, modify test/functional/tool_cli_completion.py and re-autogenerate
+# this file via the --overwrite test flag.
+
+"""
+
+# option types which are limited to certain values
+TYPED_OPTIONS = [
+        ["estimate_mode", {"UNSET", "ECONOMICAL", "CONSERVATIVE"}],
+        ["sighashtype", {"ALL", "NONE", "SINGLE", "ALL|ANYONECANPAY",
+                         "NONE|ANYONECANPAY", "SINGLE|ANYONECANPAY"}]
+]
+
+
+class PossibleArgs():
+    """ Helper class to store options associated to a command. """
+    def __init__(self, command):
+        self.command = command
+        self.arguments = {}
+
+    def set_args(self, position, values):
+        """ Set the position-th positional argument as having values as possible values. """
+        if position in self.arguments:
+            raise AssertionError(f"The positional parameter at position {position} is already defined for command '{self.command}'")
+
+        self.arguments[position] = values
+        return self
+
+    def set_bool_args(self, position):
+        return self.set_args(position, {"true", "false"})
+
+    def set_file_args(self, position):
+        # We consider an empty string as a file value for the sake of simplicity (don't
+        # have to create an extra level of indirection).
+        return self.set_args(position, {""})
+
+    def set_unknown_args(self, position):
+        return self.set_args(position, {})
+
+    def set_typed_option(self, position, arg_name):
+        """ Checks if arg_name is a typed option; if it is, sets it and return True. """
+        for option_type in TYPED_OPTIONS:
+            if arg_name == option_type[0]:
+                self.set_args(position, option_type[1])
+                return True
+        return False
+
+    def has_option(self, position):
+        return position in self.arguments and len(self.arguments[position]) > 0
+
+    def get_num_args(self):
+        """ Return the max number of positional argument the option accepts. """
+        pos = list(self.arguments.keys())
+        if len(pos) == 0:
+            return 0
+
+        return max(pos)
+
+    def generate_autocomplete(self, pos):
+        """ Generate the autocomplete file line relevent to the given position pos. """
+        if len(self.arguments[pos]) == 0:
+            raise AssertionError(f"generating undefined arg id {pos} ({self.arguments})")
+
+        # handle special file case
+        if len(self.arguments[pos]) == 1 and len(next(iter(self.arguments[pos]))) == 0:
+            return "_filedir"
+
+        # a set order is undefined, so we order args alphabetically
+        args = list(self.arguments[pos])
+        args.sort()
+
+        return "COMPREPLY=( $( compgen -W \"" + ' '.join(args) + "\" -- \"$cur\" ) )"
+
+# commands where the option type can only be difficultly derived from the help message
+SPECIAL_OPTIONS = [
+        PossibleArgs("addnode").set_args(2, {"add", "remove", "onetry"}),
+        PossibleArgs("setban").set_args(2, {"add", "remove"}),
+]
+
+
+def generate_start_complete(cword):
+    """ Generate the start of an autocomplete block (beware of indentation). """
+    if cword > 1:
+        return f"""    if ((cword > {cword})); then
+        case ${{words[cword-{cword}]}} in"""
+
+    return "    case \"$prev\" in"
+
+
+def generate_end_complete(cword):
+    """ Generate the end of an autocomplete block. """
+    if cword > 1:
+        return f"\n{' ' * 8}esac\n{' ' * 4}fi\n\n"
+
+    return f"\n{' ' * 4}esac\n"
+
+
+class CliCompletionTest(BitcoinTestFramework):
+    def set_test_params(self):
+        self.num_nodes = 1
+
+    def skip_test_if_missing_module(self):
+        self.skip_if_no_cli()
+        # self.skip_if_no_wallet()
+        self.skip_if_no_bitcoind_zmq()
+
+    def add_options(self, parser):
+        parser.add_argument(
+                '--header',
+                help='Static header part of the bash completion file',
+        )
+
+        parser.add_argument(
+                '--footer',
+                help='Static footer part of the bash completion file',
+        )
+
+        parser.add_argument(
+                '--completion',
+                help='Location of the current bash completion file',
+        )
+
+        parser.add_argument(
+                '--overwrite',
+                default=False,
+                action='store_true',
+                help='Force the test to overwrite the file pointer to by the --completion'
+                    'to the newly generated completion file',
+        )
+    def parse_single_helper(self, option):
+        """ Complete the arguments of option via the RPC format command. """
+
+        res = self.nodes[0].format(command=option.command, output='args_cli')
+        if len(res) == 0:
+            return option
+
+        if res.count('\n') > 1:
+            raise AssertionError(
+                f"command {option.command} doesn't support format RPC. Should it be a hidden command? "
+                f"Please call RPCHelpMan::Check when adding a new non-hidden command. Returned: {res}"
+            )
+
+        for idx, argument in enumerate(res.split(",")):
+            elems = argument.split(":")
+
+            if option.set_typed_option(idx+1, elems[0]):
+                continue
+
+            if elems[1] == "boolean":
+                option.set_bool_args(idx+1)
+                continue
+
+            if elems[1] == "file":
+                option.set_file_args(idx+1)
+                continue
+
+            if not option.has_option(idx+1):
+                option.set_unknown_args(idx+1)
+
+        return option
+
+    def get_command_options(self, command):
+        """ Returns the corresponding PossibleArgs for the command. """
+
+        # verify it's not a special option first
+        for soption in SPECIAL_OPTIONS:
+            if command == soption.command:
+                return self.parse_single_helper(soption)
+
+        return self.parse_single_helper(PossibleArgs(command))
+
+    def generate_completion_block(self, options):
+        commands = [o.command for o in options]
+        self.log.info(f"Generating part of the completion file for options {commands}")
+
+        if len(options) == 0:
+            return ""
+
+        generated = ""
+        max_pos_options = max(options, key=lambda o: o.get_num_args()).get_num_args()
+        for cword in range(max_pos_options, 0, -1):
+            this_options = [option for option in options if option.has_option(cword)]
+            if len(this_options) == 0:
+                continue
+
+            # group options by their arguments value
+            grouped_options = defaultdict(list)
+            for option in this_options:
+                arg = option.generate_autocomplete(cword)
+                grouped_options[arg].append(option)
+
+            # generate the cword block
+            indent = 12 if cword > 1 else 8
+            generated += generate_start_complete(cword)
+            for line, opt_gr in grouped_options.items():
+                opt_gr.sort(key=lambda o: o.command)  # show options alphabetically for clarity
+                args = '|'.join([o.command for o in opt_gr])
+                generated += f"\n{' '*indent}{args})\n"
+                generated += f"{' ' * (indent + 4)}{line}\n{' ' * (indent + 4)}return 0\n{' ' * (indent + 4)};;"
+            generated += generate_end_complete(cword)
+
+        return generated
+
+    def generate_completion_file(self, commands):
+        try:
+            with open(self.options.header, 'r', encoding='utf-8') as header_file:
+                header = header_file.read()
+
+            with open(self.options.footer, 'r', encoding='utf-8') as footer_file:
+                footer = footer_file.read()
+        except Exception as e:
+            raise AssertionError(
+                        f"Could not read header/footer ({self.options.header} and {self.options.footer}) files. "
+                        f"Tell the test where to find them using the --header/--footer parameters ({e})."
+            )
+        return COMPLETION_HEADER + header + commands + footer
+
+    def write_completion_file(self, new_file):
+        try:
+            with open(self.options.completion, 'w', encoding='utf-8') as completion_file:
+                completion_file.write(new_file)
+        except Exception as e:
+            raise AssertionError(
+                        f"Could not write the autocomplete file to {self.options.completion}. "
+                        f"Tell the test where to find it using the --completion parameters ({e})."
+            )
+
+    def read_completion_file(self):
+        try:
+            with open(self.options.completion, 'r', encoding='utf-8') as completion_file:
+                return completion_file.read()
+        except Exception as e:
+            raise AssertionError(
+                        f"Could not read the autocomplete file ({self.options.completion}) file. "
+                        f"Tell the test where to find it using the --completion parameters ({e})."
+            )
+
+
+    def run_test(self):
+        # self.config is not available in self.add_options, so complete filepaths here
+        src_dir = self.config["environment"]["SRCDIR"]
+        test_data_dir = path.join(src_dir, 'test', 'functional', 'data', 'completion')
+        if self.options.header is None or len(self.options.header) == 0:
+            self.options.header = path.join(test_data_dir, 'bitcoin-cli.header.bash-completion')
+
+        if self.options.footer is None or len(self.options.footer) == 0:
+            self.options.footer = path.join(test_data_dir, 'bitcoin-cli.footer.bash-completion')
+
+        if self.options.completion is None or len(self.options.completion) == 0:
+            self.options.completion = path.join(src_dir, 'contrib', 'completions', 'bash', 'bitcoin-cli.bash')
+
+        self.log.info('Parsing help commands to get all the command arguments...')
+        commands = self.nodes[0].help().split("\n")
+        commands = [c.split(' ')[0] for c in commands if not c.startswith("== ") and len(c) > 0]
+        commands = [self.get_command_options(c) for c in commands]
+
+        self.log.info('Generating new autocompletion file...')
+        commands = self.generate_completion_block(commands)
+        new_completion = self.generate_completion_file(commands)
+
+        if self.options.overwrite:
+            self.log.info("Overwriting the completion file...")
+            self.write_completion_file(new_completion)
+
+        self.log.info('Checking if the generated and the original completion files matches...')
+        completion = self.read_completion_file()
+        assert_equal(new_completion, completion)
+
+if __name__ == '__main__':
+    CliCompletionTest(__file__).main()

test/functional/wallet_assumeutxo.py

@@ -7,11 +7,11 @@ See feature_assumeutxo.py for background.
 
 ## Possible test improvements
 
-- TODO: test import descriptors while background sync is in progress
 - TODO: test loading a wallet (backup) on a pruned node
 
 """
 from test_framework.address import address_to_scriptpubkey
+from test_framework.descriptors import descsum_create
 from test_framework.test_framework import BitcoinTestFramework
 from test_framework.messages import COIN
 from test_framework.util import (
@@ -20,6 +20,7 @@ from test_framework.util import (
     ensure_for,
 )
 from test_framework.wallet import MiniWallet
+from test_framework.wallet_util import get_generate_key
 
 START_HEIGHT = 199
 SNAPSHOT_BASE_HEIGHT = 299
@@ -49,6 +50,13 @@ class AssumeutxoTest(BitcoinTestFramework):
         self.add_nodes(3)
         self.start_nodes(extra_args=self.extra_args)
 
+    def import_descriptor(self, node, wallet_name, key, timestamp):
+        import_request = [{"desc": descsum_create("pkh(" + key.pubkey + ")"),
+                           "timestamp": timestamp,
+                           "label": "Descriptor import test"}]
+        wrpc = node.get_wallet_rpc(wallet_name)
+        return wrpc.importdescriptors(import_request)
+
     def run_test(self):
         """
         Bring up two (disconnected) nodes, mine some new blocks on the first,
@@ -157,6 +165,21 @@ class AssumeutxoTest(BitcoinTestFramework):
         self.log.info("Backup from before the snapshot height can't be loaded during background sync")
         assert_raises_rpc_error(-4, "Wallet loading failed. Error loading wallet. Wallet requires blocks to be downloaded, and software does not currently support loading wallets while blocks are being downloaded out of order when using assumeutxo snapshots. Wallet should be able to load successfully after node sync reaches height 299", n1.restorewallet, "w2", "backup_w2.dat")
 
+        self.log.info("Test loading descriptors during background sync")
+        wallet_name = "w1"
+        n1.createwallet(wallet_name, disable_private_keys=True)
+        key = get_generate_key()
+        time = n1.getblockchaininfo()['time']
+        timestamp = 0
+        expected_error_message = f"Rescan failed for descriptor with timestamp {timestamp}. There was an error reading a block from time {time}, which is after or within 7200 seconds of key creation, and could contain transactions pertaining to the desc. As a result, transactions and coins using this desc may not appear in the wallet. This error is likely caused by an in-progress assumeutxo background sync. Check logs or getchainstates RPC for assumeutxo background sync progress and try again later."
+        result = self.import_descriptor(n1, wallet_name, key, timestamp)
+        assert_equal(result[0]['error']['code'], -1)
+        assert_equal(result[0]['error']['message'], expected_error_message)
+
+        self.log.info("Test that rescanning blocks from before the snapshot fails when blocks are not available from the background sync yet")
+        w1 = n1.get_wallet_rpc(wallet_name)
+        assert_raises_rpc_error(-1, "Failed to rescan unavailable blocks likely due to an in-progress assumeutxo background sync. Check logs or getchainstates RPC for assumeutxo background sync progress and try again later.", w1.rescanblockchain, 100)
+
         PAUSE_HEIGHT = FINAL_HEIGHT - 40
 
         self.log.info("Restarting node to stop at height %d", PAUSE_HEIGHT)
@@ -204,6 +227,11 @@ class AssumeutxoTest(BitcoinTestFramework):
         self.wait_until(lambda: len(n2.getchainstates()['chainstates']) == 1)
         ensure_for(duration=1, f=lambda: (n2.getbalance() == 34))
 
+        self.log.info("Ensuring descriptors can be loaded after background sync")
+        n1.loadwallet(wallet_name)
+        result = self.import_descriptor(n1, wallet_name, key, timestamp)
+        assert_equal(result[0]['success'], True)
+
 
 if __name__ == '__main__':
     AssumeutxoTest(__file__).main()