Merge db442f3d6e into 85f96b01b7

9d2d9f7ce2 rpc: Include assumeutxo as a failure reason of rescanblockchain (Fabian Jahr)
595edee169 test, assumeutxo: import descriptors during background sync (Alfonso Roman Zubeldia)
d73ae603d4 rpc: Improve importdescriptor RPC error messages (Fabian Jahr)
27f99b6d63 validation: Don't assume m_chain_tx_count in GuessVerificationProgress (Fabian Jahr)
42d5d53363 interfaces: Add helper function for wallet on pruning (Fabian Jahr)

Pull request description:

  A test that is added as part of #30455 uncovered this issue: The `GuessVerificationProgress` function is used during during descriptor import and relies on `m_chain_tx_count`. In #29370 an [`Assume` was added](0fd915ee6b) expecting the `m_chaint_tx_count` to be set. However, as the test uncovered, `GuessVerificationProgress` is called with background sync blocks that have `m_chaint_tx_count = 0` when they have not been downloaded and processed yet.

  The simple fix is to remove the `Assume`. Users should not be thrown off by the `Internal bug detected` error. The behavior of `importdescriptor` is kept consistent with the behavior for blocks missing due to pruning.

  The test by alfonsoromanz is cherry-picked here to show that the [CI errors](https://cirrus-ci.com/task/5110045812195328?logs=ci#L2535) should be fixed by this change.

  This PR also improves error messages returned by the `importdescriptors` and `rescanblockchain` RPCs. The error message now changes depending on the situation of the node, i.e. if pruning is happening or an assumutxo backgroundsync is active.

ACKs for top commit:
  achow101:
    ACK 9d2d9f7ce2
  mzumsande:
    Code Review ACK 9d2d9f7ce2
  furszy:
    Code review ACK 9d2d9f7ce2

Tree-SHA512: b841a9b371e5eb8eb3bfebca35645ff2fdded7a3e5e06308d46a33a51ca42cc4c258028c9958fbbb6cda9bb990e07ab8d8504dd9ec6705ef78afe0435912b365

doc/release-notes-31600.md

@@ -0,0 +1,11 @@
+Updated RPCs
+---
+- the `getblocktemplate` RPC `curtime` (BIP22) and `mintime` (BIP23) fields now
+  account for the timewarp fix proposed in BIP94 on all networks. This ensures
+  that, in the event a timewarp fix softfork activates on mainnet, un-upgraded
+  miners will not accidentally violate the timewarp rule. (#31376, #31600)
+
+As a reminder, it's important that any software which uses the `getblocktemplate`
+RPC takes these values into account (either `curtime` or `mintime` is fine).
+Relying only on a clock can lead to invalid blocks under some circumstances,
+especially once a timewarp fix is deployed.

src/bitcoin-chainstate.cpp

@@ -106,6 +106,7 @@ int main(int argc, char* argv[])
     };
     auto notifications = std::make_unique<KernelNotifications>();
 
+    kernel::CacheSizes cache_sizes{DEFAULT_KERNEL_CACHE};
 
     // SETUP: Chainstate
     auto chainparams = CChainParams::Main();
@@ -119,11 +120,14 @@ int main(int argc, char* argv[])
         .chainparams = chainman_opts.chainparams,
         .blocks_dir = abs_datadir / "blocks",
         .notifications = chainman_opts.notifications,
+        .block_tree_db_params = DBParams{
+            .path = abs_datadir / "blocks" / "index",
+            .cache_bytes = cache_sizes.block_tree_db,
+        },
     };
     util::SignalInterrupt interrupt;
     ChainstateManager chainman{interrupt, chainman_opts, blockman_opts};
 
-    kernel::CacheSizes cache_sizes{DEFAULT_KERNEL_CACHE};
     node::ChainstateLoadOptions options;
     auto [status, error] = node::LoadChainstate(chainman, cache_sizes, options);
     if (status != node::ChainstateLoadStatus::SUCCESS) {

src/init.cpp

@@ -1057,6 +1057,10 @@ bool AppInitParameterInteraction(const ArgsManager& args)
             .chainparams = chainman_opts_dummy.chainparams,
             .blocks_dir = args.GetBlocksDirPath(),
             .notifications = chainman_opts_dummy.notifications,
+            .block_tree_db_params = DBParams{
+                .path = args.GetDataDirNet() / "blocks" / "index",
+                .cache_bytes = 0,
+            },
         };
         auto blockman_result{ApplyArgsManOptions(args, blockman_opts_dummy)};
         if (!blockman_result) {
@@ -1203,18 +1207,33 @@ static ChainstateLoadResult InitAndLoadChainstate(
         .signals = node.validation_signals.get(),
     };
     Assert(ApplyArgsManOptions(args, chainman_opts)); // no error can happen, already checked in AppInitParameterInteraction
+
     BlockManager::Options blockman_opts{
         .chainparams = chainman_opts.chainparams,
         .blocks_dir = args.GetBlocksDirPath(),
         .notifications = chainman_opts.notifications,
+        .block_tree_db_params = DBParams{
+            .path = args.GetDataDirNet() / "blocks" / "index",
+            .cache_bytes = cache_sizes.block_tree_db,
+            .wipe_data = do_reindex,
+        },
     };
     Assert(ApplyArgsManOptions(args, blockman_opts)); // no error can happen, already checked in AppInitParameterInteraction
+
+    // Creating the chainstate manager internally creates a BlockManager, opens
+    // the blocks tree db, and wipes existing block files in case of a reindex.
+    // The coinsdb is opened at a later point on LoadChainstate.
     try {
         node.chainman = std::make_unique<ChainstateManager>(*Assert(node.shutdown_signal), chainman_opts, blockman_opts);
+    } catch (dbwrapper_error& e) {
+        LogError("%s", e.what());
+        return {ChainstateLoadStatus::FAILURE, _("Error opening block database")};
     } catch (std::exception& e) {
         return {ChainstateLoadStatus::FAILURE_FATAL, Untranslated(strprintf("Failed to initialize ChainstateManager: %s", e.what()))};
     }
     ChainstateManager& chainman = *node.chainman;
+    if (chainman.m_interrupt) return {ChainstateLoadStatus::INTERRUPTED, {}};
+
     // This is defined and set here instead of inline in validation.h to avoid a hard
     // dependency between validation and index/base, since the latter is not in
     // libbitcoinkernel.
@@ -1237,7 +1256,6 @@ static ChainstateLoadResult InitAndLoadChainstate(
     };
     node::ChainstateLoadOptions options;
     options.mempool = Assert(node.mempool.get());
-    options.wipe_block_tree_db = do_reindex;
     options.wipe_chainstate_db = do_reindex || do_reindex_chainstate;
     options.prune = chainman.m_blockman.IsPruneMode();
     options.check_blocks = args.GetIntArg("-checkblocks", DEFAULT_CHECKBLOCKS);

src/interfaces/chain.h

@@ -289,6 +289,9 @@ public:
     //! Check if any block has been pruned.
     virtual bool havePruned() = 0;
 
+    //! Get the current prune height.
+    virtual std::optional<int> getPruneHeight() = 0;
+
     //! Check if the node is ready to broadcast transactions.
     virtual bool isReadyToBroadcast() = 0;
 

src/kernel/blockmanager_opts.h

@@ -5,6 +5,7 @@
 #ifndef BITCOIN_KERNEL_BLOCKMANAGER_OPTS_H
 #define BITCOIN_KERNEL_BLOCKMANAGER_OPTS_H
 
+#include <dbwrapper.h>
 #include <kernel/notifications_interface.h>
 #include <util/fs.h>
 
@@ -27,6 +28,7 @@ struct BlockManagerOpts {
     bool fast_prune{false};
     const fs::path blocks_dir;
     Notifications& notifications;
+    DBParams block_tree_db_params;
 };
 
 } // namespace kernel

src/kernel/chainstatemanager_opts.h

@@ -42,7 +42,6 @@ struct ChainstateManagerOpts {
     std::optional<uint256> assumed_valid_block{};
     //! If the tip is older than this, the node is considered to be in initial block download.
     std::chrono::seconds max_tip_age{DEFAULT_MAX_TIP_AGE};
-    DBOptions block_tree_db{};
     DBOptions coins_db{};
     CoinsViewOptions coins_view{};
     Notifications& notifications;

src/net.cpp

@@ -652,7 +652,7 @@ void CNode::CopyStats(CNodeStats& stats)
 }
 #undef X
 
-bool CNode::ReceiveMsgBytes(Span<const uint8_t> msg_bytes, bool& complete)
+bool CNode::ReceiveMsgBytes(Span<const uint8_t> msg_bytes, bool& complete, NetStats& net_stats)
 {
     complete = false;
     const auto time = GetTime<std::chrono::microseconds>();
@@ -674,6 +674,12 @@ bool CNode::ReceiveMsgBytes(Span<const uint8_t> msg_bytes, bool& complete)
                 // Message deserialization failed. Drop the message but don't disconnect the peer.
                 // store the size of the corrupt message
                 mapRecvBytesPerMsgType.at(NET_MESSAGE_TYPE_OTHER) += msg.m_raw_message_size;
+                net_stats.Record(NetStats::RECV,
+                                 ConnectedThroughNetwork(),
+                                 m_conn_type,
+                                 NET_MESSAGE_TYPE_OTHER,
+                                 /*num_messages=*/1,
+                                 msg.m_raw_message_size);
                 continue;
             }
 
@@ -685,6 +691,12 @@ bool CNode::ReceiveMsgBytes(Span<const uint8_t> msg_bytes, bool& complete)
             }
             assert(i != mapRecvBytesPerMsgType.end());
             i->second += msg.m_raw_message_size;
+            net_stats.Record(NetStats::RECV,
+                             ConnectedThroughNetwork(),
+                             m_conn_type,
+                             /*msg_type=*/i->first,
+                             /*num_messages=*/1,
+                             msg.m_raw_message_size);
 
             // push the message to the process queue,
             vRecvMsg.push_back(std::move(msg));
@@ -1585,7 +1597,7 @@ Transport::Info V2Transport::GetInfo() const noexcept
     return info;
 }
 
-std::pair<size_t, bool> CConnman::SocketSendData(CNode& node) const
+std::pair<size_t, bool> CConnman::SocketSendData(CNode& node)
 {
     auto it = node.vSendMsg.begin();
     size_t nSentSize = 0;
@@ -1598,9 +1610,16 @@ std::pair<size_t, bool> CConnman::SocketSendData(CNode& node) const
             // there is an existing message still being sent, or (for v2 transports) when the
             // handshake has not yet completed.
             size_t memusage = it->GetMemoryUsage();
+            const auto msg_type = it->m_type;
             if (node.m_transport->SetMessageToSend(*it)) {
                 // Update memory usage of send buffer (as *it will be deleted).
                 node.m_send_memusage -= memusage;
+                m_net_stats.Record(NetStats::SENT,
+                                   node.ConnectedThroughNetwork(),
+                                   node.m_conn_type,
+                                   msg_type,
+                                   /*num_messages=*/1,
+                                   /*num_bytes=*/0);
                 ++it;
             }
         }
@@ -1636,6 +1655,12 @@ std::pair<size_t, bool> CConnman::SocketSendData(CNode& node) const
             // Update statistics per message type.
             if (!msg_type.empty()) { // don't report v2 handshake bytes for now
                 node.AccountForSentBytes(msg_type, nBytes);
+                m_net_stats.Record(NetStats::SENT,
+                                   node.ConnectedThroughNetwork(),
+                                   node.m_conn_type,
+                                   msg_type,
+                                   /*num_messages=*/0,
+                                   nBytes);
             }
             nSentSize += nBytes;
             if ((size_t)nBytes != data.size()) {
@@ -2146,7 +2171,7 @@ void CConnman::SocketHandlerConnected(const std::vector<CNode*>& nodes,
             if (nBytes > 0)
             {
                 bool notify = false;
-                if (!pnode->ReceiveMsgBytes({pchBuf, (size_t)nBytes}, notify)) {
+                if (!pnode->ReceiveMsgBytes({pchBuf, (size_t)nBytes}, notify, m_net_stats)) {
                     LogDebug(BCLog::NET,
                         "receiving message bytes failed, %s\n",
                         pnode->DisconnectMsg(fLogIPs)
@@ -3670,6 +3695,150 @@ void CConnman::RecordBytesSent(uint64_t bytes)
     nMaxOutboundTotalBytesSentInCycle += bytes;
 }
 
+NetStats::NetStats()
+    : m_msg_type_to_index{[]() {
+          MsgTypeToIndex m;
+          size_t i{0};
+          for (const auto& msg_type : ALL_NET_MESSAGE_TYPES) {
+              m[msg_type] = i++;
+          }
+          return m;
+      }()}
+{
+}
+
+void NetStats::Record(Direction direction,
+                      Network net,
+                      ConnectionType conn_type,
+                      const std::string& msg_type,
+                      size_t num_messages,
+                      size_t num_bytes)
+{
+    auto& d = m_data.at(DirectionToIndex(direction))
+                  .at(NetworkToIndex(net))
+                  .at(ConnectionTypeToIndex(conn_type))
+                  .at(MessageTypeToIndex(msg_type));
+    d.count += num_messages;
+    d.bytes += num_bytes;
+}
+
+void NetStats::ForEach(std::function<void(NetStats::Direction dir,
+                                          Network net,
+                                          ConnectionType con,
+                                          const std::string& msg,
+                                          const BytesAndCount& data)> func) const
+{
+    for (size_t dir_i = 0; dir_i < m_data.size(); ++dir_i) {
+        for (size_t net_i = 0; net_i < m_data[dir_i].size(); ++net_i) {
+            for (size_t con_i = 0; con_i < m_data[dir_i][net_i].size(); ++con_i) {
+                for (size_t msg_i = 0; msg_i < m_data[dir_i][net_i][con_i].size(); ++msg_i) {
+                    func(DirectionFromIndex(dir_i),
+                         NetworkFromIndex(net_i),
+                         ConnectionTypeFromIndex(con_i),
+                         MessageTypeFromIndex(msg_i),
+                         m_data[dir_i][net_i][con_i][msg_i]);
+                }
+            }
+        }
+    }
+}
+
+constexpr size_t NetStats::DirectionToIndex(Direction direction)
+{
+    switch (direction) {
+    case SENT: return 0;
+    case RECV: return 1;
+    }
+    assert(false);
+}
+
+constexpr NetStats::Direction NetStats::DirectionFromIndex(size_t index)
+{
+    switch (index) {
+    case 0: return SENT;
+    case 1: return RECV;
+    }
+    assert(false);
+}
+
+constexpr size_t NetStats::NetworkToIndex(Network net)
+{
+    switch (net) {
+    case NET_UNROUTABLE: return 0;
+    case NET_IPV4: return 1;
+    case NET_IPV6: return 2;
+    case NET_ONION: return 3;
+    case NET_I2P: return 4;
+    case NET_CJDNS: return 5;
+    case NET_INTERNAL: return 6;
+    case NET_MAX: assert(false);
+    }
+    assert(false);
+}
+
+constexpr Network NetStats::NetworkFromIndex(size_t index)
+{
+    switch (index) {
+    case 0: return NET_UNROUTABLE;
+    case 1: return NET_IPV4;
+    case 2: return NET_IPV6;
+    case 3: return NET_ONION;
+    case 4: return NET_I2P;
+    case 5: return NET_CJDNS;
+    case 6: return NET_INTERNAL;
+    }
+    assert(false);
+}
+
+constexpr size_t NetStats::ConnectionTypeToIndex(ConnectionType conn_type)
+{
+    switch (conn_type) {
+    case ConnectionType::INBOUND: return 0;
+    case ConnectionType::OUTBOUND_FULL_RELAY: return 1;
+    case ConnectionType::MANUAL: return 2;
+    case ConnectionType::FEELER: return 3;
+    case ConnectionType::BLOCK_RELAY: return 4;
+    case ConnectionType::ADDR_FETCH: return 5;
+    }
+    assert(false);
+}
+
+constexpr ConnectionType NetStats::ConnectionTypeFromIndex(size_t index)
+{
+    switch (index) {
+    case 0: return ConnectionType::INBOUND;
+    case 1: return ConnectionType::OUTBOUND_FULL_RELAY;
+    case 2: return ConnectionType::MANUAL;
+    case 3: return ConnectionType::FEELER;
+    case 4: return ConnectionType::BLOCK_RELAY;
+    case 5: return ConnectionType::ADDR_FETCH;
+    }
+    assert(false);
+}
+
+size_t NetStats::MessageTypeToIndex(const std::string& msg_type) const
+{
+    auto it = m_msg_type_to_index.find(msg_type);
+    if (it != m_msg_type_to_index.end()) {
+        return it->second;
+    }
+    // Unknown message (NET_MESSAGE_TYPE_OTHER), use the last entry in the array.
+    return ALL_NET_MESSAGE_TYPES.size();
+}
+
+std::string NetStats::MessageTypeFromIndex(size_t index)
+{
+    if (index == ALL_NET_MESSAGE_TYPES.size()) {
+        return NET_MESSAGE_TYPE_OTHER;
+    }
+    return ALL_NET_MESSAGE_TYPES.at(index);
+}
+
+const NetStats& CConnman::GetNetStats() const
+{
+    return m_net_stats;
+}
+
 uint64_t CConnman::GetMaxOutboundTarget() const
 {
     AssertLockNotHeld(m_total_bytes_sent_mutex);

src/net.h

@@ -659,6 +659,84 @@ public:
     Info GetInfo() const noexcept override EXCLUSIVE_LOCKS_REQUIRED(!m_recv_mutex);
 };
 
+/**
+ * Network traffic (bytes and number of messages). Split by direction, network, connection type and message type.
+ */
+class NetStats
+{
+public:
+    /// Used to designate the direction of the recorded traffic.
+    enum Direction { SENT, RECV };
+
+    /// Number of elements in `Direction`.
+    static constexpr size_t NUM_DIRECTIONS{2};
+
+    struct BytesAndCount {
+        std::atomic_uint64_t bytes{0}; //!< Number of bytes transferred.
+        std::atomic_uint64_t count{0}; //!< Number of messages transferred.
+
+        BytesAndCount& operator+=(const BytesAndCount& toadd)
+        {
+            bytes += toadd.bytes;
+            count += toadd.count;
+            return *this;
+        }
+    };
+
+    NetStats();
+
+    /**
+     * Increment the number of messages transferred by `num_messages` and the number of bytes by `num_bytes`.
+     */
+    void Record(Direction direction,
+                Network net,
+                ConnectionType conn_type,
+                const std::string& msg_type,
+                size_t num_messages,
+                size_t num_bytes);
+
+    /**
+     * Call the provided function for each stat.
+     */
+    void ForEach(std::function<void(NetStats::Direction dir,
+                                    Network net,
+                                    ConnectionType con,
+                                    const std::string& msg,
+                                    const BytesAndCount& data)> func) const;
+
+private:
+    // The ...FromIndex() and ...ToIndex() methods below convert from/to
+    // indexes of `m_data[]` to the actual values they represent. For example,
+    // assuming MessageTypeToIndex("ping") == 15, then everything stored in
+    // m_data[i][j][k][15] is traffic from "ping" messages (for any i, j, k).
+
+    static constexpr size_t DirectionToIndex(Direction direction);
+    static constexpr Direction DirectionFromIndex(size_t index);
+
+    static constexpr size_t NetworkToIndex(Network net);
+    static constexpr Network NetworkFromIndex(size_t index);
+
+    static constexpr size_t ConnectionTypeToIndex(ConnectionType conn_type);
+    static constexpr ConnectionType ConnectionTypeFromIndex(size_t index);
+
+    size_t MessageTypeToIndex(const std::string& msg_type) const;
+    static std::string MessageTypeFromIndex(size_t index);
+
+    // Access like m_data[direction index][net index][conn type index][msg type index].bytes = 123;
+    // Arrays are used so that this can be accessed from multiple threads without a mutex protection.
+    std::array<std::array<std::array<std::array<BytesAndCount, ALL_NET_MESSAGE_TYPES.size() + 1>,
+                                     NUM_CONNECTION_TYPES>,
+                          NET_MAX>,
+               NUM_DIRECTIONS>
+        m_data;
+
+
+    using MsgTypeToIndex = std::unordered_map<std::string, size_t>;
+
+    /// Holds the index `i` in `m_data[][][][i]` of a given message type for quick lookup.
+    const MsgTypeToIndex m_msg_type_to_index;
+};
+
 struct CNodeOptions
 {
     NetPermissionFlags permission_flags = NetPermissionFlags::None;
@@ -914,7 +992,8 @@ public:
      * @return  True if the peer should stay connected,
      *          False if the peer should be disconnected from.
      */
-    bool ReceiveMsgBytes(Span<const uint8_t> msg_bytes, bool& complete) EXCLUSIVE_LOCKS_REQUIRED(!cs_vRecv);
+    bool ReceiveMsgBytes(Span<const uint8_t> msg_bytes, bool& complete, NetStats& net_stats)
+        EXCLUSIVE_LOCKS_REQUIRED(!cs_vRecv);
 
     void SetCommonVersion(int greatest_common_version)
     {
@@ -1272,6 +1351,8 @@ public:
 
     bool MultipleManualOrFullOutboundConns(Network net) const EXCLUSIVE_LOCKS_REQUIRED(m_nodes_mutex);
 
+    const NetStats& GetNetStats() const;
+
 private:
     struct ListenSocket {
     public:
@@ -1371,7 +1452,7 @@ private:
     NodeId GetNewNodeId();
 
     /** (Try to) send data from node's vSendMsg. Returns (bytes_sent, data_left). */
-    std::pair<size_t, bool> SocketSendData(CNode& node) const EXCLUSIVE_LOCKS_REQUIRED(node.cs_vSend);
+    std::pair<size_t, bool> SocketSendData(CNode& node) EXCLUSIVE_LOCKS_REQUIRED(node.cs_vSend);
 
     void DumpAddresses();
 
@@ -1413,6 +1494,8 @@ private:
     std::atomic<uint64_t> nTotalBytesRecv{0};
     uint64_t nTotalBytesSent GUARDED_BY(m_total_bytes_sent_mutex) {0};
 
+    NetStats m_net_stats;
+
     // outbound limit & stats
     uint64_t nMaxOutboundTotalBytesSentInCycle GUARDED_BY(m_total_bytes_sent_mutex) {0};
     std::chrono::seconds nMaxOutboundCycleStartTime GUARDED_BY(m_total_bytes_sent_mutex) {0};

src/node/blockmanager_args.cpp

@@ -6,6 +6,7 @@
 
 #include <common/args.h>
 #include <node/blockstorage.h>
+#include <node/database_args.h>
 #include <tinyformat.h>
 #include <util/result.h>
 #include <util/translation.h>
@@ -34,6 +35,8 @@ util::Result<void> ApplyArgsManOptions(const ArgsManager& args, BlockManager::Op
 
     if (auto value{args.GetBoolArg("-fastprune")}) opts.fast_prune = *value;
 
+    ReadDatabaseArgs(args, opts.block_tree_db_params.options);
+
     return {};
 }
 } // namespace node

src/node/blockstorage.cpp

@@ -36,6 +36,7 @@
 #include <util/translation.h>
 #include <validation.h>
 
+#include <cstddef>
 #include <map>
 #include <ranges>
 #include <unordered_map>
@@ -1169,7 +1170,19 @@ BlockManager::BlockManager(const util::SignalInterrupt& interrupt, Options opts)
       m_opts{std::move(opts)},
       m_block_file_seq{FlatFileSeq{m_opts.blocks_dir, "blk", m_opts.fast_prune ? 0x4000 /* 16kB */ : BLOCKFILE_CHUNK_SIZE}},
       m_undo_file_seq{FlatFileSeq{m_opts.blocks_dir, "rev", UNDOFILE_CHUNK_SIZE}},
-      m_interrupt{interrupt} {}
+      m_interrupt{interrupt}
+{
+    m_block_tree_db = std::make_unique<BlockTreeDB>(m_opts.block_tree_db_params);
+
+    if (m_opts.block_tree_db_params.wipe_data) {
+        m_block_tree_db->WriteReindexing(true);
+        m_blockfiles_indexed = false;
+        // If we're reindexing in prune mode, wipe away unusable block files and all undo data files
+        if (m_prune_mode) {
+            CleanupBlockRevFiles();
+        }
+    }
+}
 
 class ImportingNow
 {

src/node/chainstate.cpp

@@ -23,10 +23,7 @@
 #include <validation.h>
 
 #include <algorithm>
-#include <atomic>
 #include <cassert>
-#include <limits>
-#include <memory>
 #include <vector>
 
 using kernel::CacheSizes;
@@ -36,34 +33,8 @@ namespace node {
 // to ChainstateManager::InitializeChainstate().
 static ChainstateLoadResult CompleteChainstateInitialization(
     ChainstateManager& chainman,
-    const CacheSizes& cache_sizes,
     const ChainstateLoadOptions& options) EXCLUSIVE_LOCKS_REQUIRED(::cs_main)
 {
-    auto& pblocktree{chainman.m_blockman.m_block_tree_db};
-    // new BlockTreeDB tries to delete the existing file, which
-    // fails if it's still open from the previous loop. Close it first:
-    pblocktree.reset();
-    try {
-        pblocktree = std::make_unique<BlockTreeDB>(DBParams{
-            .path = chainman.m_options.datadir / "blocks" / "index",
-            .cache_bytes = cache_sizes.block_tree_db,
-            .memory_only = options.block_tree_db_in_memory,
-            .wipe_data = options.wipe_block_tree_db,
-            .options = chainman.m_options.block_tree_db});
-    } catch (dbwrapper_error& err) {
-        LogError("%s\n", err.what());
-        return {ChainstateLoadStatus::FAILURE, _("Error opening block database")};
-    }
-
-    if (options.wipe_block_tree_db) {
-        pblocktree->WriteReindexing(true);
-        chainman.m_blockman.m_blockfiles_indexed = false;
-        //If we're reindexing in prune mode, wipe away unusable block files and all undo data files
-        if (options.prune) {
-            chainman.m_blockman.CleanupBlockRevFiles();
-        }
-    }
-
     if (chainman.m_interrupt) return {ChainstateLoadStatus::INTERRUPTED, {}};
 
     // LoadBlockIndex will load m_have_pruned if we've ever removed a
@@ -155,14 +126,12 @@ static ChainstateLoadResult CompleteChainstateInitialization(
         }
     }
 
-    if (!options.wipe_block_tree_db) {
-        auto chainstates{chainman.GetAll()};
-        if (std::any_of(chainstates.begin(), chainstates.end(),
-                        [](const Chainstate* cs) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { return cs->NeedsRedownload(); })) {
-            return {ChainstateLoadStatus::FAILURE, strprintf(_("Witness data for blocks after height %d requires validation. Please restart with -reindex."),
-                                                             chainman.GetConsensus().SegwitHeight)};
-        };
-    }
+    auto chainstates{chainman.GetAll()};
+    if (std::any_of(chainstates.begin(), chainstates.end(),
+                    [](const Chainstate* cs) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { return cs->NeedsRedownload(); })) {
+        return {ChainstateLoadStatus::FAILURE, strprintf(_("Witness data for blocks after height %d requires validation. Please restart with -reindex."),
+                                                         chainman.GetConsensus().SegwitHeight)};
+    };
 
     // Now that chainstates are loaded and we're able to flush to
     // disk, rebalance the coins caches to desired levels based
@@ -208,7 +177,7 @@ ChainstateLoadResult LoadChainstate(ChainstateManager& chainman, const CacheSize
         }
     }
 
-    auto [init_status, init_error] = CompleteChainstateInitialization(chainman, cache_sizes, options);
+    auto [init_status, init_error] = CompleteChainstateInitialization(chainman, options);
     if (init_status != ChainstateLoadStatus::SUCCESS) {
         return {init_status, init_error};
     }
@@ -244,7 +213,7 @@ ChainstateLoadResult LoadChainstate(ChainstateManager& chainman, const CacheSize
         // for the fully validated chainstate.
         chainman.ActiveChainstate().ClearBlockIndexCandidates();
 
-        auto [init_status, init_error] = CompleteChainstateInitialization(chainman, cache_sizes, options);
+        auto [init_status, init_error] = CompleteChainstateInitialization(chainman, options);
         if (init_status != ChainstateLoadStatus::SUCCESS) {
             return {init_status, init_error};
         }

src/node/chainstate.h

@@ -22,12 +22,7 @@ namespace node {
 
 struct ChainstateLoadOptions {
     CTxMemPool* mempool{nullptr};
-    bool block_tree_db_in_memory{false};
     bool coins_db_in_memory{false};
-    // Whether to wipe the block tree database when loading it. If set, this
-    // will also set a reindexing flag so any existing block data files will be
-    // scanned and added to the database.
-    bool wipe_block_tree_db{false};
     // Whether to wipe the chainstate database when loading it. If set, this
     // will cause the chainstate database to be rebuilt starting from genesis.
     bool wipe_chainstate_db{false};

src/node/chainstatemanager_args.cpp

@@ -49,7 +49,6 @@ util::Result<void> ApplyArgsManOptions(const ArgsManager& args, ChainstateManage
 
     if (auto value{args.GetIntArg("-maxtipage")}) opts.max_tip_age = std::chrono::seconds{*value};
 
-    ReadDatabaseArgs(args, opts.block_tree_db);
     ReadDatabaseArgs(args, opts.coins_db);
     ReadCoinsViewArgs(args, opts.coins_view);
 

src/node/connection_types.h

@@ -12,9 +12,11 @@
  * information we have available at the time of opening or accepting the
  * connection. Aside from INBOUND, all types are initiated by us.
  *
- * If adding or removing types, please update CONNECTION_TYPE_DOC in
- * src/rpc/net.cpp and src/qt/rpcconsole.cpp, as well as the descriptions in
- * src/qt/guiutil.cpp and src/bitcoin-cli.cpp::NetinfoRequestHandler. */
+ * If adding or removing types, please update:
+ * - CONNECTION_TYPE_DOC in src/rpc/net.cpp and src/qt/rpcconsole.cpp
+ * - the descriptions in src/qt/guiutil.cpp and src/bitcoin-cli.cpp::NetinfoRequestHandler
+ * - NUM_CONNECTION_TYPES below.
+ */
 enum class ConnectionType {
     /**
      * Inbound connections are those initiated by a peer. This is the only
@@ -77,6 +79,9 @@ enum class ConnectionType {
     ADDR_FETCH,
 };
 
+/** Number of entries in ConnectionType. */
+static constexpr size_t NUM_CONNECTION_TYPES{6};
+
 /** Convert ConnectionType enum to a string value */
 std::string ConnectionTypeAsString(ConnectionType conn_type);
 

src/node/interfaces.cpp

@@ -46,6 +46,7 @@
 #include <policy/settings.h>
 #include <primitives/block.h>
 #include <primitives/transaction.h>
+#include <rpc/blockchain.h>
 #include <rpc/protocol.h>
 #include <rpc/server.h>
 #include <support/allocators/secure.h>
@@ -770,6 +771,11 @@ public:
         LOCK(::cs_main);
         return chainman().m_blockman.m_have_pruned;
     }
+    std::optional<int> getPruneHeight() override
+    {
+        LOCK(chainman().GetMutex());
+        return GetPruneHeight(chainman().m_blockman, chainman().ActiveChain());
+    }
     bool isReadyToBroadcast() override { return !chainman().m_blockman.LoadingBlocks() && !isInitialBlockDownload(); }
     bool isInitialBlockDownload() override
     {

src/node/miner.cpp

@@ -28,16 +28,25 @@
 #include <utility>
 
 namespace node {
+
+int64_t GetMinimumTime(const CBlockIndex* pindexPrev, const int64_t difficulty_adjustment_interval)
+{
+    int64_t min_time{pindexPrev->GetMedianTimePast() + 1};
+    // Height of block to be mined.
+    const int height{pindexPrev->nHeight + 1};
+    // Account for BIP94 timewarp rule on all networks. This makes future
+    // activation safer.
+    if (height % difficulty_adjustment_interval == 0) {
+        min_time = std::max<int64_t>(min_time, pindexPrev->GetBlockTime() - MAX_TIMEWARP);
+    }
+    return min_time;
+}
+
 int64_t UpdateTime(CBlockHeader* pblock, const Consensus::Params& consensusParams, const CBlockIndex* pindexPrev)
 {
     int64_t nOldTime = pblock->nTime;
-    int64_t nNewTime{std::max<int64_t>(pindexPrev->GetMedianTimePast() + 1, TicksSinceEpoch<std::chrono::seconds>(NodeClock::now()))};
-
-    // Height of block to be mined.
-    const int height{pindexPrev->nHeight + 1};
-    if (height % consensusParams.DifficultyAdjustmentInterval() == 0) {
-        nNewTime = std::max<int64_t>(nNewTime, pindexPrev->GetBlockTime() - MAX_TIMEWARP);
-    }
+    int64_t nNewTime{std::max<int64_t>(GetMinimumTime(pindexPrev, consensusParams.DifficultyAdjustmentInterval()),
+                                       TicksSinceEpoch<std::chrono::seconds>(NodeClock::now()))};
 
     if (nOldTime < nNewTime) {
         pblock->nTime = nNewTime;

src/node/miner.h

@@ -211,6 +211,13 @@ private:
     void SortForBlock(const CTxMemPool::setEntries& package, std::vector<CTxMemPool::txiter>& sortedEntries);
 };
 
+/**
+ * Get the minimum time a miner should use in the next block. This always
+ * accounts for the BIP94 timewarp rule, so does not necessarily reflect the
+ * consensus limit.
+ */
+int64_t GetMinimumTime(const CBlockIndex* pindexPrev, const int64_t difficulty_adjustment_interval);
+
 int64_t UpdateTime(CBlockHeader* pblock, const Consensus::Params& consensusParams, const CBlockIndex* pindexPrev);
 
 /** Update an old GenerateCoinbaseCommitment from CreateNewBlock after the block txs have changed */

src/rpc/client.cpp

@@ -253,6 +253,7 @@ static const CRPCConvertParam vRPCConvertParams[] =
     { "verifychain", 1, "nblocks" },
     { "getblockstats", 0, "hash_or_height" },
     { "getblockstats", 1, "stats" },
+    { "getnetmsgstats", 0, "aggregate_by"},
     { "pruneblockchain", 0, "height" },
     { "keypoolrefill", 0, "newsize" },
     { "getrawmempool", 0, "verbose" },

src/rpc/mining.cpp

@@ -49,6 +49,7 @@
 using interfaces::BlockTemplate;
 using interfaces::Mining;
 using node::BlockAssembler;
+using node::GetMinimumTime;
 using node::NodeContext;
 using node::RegenerateCommitments;
 using node::UpdateTime;
@@ -674,7 +675,7 @@ static RPCHelpMan getblocktemplate()
                 {RPCResult::Type::NUM, "coinbasevalue", "maximum allowable input to coinbase transaction, including the generation award and transaction fees (in satoshis)"},
                 {RPCResult::Type::STR, "longpollid", "an id to include with a request to longpoll on an update to this template"},
                 {RPCResult::Type::STR, "target", "The hash target"},
-                {RPCResult::Type::NUM_TIME, "mintime", "The minimum timestamp appropriate for the next block time, expressed in " + UNIX_EPOCH_TIME},
+                {RPCResult::Type::NUM_TIME, "mintime", "The minimum timestamp appropriate for the next block time, expressed in " + UNIX_EPOCH_TIME + ". Adjusted for the proposed BIP94 timewarp rule."},
                 {RPCResult::Type::ARR, "mutable", "list of ways the block template may be changed",
                 {
                     {RPCResult::Type::STR, "value", "A way the block template may be changed, e.g. 'time', 'transactions', 'prevblock'"},
@@ -683,7 +684,7 @@ static RPCHelpMan getblocktemplate()
                 {RPCResult::Type::NUM, "sigoplimit", "limit of sigops in blocks"},
                 {RPCResult::Type::NUM, "sizelimit", "limit of block size"},
                 {RPCResult::Type::NUM, "weightlimit", /*optional=*/true, "limit of block weight"},
-                {RPCResult::Type::NUM_TIME, "curtime", "current timestamp in " + UNIX_EPOCH_TIME},
+                {RPCResult::Type::NUM_TIME, "curtime", "current timestamp in " + UNIX_EPOCH_TIME + ". Adjusted for the proposed BIP94 timewarp rule."},
                 {RPCResult::Type::STR, "bits", "compressed target of next block"},
                 {RPCResult::Type::NUM, "height", "The height of the next block"},
                 {RPCResult::Type::STR_HEX, "signet_challenge", /*optional=*/true, "Only on signet"},
@@ -977,7 +978,7 @@ static RPCHelpMan getblocktemplate()
     result.pushKV("coinbasevalue", (int64_t)block.vtx[0]->vout[0].nValue);
     result.pushKV("longpollid", tip.GetHex() + ToString(nTransactionsUpdatedLast));
     result.pushKV("target", hashTarget.GetHex());
-    result.pushKV("mintime", (int64_t)pindexPrev->GetMedianTimePast()+1);
+    result.pushKV("mintime", GetMinimumTime(pindexPrev, consensusParams.DifficultyAdjustmentInterval()));
     result.pushKV("mutable", std::move(aMutable));
     result.pushKV("noncerange", "00000000ffffffff");
     int64_t nSigOpLimit = MAX_BLOCK_SIGOPS_COST;

src/rpc/net.cpp

@@ -546,6 +546,178 @@ static RPCHelpMan getaddednodeinfo()
     };
 }
 
+namespace net_stats {
+
+namespace aggregate_dimensions {
+static std::string DIRECTION{"direction"};
+static std::string NETWORK{"network"};
+static std::string CONNECTION_TYPE{"connection_type"};
+static std::string MESSAGE_TYPE{"message_type"};
+};
+
+UniValue CreateJSON(const RPCHelpMan&, const JSONRPCRequest& request)
+{
+    const NodeContext& node = EnsureAnyNodeContext(request.context);
+    const CConnman& connman = EnsureConnman(node);
+
+    // Used for a quick check if a string is in request.params[0] which is
+    // supposed to be a JSON array, e.g. ["direction", "network"].
+    std::unordered_set<std::string> aggregate_by;
+    if (request.params[0].isArray()) {
+        const UniValue& arr{request.params[0].get_array()};
+        for (size_t i = 0; i < arr.size(); ++i) {
+            const auto& agg{arr[i].get_str()};
+            if (agg != aggregate_dimensions::DIRECTION &&
+                agg != aggregate_dimensions::NETWORK &&
+                agg != aggregate_dimensions::CONNECTION_TYPE &&
+                agg != aggregate_dimensions::MESSAGE_TYPE) {
+                throw JSONRPCError(
+                    RPC_INVALID_PARAMS,
+                    strprintf(
+                        R"(Unrecognized aggregation parameter: "%s". The array should consist of zero or more of "%s", "%s", "%s", "%s".)",
+                        agg,
+                        aggregate_dimensions::DIRECTION,
+                        aggregate_dimensions::NETWORK,
+                        aggregate_dimensions::CONNECTION_TYPE,
+                        aggregate_dimensions::MESSAGE_TYPE));
+            }
+            aggregate_by.insert(agg);
+        }
+    }
+
+    // The keys might as well be an empty string (if aggregating by that dimension).
+    std::unordered_map<
+        std::string, // "sent" or "recv"
+        std::unordered_map<std::string, // "ipv4", "tor", ...
+                           std::unordered_map<std::string, // "outbound-full-relay", "feeler", ...
+                                              std::unordered_map<std::string, // "verack", "ping", ...
+                                                                 NetStats::BytesAndCount>>>>
+        result_map;
+
+    connman.GetNetStats().ForEach([&aggregate_by, &result_map](NetStats::Direction dir,
+                                                               Network net,
+                                                               ConnectionType con,
+                                                               const std::string& msg,
+                                                               const NetStats::BytesAndCount& data) {
+        const std::string dir_str{aggregate_by.contains(aggregate_dimensions::DIRECTION) ? "" : dir == NetStats::SENT ? "sent" : "recv"};
+        const std::string net_str{aggregate_by.contains(aggregate_dimensions::NETWORK) ? "" : GetNetworkName(net)};
+        const std::string con_str{aggregate_by.contains(aggregate_dimensions::CONNECTION_TYPE) ? "" : ConnectionTypeAsString(con)};
+        const std::string msg_str{aggregate_by.contains(aggregate_dimensions::MESSAGE_TYPE) ? "" : msg};
+
+        result_map[dir_str][net_str][con_str][msg_str] += data;
+    });
+
+    auto Add = [](UniValue& target, const std::string& key, const UniValue& val) {
+        if (val.empty()) {
+            return;
+        }
+        if (key.empty()) {
+            target = val;
+            return;
+        }
+        target.pushKV(key, val);
+    };
+
+    UniValue dir_json{UniValue::VOBJ};
+    for (const auto& [dir_key, dir_val] : result_map) {
+        UniValue net_json{UniValue::VOBJ};
+        for (const auto& [net_key, net_val] : dir_val) {
+            UniValue con_json{UniValue::VOBJ};
+            for (const auto& [con_key, con_val] : net_val) {
+                UniValue msg_json{UniValue::VOBJ};
+                for (const auto& [msg_key, stats] : con_val) {
+                    const auto bytes = stats.bytes.load();
+                    const auto count = stats.count.load();
+                    if (bytes == 0 || count == 0) {
+                        continue;
+                    }
+                    UniValue bytes_and_count_json{UniValue::VOBJ};
+                    bytes_and_count_json.pushKV("bytes", bytes);
+                    bytes_and_count_json.pushKV("count", count);
+
+                    Add(msg_json, msg_key, bytes_and_count_json);
+                }
+                Add(con_json, con_key, msg_json);
+            }
+            Add(net_json, net_key, con_json);
+        }
+        Add(dir_json, dir_key, net_json);
+    }
+
+    return dir_json;
+}
+}; // namespace net_stats
+
+static RPCHelpMan getnetmsgstats()
+{
+    return RPCHelpMan{
+        "getnetmsgstats",
+        "\nReturns the messages count and total number of bytes for network traffic.\n"
+        "Results may optionally be aggregated.\n",
+        {RPCArg{
+            "aggregate_by",
+            RPCArg::Type::ARR,
+            RPCArg::DefaultHint{"empty, no aggregation"},
+            "An array of keywords for aggregating the results.",
+            {RPCArg{"direction",
+                    RPCArg::Type::STR,
+                    RPCArg::Optional::OMITTED,
+                    "Aggregate by direction and don't show direction in the result."},
+             RPCArg{"network",
+                    RPCArg::Type::STR,
+                    RPCArg::Optional::OMITTED,
+                    "Aggregate by network and don't show network in the result."},
+             RPCArg{"connection_type",
+                    RPCArg::Type::STR,
+                    RPCArg::Optional::OMITTED,
+                    "Aggregate by connection type and don't show connection type in the result."},
+             RPCArg{"message_type",
+                    RPCArg::Type::STR,
+                    RPCArg::Optional::OMITTED,
+                    "Aggregate by message type and don't show message type in the result."}}}},
+        {RPCResult{
+            RPCResult::Type::OBJ_DYN,
+            "",
+            false,
+            "When a direction, network,\n"
+            "connection type or message type is not listed,\n"
+            "the statistics for that are 0.",
+            {RPCResult{
+                RPCResult::Type::OBJ_DYN,
+                "sent",
+                true,
+                "The direction in which the traffic occurred.",
+                {RPCResult{
+                    RPCResult::Type::OBJ_DYN,
+                    "ipv4",
+                    true,
+                    "The network over which the traffic occurred.",
+                    {RPCResult{
+                        RPCResult::Type::OBJ_DYN,
+                        "inbound",
+                        true,
+                        "The connection type over which the traffic occurred.",
+                        {RPCResult{
+                            RPCResult::Type::OBJ,
+                            "verack",
+                            true,
+                            "Type of the messages transferred.",
+                            {
+                                RPCResult{RPCResult::Type::NUM, "bytes", false, "Total number of bytes.", {}, true},
+                                RPCResult{RPCResult::Type::NUM, "count", false, "Total number of messages.", {}, true}
+                            },
+                            true}},
+                        true}},
+                    true}},
+                true}},
+            true}},
+        RPCExamples{HelpExampleCli("getnetmsgstats", "") +
+                    HelpExampleCli("getnetmsgstats", R"('["network", "message_type"]')") +
+                    HelpExampleRpc("getnetmsgstats", "") +
+                    HelpExampleRpc("getnetmsgstats", R"(["network", "message_type"])")},
+        net_stats::CreateJSON};
+}
+
 static RPCHelpMan getnettotals()
 {
     return RPCHelpMan{"getnettotals",
@@ -1188,6 +1360,7 @@ void RegisterNetRPCCommands(CRPCTable& t)
         {"network", &addnode},
         {"network", &disconnectnode},
         {"network", &getaddednodeinfo},
+        {"network", &getnetmsgstats},
         {"network", &getnettotals},
         {"network", &getnetworkinfo},
         {"network", &setban},

src/test/blockmanager_tests.cpp

@@ -33,6 +33,10 @@ BOOST_AUTO_TEST_CASE(blockmanager_find_block_pos)
         .chainparams = *params,
         .blocks_dir = m_args.GetBlocksDirPath(),
         .notifications = notifications,
+        .block_tree_db_params = DBParams{
+            .path = m_args.GetDataDirNet() / "blocks" / "index",
+            .cache_bytes = 0,
+        },
     };
     BlockManager blockman{*Assert(m_node.shutdown_signal), blockman_opts};
     // simulate adding a genesis block normally
@@ -140,6 +144,10 @@ BOOST_AUTO_TEST_CASE(blockmanager_flush_block_file)
         .chainparams = Params(),
         .blocks_dir = m_args.GetBlocksDirPath(),
         .notifications = notifications,
+        .block_tree_db_params = DBParams{
+            .path = m_args.GetDataDirNet() / "blocks" / "index",
+            .cache_bytes = 0,
+        },
     };
     BlockManager blockman{*Assert(m_node.shutdown_signal), blockman_opts};
 

src/test/fuzz/net.cpp

@@ -39,6 +39,7 @@ FUZZ_TARGET(net, .init = initialize_net)
     {
         node.SetAddrLocal(*service_opt);
     }
+    NetStats net_stats;
     LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) {
         CallOneOf(
             fuzzed_data_provider,
@@ -61,7 +62,7 @@ FUZZ_TARGET(net, .init = initialize_net)
             [&] {
                 const std::vector<uint8_t> b = ConsumeRandomLengthByteVector(fuzzed_data_provider);
                 bool complete;
-                node.ReceiveMsgBytes(b, complete);
+                node.ReceiveMsgBytes(b, complete, net_stats);
             });
     }
 

src/test/fuzz/rpc.cpp

@@ -140,6 +140,7 @@ const std::vector<std::string> RPC_COMMANDS_SAFE_FOR_FUZZING{
     "getmempoolentry",
     "getmempoolinfo",
     "getmininginfo",
+    "getnetmsgstats",
     "getnettotals",
     "getnetworkhashps",
     "getnetworkinfo",

src/test/util/net.cpp

@@ -68,9 +68,9 @@ void ConnmanTestMsg::Handshake(CNode& node,
     }
 }
 
-void ConnmanTestMsg::NodeReceiveMsgBytes(CNode& node, Span<const uint8_t> msg_bytes, bool& complete) const
+void ConnmanTestMsg::NodeReceiveMsgBytes(CNode& node, Span<const uint8_t> msg_bytes, bool& complete)
 {
-    assert(node.ReceiveMsgBytes(msg_bytes, complete));
+    assert(node.ReceiveMsgBytes(msg_bytes, complete, m_net_stats));
     if (complete) {
         node.MarkReceivedMsgsForProcessing();
     }
@@ -88,7 +88,7 @@ void ConnmanTestMsg::FlushSendBuffer(CNode& node) const
     }
 }
 
-bool ConnmanTestMsg::ReceiveMsgFrom(CNode& node, CSerializedNetMsg&& ser_msg) const
+bool ConnmanTestMsg::ReceiveMsgFrom(CNode& node, CSerializedNetMsg&& ser_msg)
 {
     bool queued = node.m_transport->SetMessageToSend(ser_msg);
     assert(queued);

src/test/util/net.h

@@ -78,9 +78,9 @@ struct ConnmanTestMsg : public CConnman {
         return m_msgproc->ProcessMessages(&node, flagInterruptMsgProc);
     }
 
-    void NodeReceiveMsgBytes(CNode& node, Span<const uint8_t> msg_bytes, bool& complete) const;
+    void NodeReceiveMsgBytes(CNode& node, Span<const uint8_t> msg_bytes, bool& complete);
 
-    bool ReceiveMsgFrom(CNode& node, CSerializedNetMsg&& ser_msg) const;
+    bool ReceiveMsgFrom(CNode& node, CSerializedNetMsg&& ser_msg);
     void FlushSendBuffer(CNode& node) const;
 
     bool AlreadyConnectedPublic(const CAddress& addr) { return AlreadyConnectedToAddress(addr); };

src/test/util/setup_common.cpp

@@ -62,7 +62,6 @@
 #include <stdexcept>
 
 using namespace util::hex_literals;
-using kernel::BlockTreeDB;
 using node::ApplyArgsManOptions;
 using node::BlockAssembler;
 using node::BlockManager;
@@ -252,14 +251,14 @@ ChainTestingSetup::ChainTestingSetup(const ChainType chainType, TestOpts opts)
             .chainparams = chainman_opts.chainparams,
             .blocks_dir = m_args.GetBlocksDirPath(),
             .notifications = chainman_opts.notifications,
+            .block_tree_db_params = DBParams{
+                .path = m_args.GetDataDirNet() / "blocks" / "index",
+                .cache_bytes = m_kernel_cache_sizes.block_tree_db,
+                .memory_only = opts.block_tree_db_in_memory,
+                .wipe_data = m_args.GetBoolArg("-reindex", false),
+            },
         };
         m_node.chainman = std::make_unique<ChainstateManager>(*Assert(m_node.shutdown_signal), chainman_opts, blockman_opts);
-        LOCK(m_node.chainman->GetMutex());
-        m_node.chainman->m_blockman.m_block_tree_db = std::make_unique<BlockTreeDB>(DBParams{
-            .path = m_args.GetDataDirNet() / "blocks" / "index",
-            .cache_bytes = m_kernel_cache_sizes.block_tree_db,
-            .memory_only = true,
-        });
     };
     m_make_chainman();
 }
@@ -285,9 +284,7 @@ void ChainTestingSetup::LoadVerifyActivateChainstate()
     auto& chainman{*Assert(m_node.chainman)};
     node::ChainstateLoadOptions options;
     options.mempool = Assert(m_node.mempool.get());
-    options.block_tree_db_in_memory = m_block_tree_db_in_memory;
     options.coins_db_in_memory = m_coins_db_in_memory;
-    options.wipe_block_tree_db = m_args.GetBoolArg("-reindex", false);
     options.wipe_chainstate_db = m_args.GetBoolArg("-reindex", false) || m_args.GetBoolArg("-reindex-chainstate", false);
     options.prune = chainman.m_blockman.IsPruneMode();
     options.check_blocks = m_args.GetIntArg("-checkblocks", DEFAULT_CHECKBLOCKS);

src/test/validation_chainstatemanager_tests.cpp

@@ -393,6 +393,11 @@ struct SnapshotTestSetup : TestChain100Setup {
                 .chainparams = chainman_opts.chainparams,
                 .blocks_dir = m_args.GetBlocksDirPath(),
                 .notifications = chainman_opts.notifications,
+                .block_tree_db_params = DBParams{
+                    .path = chainman.m_options.datadir / "blocks" / "index",
+                    .cache_bytes = m_kernel_cache_sizes.block_tree_db,
+                    .memory_only = m_block_tree_db_in_memory,
+                },
             };
             // For robustness, ensure the old manager is destroyed before creating a
             // new one.

src/validation.cpp

@@ -5623,9 +5623,8 @@ double ChainstateManager::GuessVerificationProgress(const CBlockIndex* pindex) c
         return 0.0;
     }
 
-    if (!Assume(pindex->m_chain_tx_count > 0)) {
-        LogWarning("Internal bug detected: block %d has unset m_chain_tx_count (%s %s). Please report this issue here: %s\n",
-                   pindex->nHeight, CLIENT_NAME, FormatFullVersion(), CLIENT_BUGREPORT);
+    if (pindex->m_chain_tx_count == 0) {
+        LogDebug(BCLog::VALIDATION, "Block %d has unset m_chain_tx_count. Unable to estimate verification progress.\n", pindex->nHeight);
         return 0.0;
     }
 

src/wallet/rpc/backup.cpp

@@ -1745,20 +1745,27 @@ RPCHelpMan importdescriptors()
                 if (scanned_time <= GetImportTimestamp(request, now) || results.at(i).exists("error")) {
                     response.push_back(results.at(i));
                 } else {
+                    std::string error_msg{strprintf("Rescan failed for descriptor with timestamp %d. There "
+                            "was an error reading a block from time %d, which is after or within %d seconds "
+                            "of key creation, and could contain transactions pertaining to the desc. As a "
+                            "result, transactions and coins using this desc may not appear in the wallet.",
+                            GetImportTimestamp(request, now), scanned_time - TIMESTAMP_WINDOW - 1, TIMESTAMP_WINDOW)};
+                    if (pwallet->chain().havePruned()) {
+                        error_msg += strprintf(" This error could be caused by pruning or data corruption "
+                                "(see bitcoind log for details) and could be dealt with by downloading and "
+                                "rescanning the relevant blocks (see -reindex option and rescanblockchain RPC).");
+                    } else if (pwallet->chain().hasAssumedValidChain()) {
+                        error_msg += strprintf(" This error is likely caused by an in-progress assumeutxo "
+                                "background sync. Check logs or getchainstates RPC for assumeutxo background "
+                                "sync progress and try again later.");
+                    } else {
+                        error_msg += strprintf(" This error could potentially caused by data corruption. If "
+                                "the issue persists you may want to reindex (see -reindex option).");
+                    }
+
                     UniValue result = UniValue(UniValue::VOBJ);
                     result.pushKV("success", UniValue(false));
-                    result.pushKV(
-                        "error",
-                        JSONRPCError(
-                            RPC_MISC_ERROR,
-                            strprintf("Rescan failed for descriptor with timestamp %d. There was an error reading a "
-                                      "block from time %d, which is after or within %d seconds of key creation, and "
-                                      "could contain transactions pertaining to the desc. As a result, transactions "
-                                      "and coins using this desc may not appear in the wallet. This error could be "
-                                      "caused by pruning or data corruption (see bitcoind log for details) and could "
-                                      "be dealt with by downloading and rescanning the relevant blocks (see -reindex "
-                                      "option and rescanblockchain RPC).",
-                                GetImportTimestamp(request, now), scanned_time - TIMESTAMP_WINDOW - 1, TIMESTAMP_WINDOW)));
+                    result.pushKV("error", JSONRPCError(RPC_MISC_ERROR, error_msg));
                     response.push_back(std::move(result));
                 }
             }

src/wallet/rpc/transactions.cpp

@@ -6,6 +6,7 @@
 #include <key_io.h>
 #include <policy/rbf.h>
 #include <rpc/util.h>
+#include <rpc/blockchain.h>
 #include <util/vector.h>
 #include <wallet/receive.h>
 #include <wallet/rpc/util.h>
@@ -909,9 +910,15 @@ RPCHelpMan rescanblockchain()
             }
         }
 
-        // We can't rescan beyond non-pruned blocks, stop and throw an error
+        // We can't rescan unavailable blocks, stop and throw an error
         if (!pwallet->chain().hasBlocks(pwallet->GetLastBlockHash(), start_height, stop_height)) {
-            throw JSONRPCError(RPC_MISC_ERROR, "Can't rescan beyond pruned data. Use RPC call getblockchaininfo to determine your pruned height.");
+            if (pwallet->chain().havePruned() && pwallet->chain().getPruneHeight() >= start_height) {
+                throw JSONRPCError(RPC_MISC_ERROR, "Can't rescan beyond pruned data. Use RPC call getblockchaininfo to determine your pruned height.");
+            }
+            if (pwallet->chain().hasAssumedValidChain()) {
+                throw JSONRPCError(RPC_MISC_ERROR, "Failed to rescan unavailable blocks likely due to an in-progress assumeutxo background sync. Check logs or getchainstates RPC for assumeutxo background sync progress and try again later.");
+            }
+            throw JSONRPCError(RPC_MISC_ERROR, "Failed to rescan unavailable blocks, potentially caused by data corruption. If the issue persists you may want to reindex (see -reindex option).");
         }
 
         CHECK_NONFATAL(pwallet->chain().findAncestorByHeight(pwallet->GetLastBlockHash(), start_height, FoundBlock().hash(start_block)));

test/functional/mining_basic.py

@@ -153,6 +153,8 @@ class MiningTest(BitcoinTestFramework):
         # The template will have an adjusted timestamp, which we then modify
         tmpl = node.getblocktemplate(NORMAL_GBT_REQUEST_PARAMS)
         assert_greater_than_or_equal(tmpl['curtime'], t + MAX_FUTURE_BLOCK_TIME - MAX_TIMEWARP)
+        # mintime and curtime should match
+        assert_equal(tmpl['mintime'], tmpl['curtime'])
 
         block = CBlock()
         block.nVersion = tmpl["version"]

test/functional/rpc_blockchain.py

@@ -549,6 +549,7 @@ class BlockchainTest(BitcoinTestFramework):
         # The chain has probably already been restored by the time reconsiderblock returns,
         # but poll anyway.
         self.wait_until(lambda: node.waitfornewblock(timeout=100)['hash'] == current_hash)
+        assert_raises_rpc_error(-1, "Negative timeout", node.waitfornewblock, -1)
 
     def _test_waitforblockheight(self):
         self.log.info("Test waitforblockheight")

test/functional/rpc_net.py

@@ -10,6 +10,7 @@ Tests correspond to code in rpc/net.cpp.
 from decimal import Decimal
 from itertools import product
 import platform
+import random
 import time
 
 import test_framework.messages
@@ -90,6 +91,7 @@ class NetTest(BitcoinTestFramework):
         self.test_sendmsgtopeer()
         self.test_getaddrmaninfo()
         self.test_getrawaddrman()
+        self.test_getnetmsgstats()
 
     def test_connection_count(self):
         self.log.info("Test getconnectioncount")
@@ -202,6 +204,177 @@ class NetTest(BitcoinTestFramework):
             self.wait_until(lambda: peer_after()['bytesrecv_per_msg'].get('pong', 0) >= peer_before['bytesrecv_per_msg'].get('pong', 0) + ping_size, timeout=1)
             self.wait_until(lambda: peer_after()['bytessent_per_msg'].get('ping', 0) >= peer_before['bytessent_per_msg'].get('ping', 0) + ping_size, timeout=1)
 
+    def test_getnetmsgstats(self):
+        self.log.info("Test getnetmsgstats")
+
+        self.restart_node(0)
+        node0 = self.nodes[0]
+        self.connect_nodes(0, 1) # Generate some traffic.
+        # Wait for the initial messages to be sent/received (don't disconnect too early). "sendheaders" is the last one.
+        self.wait_until(lambda: "sendheaders" in node0.getnetmsgstats()["recv"]["not_publicly_routable"]["manual"])
+        self.wait_until(lambda: "sendheaders" in node0.getnetmsgstats()["sent"]["not_publicly_routable"]["manual"])
+        self.disconnect_nodes(0, 1) # Avoid random/unpredictable packets (e.g. ping) messing with the tests below.
+        assert_equal(len(node0.getpeerinfo()), 0)
+
+        # In v2 getnettotals counts also bytes that are not accounted at any message (the v2 handshake).
+        # Also the v2 handshake's size could vary.
+        if not self.options.v2transport:
+            self.log.debug("Compare byte count getnetmsgstats vs getnettotals")
+            nettotals = self.nodes[0].getnettotals()
+            stats_net_con_msg = self.nodes[0].getnetmsgstats(aggregate_by=["network", "connection_type", "message_type"])
+            assert_equal(nettotals["totalbytessent"], stats_net_con_msg["sent"]["bytes"])
+            assert_equal(nettotals["totalbytesrecv"], stats_net_con_msg["recv"]["bytes"])
+
+        self.log.debug("Test full (un-aggregated) output is as expected")
+        stats_full = node0.getnetmsgstats()
+        if self.options.v2transport:
+            assert_equal(
+                stats_full,
+                {
+                    "recv": {
+                        "not_publicly_routable": {
+                            "manual": {
+                                "addrv2": {"bytes": 63, "count": 1},
+                                "feefilter": {"bytes": 29, "count": 1},
+                                "getheaders": {"bytes": 666, "count": 1},
+                                "headers": {"bytes": 103, "count": 1},
+                                "ping": {"bytes": 29, "count": 1},
+                                "pong": {"bytes": 29, "count": 1},
+                                "sendaddrv2": {"bytes": 33, "count": 1},
+                                "sendcmpct": {"bytes": 30, "count": 1},
+                                "sendheaders": {"bytes": 33, "count": 1},
+                                "verack": {"bytes": 33, "count": 1},
+                                "version": {"bytes": 147, "count": 1},
+                                "wtxidrelay": {"bytes": 33, "count": 1},
+                            }
+                        }
+                    },
+                    "sent": {
+                        "not_publicly_routable": {
+                            "manual": {
+                                "feefilter": {"bytes": 29, "count": 1},
+                                "getaddr": {"bytes": 33, "count": 1},
+                                "getheaders": {"bytes": 666, "count": 1},
+                                "headers": {"bytes": 103, "count": 1},
+                                "ping": {"bytes": 29, "count": 1},
+                                "pong": {"bytes": 29, "count": 1},
+                                "sendaddrv2": {"bytes": 33, "count": 1},
+                                "sendcmpct": {"bytes": 30, "count": 1},
+                                "sendheaders": {"bytes": 33, "count": 1},
+                                "verack": {"bytes": 33, "count": 1},
+                                "version": {"bytes": 147, "count": 1},
+                                "wtxidrelay": {"bytes": 33, "count": 1},
+                            }
+                        }
+                    }
+                }
+            )
+        else:
+            assert_equal(
+                stats_full,
+                {
+                    "recv": {
+                        "not_publicly_routable": {
+                            "manual": {
+                                "addrv2": {"bytes": 66, "count": 1},
+                                "feefilter": {"bytes": 32, "count": 1},
+                                "getheaders": {"bytes": 669, "count": 1},
+                                "headers": {"bytes": 106, "count": 1},
+                                "ping": {"bytes": 32, "count": 1},
+                                "pong": {"bytes": 32, "count": 1},
+                                "sendaddrv2": {"bytes": 24, "count": 1},
+                                "sendcmpct": {"bytes": 33, "count": 1},
+                                "sendheaders": {"bytes": 24, "count": 1},
+                                "verack": {"bytes": 24, "count": 1},
+                                "version": {"bytes": 138, "count": 1},
+                                "wtxidrelay": {"bytes": 24, "count": 1},
+                            }
+                        }
+                    },
+                    "sent": {
+                        "not_publicly_routable": {
+                            "manual": {
+                                "feefilter": {"bytes": 32, "count": 1},
+                                "getaddr": {"bytes": 24, "count": 1},
+                                "getheaders": {"bytes": 669, "count": 1},
+                                "headers": {"bytes": 106, "count": 1},
+                                "ping": {"bytes": 32, "count": 1},
+                                "pong": {"bytes": 32, "count": 1},
+                                "sendaddrv2": {"bytes": 24, "count": 1},
+                                "sendcmpct": {"bytes": 33, "count": 1},
+                                "sendheaders": {"bytes": 24, "count": 1},
+                                "verack": {"bytes": 24, "count": 1},
+                                "version": {"bytes": 138, "count": 1},
+                                "wtxidrelay": {"bytes": 24, "count": 1},
+                            }
+                        }
+                    }
+                }
+            )
+
+        self.log.debug("Check that aggregation works correctly")
+
+        def sum_all(json):
+            if "bytes" in json:
+                return dict(bytes=json["bytes"], count=json["count"])
+
+            s = dict(bytes=0, count=0)
+            #print(f'S json={json}')
+            for k, v in json.items():
+                #print(f'S k={k}, v={v}')
+                #print(f'S diving into {v}')
+                sub = sum_all(v)
+                s["bytes"] += sub["bytes"]
+                s["count"] += sub["count"]
+            return s
+
+        stats_aggregated_by_bitcoind = node0.getnetmsgstats(aggregate_by=["direction", "network", "connection_type", "message_type"])
+        stats_aggregated_by_test = sum_all(stats_full)
+        assert_equal(stats_aggregated_by_bitcoind, stats_aggregated_by_test)
+        if not self.options.v2transport:
+            assert_equal(nettotals["totalbytessent"] + nettotals["totalbytesrecv"], stats_aggregated_by_test["bytes"])
+
+        for i in range(1, 16):
+            keywords = []
+            if i & 1:
+                keywords.append("direction")
+            if i & 2:
+                keywords.append("network")
+            if i & 4:
+                keywords.append("connection_type")
+            if i & 8:
+                keywords.append("message_type")
+            random.shuffle(keywords)
+            self.log.debug(f"Test values add up correctly when aggregated by {keywords}")
+            assert_equal(stats_aggregated_by_test, sum_all(node0.getnetmsgstats(aggregate_by=keywords)))
+
+        def get_stats(node):
+            return node.getnetmsgstats(aggregate_by=["network", "connection_type"])
+
+        self.log.debug("Test that message count and total number of bytes increment when a ping message is sent")
+        stats_before_connect = get_stats(node0)
+        node2 = node0.add_p2p_connection(P2PInterface())
+        assert_equal(len(node0.getpeerinfo()), 1)
+        # Wait for the initial PING (that is sent immediately after the connection is estabilshed) to go through.
+        self.wait_until(lambda: get_stats(node0)["sent"]["ping"]["count"] > stats_before_connect["sent"]["ping"]["count"])
+        stats_before_ping = get_stats(node0)
+        node0.ping()
+        self.wait_until(lambda: get_stats(node0)["sent"]["ping"]["count"] > stats_before_ping["sent"]["ping"]["count"])
+        self.wait_until(lambda: get_stats(node0)["sent"]["ping"]["bytes"] > stats_before_ping["sent"]["ping"]["bytes"])
+
+        self.log.debug("Test that when a message is broken in two, the stats only update once the full message has been received")
+        ping_msg = node2.build_message(test_framework.messages.msg_ping(nonce=12345))
+        stats_before_ping = get_stats(node0)
+        # Send the message in two pieces.
+        cut_pos = 7 # Chosen at an arbitrary position within the header.
+        node2.send_raw_message(ping_msg[:cut_pos])
+        assert_equal(get_stats(node0)["recv"]["ping"], stats_before_ping["recv"]["ping"])
+        # Send the rest of the ping.
+        node2.send_raw_message(ping_msg[cut_pos:])
+        self.wait_until(lambda: get_stats(node0)["recv"]["ping"]["count"] == stats_before_ping["recv"]["ping"]["count"] + 1)
+
+        node2.peer_disconnect()
+
     def test_getnetworkinfo(self):
         self.log.info("Test getnetworkinfo")
         info = self.nodes[0].getnetworkinfo()

test/functional/wallet_assumeutxo.py

@@ -7,11 +7,11 @@ See feature_assumeutxo.py for background.
 
 ## Possible test improvements
 
-- TODO: test import descriptors while background sync is in progress
 - TODO: test loading a wallet (backup) on a pruned node
 
 """
 from test_framework.address import address_to_scriptpubkey
+from test_framework.descriptors import descsum_create
 from test_framework.test_framework import BitcoinTestFramework
 from test_framework.messages import COIN
 from test_framework.util import (
@@ -20,6 +20,7 @@ from test_framework.util import (
     ensure_for,
 )
 from test_framework.wallet import MiniWallet
+from test_framework.wallet_util import get_generate_key
 
 START_HEIGHT = 199
 SNAPSHOT_BASE_HEIGHT = 299
@@ -49,6 +50,13 @@ class AssumeutxoTest(BitcoinTestFramework):
         self.add_nodes(3)
         self.start_nodes(extra_args=self.extra_args)
 
+    def import_descriptor(self, node, wallet_name, key, timestamp):
+        import_request = [{"desc": descsum_create("pkh(" + key.pubkey + ")"),
+                           "timestamp": timestamp,
+                           "label": "Descriptor import test"}]
+        wrpc = node.get_wallet_rpc(wallet_name)
+        return wrpc.importdescriptors(import_request)
+
     def run_test(self):
         """
         Bring up two (disconnected) nodes, mine some new blocks on the first,
@@ -157,6 +165,21 @@ class AssumeutxoTest(BitcoinTestFramework):
         self.log.info("Backup from before the snapshot height can't be loaded during background sync")
         assert_raises_rpc_error(-4, "Wallet loading failed. Error loading wallet. Wallet requires blocks to be downloaded, and software does not currently support loading wallets while blocks are being downloaded out of order when using assumeutxo snapshots. Wallet should be able to load successfully after node sync reaches height 299", n1.restorewallet, "w2", "backup_w2.dat")
 
+        self.log.info("Test loading descriptors during background sync")
+        wallet_name = "w1"
+        n1.createwallet(wallet_name, disable_private_keys=True)
+        key = get_generate_key()
+        time = n1.getblockchaininfo()['time']
+        timestamp = 0
+        expected_error_message = f"Rescan failed for descriptor with timestamp {timestamp}. There was an error reading a block from time {time}, which is after or within 7200 seconds of key creation, and could contain transactions pertaining to the desc. As a result, transactions and coins using this desc may not appear in the wallet. This error is likely caused by an in-progress assumeutxo background sync. Check logs or getchainstates RPC for assumeutxo background sync progress and try again later."
+        result = self.import_descriptor(n1, wallet_name, key, timestamp)
+        assert_equal(result[0]['error']['code'], -1)
+        assert_equal(result[0]['error']['message'], expected_error_message)
+
+        self.log.info("Test that rescanning blocks from before the snapshot fails when blocks are not available from the background sync yet")
+        w1 = n1.get_wallet_rpc(wallet_name)
+        assert_raises_rpc_error(-1, "Failed to rescan unavailable blocks likely due to an in-progress assumeutxo background sync. Check logs or getchainstates RPC for assumeutxo background sync progress and try again later.", w1.rescanblockchain, 100)
+
         PAUSE_HEIGHT = FINAL_HEIGHT - 40
 
         self.log.info("Restarting node to stop at height %d", PAUSE_HEIGHT)
@@ -204,6 +227,11 @@ class AssumeutxoTest(BitcoinTestFramework):
         self.wait_until(lambda: len(n2.getchainstates()['chainstates']) == 1)
         ensure_for(duration=1, f=lambda: (n2.getbalance() == 34))
 
+        self.log.info("Ensuring descriptors can be loaded after background sync")
+        n1.loadwallet(wallet_name)
+        result = self.import_descriptor(n1, wallet_name, key, timestamp)
+        assert_equal(result[0]['success'], True)
+
 
 if __name__ == '__main__':
     AssumeutxoTest(__file__).main()