From ce50acc54fa313a92d48ed03e46ce8aabcf267e5 Mon Sep 17 00:00:00 2001 From: furszy Date: Thu, 15 Sep 2022 17:38:26 -0300 Subject: [PATCH 1/2] rpc: scanblocks, do not traverse the whole chain block by block The current flow walks-through every block in the active chain until hits the chain tip or processes 10k blocks, then calls `lookupFilterRange()` to obtain all the filters from that particular range. This is only done to obtain the heights range to look up the block filters. Which is unneeded. As `scanblocks` only lookup block filters in the active chain, we can directly calculate the lookup range heights, by using the chain tip, without requiring to traverse the chain block by block. --- src/rpc/blockchain.cpp | 98 +++++++++++++++++++++--------------------- 1 file changed, 49 insertions(+), 49 deletions(-) diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 1b2543c77a..301cfbfb25 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -2383,27 +2383,28 @@ static RPCHelpMan scanblocks() ChainstateManager& chainman = EnsureChainman(node); // set the start-height - const CBlockIndex* block = nullptr; + const CBlockIndex* start_index = nullptr; const CBlockIndex* stop_block = nullptr; { LOCK(cs_main); CChain& active_chain = chainman.ActiveChain(); - block = active_chain.Genesis(); - stop_block = active_chain.Tip(); + start_index = active_chain.Genesis(); + stop_block = active_chain.Tip(); // If no stop block is provided, stop at the chain tip. if (!request.params[2].isNull()) { - block = active_chain[request.params[2].getInt()]; - if (!block) { + start_index = active_chain[request.params[2].getInt()]; + if (!start_index) { throw JSONRPCError(RPC_MISC_ERROR, "Invalid start_height"); } } if (!request.params[3].isNull()) { stop_block = active_chain[request.params[3].getInt()]; - if (!stop_block || stop_block->nHeight < block->nHeight) { + if (!stop_block || stop_block->nHeight < start_index->nHeight) { throw JSONRPCError(RPC_MISC_ERROR, "Invalid stop_height"); } } } - CHECK_NONFATAL(block); + CHECK_NONFATAL(start_index); + CHECK_NONFATAL(stop_block); // loop through the scan objects, add scripts to the needle_set GCSFilter::ElementSet needle_set; @@ -2416,63 +2417,62 @@ static RPCHelpMan scanblocks() } UniValue blocks(UniValue::VARR); const int amount_per_chunk = 10000; - const CBlockIndex* start_index = block; // for remembering the start of a blockfilter range std::vector filters; - const CBlockIndex* start_block = block; // for progress reporting - const int total_blocks_to_process = stop_block->nHeight - start_block->nHeight; + int start_block_height = start_index->nHeight; // for progress reporting + const int total_blocks_to_process = stop_block->nHeight - start_block_height; g_scanfilter_should_abort_scan = false; g_scanfilter_progress = 0; - g_scanfilter_progress_height = start_block->nHeight; + g_scanfilter_progress_height = start_block_height; - while (block) { + const CBlockIndex* end_range = nullptr; + do { node.rpc_interruption_point(); // allow a clean shutdown if (g_scanfilter_should_abort_scan) { - LogPrintf("scanblocks RPC aborted at height %d.\n", block->nHeight); + LogPrintf("scanblocks RPC aborted at height %d.\n", end_range->nHeight); break; } - const CBlockIndex* next = nullptr; - { - LOCK(cs_main); - CChain& active_chain = chainman.ActiveChain(); - next = active_chain.Next(block); - if (block == stop_block) next = nullptr; - } - if (start_index->nHeight + amount_per_chunk == block->nHeight || next == nullptr) { - LogPrint(BCLog::RPC, "Fetching blockfilters from height %d to height %d.\n", start_index->nHeight, block->nHeight); - if (index->LookupFilterRange(start_index->nHeight, block, filters)) { - for (const BlockFilter& filter : filters) { - // compare the elements-set with each filter - if (filter.GetFilter().MatchAny(needle_set)) { - if (filter_false_positives) { - // Double check the filter matches by scanning the block - const CBlockIndex& blockindex = *CHECK_NONFATAL(WITH_LOCK(cs_main, return chainman.m_blockman.LookupBlockIndex(filter.GetBlockHash()))); - if (!CheckBlockFilterMatches(chainman.m_blockman, blockindex, needle_set)) { - continue; - } + // split the lookup range in chunks if we are deeper than 'amount_per_chunk' blocks from the stopping block + int start_block = !end_range ? start_index->nHeight : start_index->nHeight + 1; // to not include the previous round 'end_range' block + end_range = (start_block + amount_per_chunk < stop_block->nHeight) ? + WITH_LOCK(::cs_main, return chainman.ActiveChain()[start_block + amount_per_chunk]) : + stop_block; + + if (index->LookupFilterRange(start_block, end_range, filters)) { + for (const BlockFilter& filter : filters) { + // compare the elements-set with each filter + if (filter.GetFilter().MatchAny(needle_set)) { + if (filter_false_positives) { + // Double check the filter matches by scanning the block + const CBlockIndex& blockindex = *CHECK_NONFATAL(WITH_LOCK(cs_main, return chainman.m_blockman.LookupBlockIndex(filter.GetBlockHash()))); + + if (!CheckBlockFilterMatches(chainman.m_blockman, blockindex, needle_set)) { + continue; } - - blocks.push_back(filter.GetBlockHash().GetHex()); - LogPrint(BCLog::RPC, "scanblocks: found match in %s\n", filter.GetBlockHash().GetHex()); } + + blocks.push_back(filter.GetBlockHash().GetHex()); + LogPrint(BCLog::RPC, "scanblocks: found match in %s\n", filter.GetBlockHash().GetHex()); } } - start_index = block; - - // update progress - int blocks_processed = block->nHeight - start_block->nHeight; - if (total_blocks_to_process > 0) { // avoid division by zero - g_scanfilter_progress = (int)(100.0 / total_blocks_to_process * blocks_processed); - } else { - g_scanfilter_progress = 100; - } - g_scanfilter_progress_height = block->nHeight; } - block = next; - } - ret.pushKV("from_height", start_block->nHeight); - ret.pushKV("to_height", g_scanfilter_progress_height.load()); + start_index = end_range; + + // update progress + int blocks_processed = end_range->nHeight - start_block_height; + if (total_blocks_to_process > 0) { // avoid division by zero + g_scanfilter_progress = (int)(100.0 / total_blocks_to_process * blocks_processed); + } else { + g_scanfilter_progress = 100; + } + g_scanfilter_progress_height = end_range->nHeight; + + // Finish if we reached the stop block + } while (start_index != stop_block); + + ret.pushKV("from_height", start_block_height); + ret.pushKV("to_height", stop_block->nHeight); ret.pushKV("relevant_blocks", blocks); } else { From b922f6b5262884f42d7483f1e9af35650bdb53a7 Mon Sep 17 00:00:00 2001 From: furszy Date: Fri, 14 Oct 2022 20:59:48 -0300 Subject: [PATCH 2/2] rpc: scanblocks, add "completed" flag to the result obj To tell the user whether the process was aborted or not. Plus, as the process can be aborted prior to the end range, have also changed the "to_height" result value to return the last scanned block instead of the end range block. --- src/rpc/blockchain.cpp | 11 ++++++----- test/functional/rpc_scanblocks.py | 1 + 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 301cfbfb25..9d8e0ceb61 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -2321,6 +2321,7 @@ static RPCHelpMan scanblocks() {RPCResult::Type::ARR, "relevant_blocks", "Blocks that may have matched a scanobject.", { {RPCResult::Type::STR_HEX, "blockhash", "A relevant blockhash"}, }}, + {RPCResult::Type::BOOL, "completed", "true if the scan process was not aborted"} }}, RPCResult{"when action=='status' and a scan is currently in progress", RPCResult::Type::OBJ, "", "", { {RPCResult::Type::NUM, "progress", "Approximate percent complete"}, @@ -2358,8 +2359,7 @@ static RPCHelpMan scanblocks() // set the abort flag g_scanfilter_should_abort_scan = true; return true; - } - else if (request.params[0].get_str() == "start") { + } else if (request.params[0].get_str() == "start") { BlockFiltersScanReserver reserver; if (!reserver.reserve()) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Scan already in progress, use action \"abort\" or \"status\""); @@ -2424,12 +2424,13 @@ static RPCHelpMan scanblocks() g_scanfilter_should_abort_scan = false; g_scanfilter_progress = 0; g_scanfilter_progress_height = start_block_height; + bool completed = true; const CBlockIndex* end_range = nullptr; do { node.rpc_interruption_point(); // allow a clean shutdown if (g_scanfilter_should_abort_scan) { - LogPrintf("scanblocks RPC aborted at height %d.\n", end_range->nHeight); + completed = false; break; } @@ -2453,7 +2454,6 @@ static RPCHelpMan scanblocks() } blocks.push_back(filter.GetBlockHash().GetHex()); - LogPrint(BCLog::RPC, "scanblocks: found match in %s\n", filter.GetBlockHash().GetHex()); } } } @@ -2472,8 +2472,9 @@ static RPCHelpMan scanblocks() } while (start_index != stop_block); ret.pushKV("from_height", start_block_height); - ret.pushKV("to_height", stop_block->nHeight); + ret.pushKV("to_height", start_index->nHeight); // start_index is always the last scanned block here ret.pushKV("relevant_blocks", blocks); + ret.pushKV("completed", completed); } else { throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Invalid action '%s'", request.params[0].get_str())); diff --git a/test/functional/rpc_scanblocks.py b/test/functional/rpc_scanblocks.py index 126e95362b..51157a3b7e 100755 --- a/test/functional/rpc_scanblocks.py +++ b/test/functional/rpc_scanblocks.py @@ -48,6 +48,7 @@ class ScanblocksTest(BitcoinTestFramework): assert blockhash in out['relevant_blocks'] assert_equal(height, out['to_height']) assert_equal(0, out['from_height']) + assert_equal(True, out['completed']) # mine another block blockhash_new = self.generate(node, 1)[0]