diff --git a/src/coins.cpp b/src/coins.cpp index 7b76c13f98..5de2ed7810 100644 --- a/src/coins.cpp +++ b/src/coins.cpp @@ -245,6 +245,14 @@ bool CCoinsViewCache::HaveInputs(const CTransaction& tx) const return true; } +void CCoinsViewCache::ReallocateCache() +{ + // Cache should be empty when we're calling this. + assert(cacheCoins.size() == 0); + cacheCoins.~CCoinsMap(); + ::new (&cacheCoins) CCoinsMap(); +} + static const size_t MIN_TRANSACTION_OUTPUT_WEIGHT = WITNESS_SCALE_FACTOR * ::GetSerializeSize(CTxOut(), PROTOCOL_VERSION); static const size_t MAX_OUTPUTS_PER_BLOCK = MAX_BLOCK_WEIGHT / MIN_TRANSACTION_OUTPUT_WEIGHT; diff --git a/src/coins.h b/src/coins.h index a3f34bb0ee..a3e241ac90 100644 --- a/src/coins.h +++ b/src/coins.h @@ -318,6 +318,13 @@ public: //! Check whether all prevouts of the transaction are present in the UTXO set represented by this view bool HaveInputs(const CTransaction& tx) const; + //! Force a reallocation of the cache map. This is required when downsizing + //! the cache because the map's allocator may be hanging onto a lot of + //! memory despite having called .clear(). + //! + //! See: https://stackoverflow.com/questions/42114044/how-to-release-unordered-map-memory + void ReallocateCache(); + private: /** * @note this is marked const, but may actually append to `cacheCoins`, increasing diff --git a/src/init.cpp b/src/init.cpp index 37e6251295..4561a10870 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -1533,7 +1533,7 @@ bool AppInitMain(const util::Ref& context, NodeContext& node) int64_t nCoinDBCache = std::min(nTotalCache / 2, (nTotalCache / 4) + (1 << 23)); // use 25%-50% of the remainder for disk cache nCoinDBCache = std::min(nCoinDBCache, nMaxCoinsDBCache << 20); // cap total coins db cache nTotalCache -= nCoinDBCache; - nCoinCacheUsage = nTotalCache; // the rest goes to in-memory cache + int64_t nCoinCacheUsage = nTotalCache; // the rest goes to in-memory cache int64_t nMempoolSizeMax = gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; LogPrintf("Cache configuration:\n"); LogPrintf("* Using %.1f MiB for block index database\n", nBlockTreeDBCache * (1.0 / 1024 / 1024)); @@ -1645,7 +1645,7 @@ bool AppInitMain(const util::Ref& context, NodeContext& node) } // The on-disk coinsdb is now in a good state, create the cache - chainstate->InitCoinsCache(); + chainstate->InitCoinsCache(nCoinCacheUsage); assert(chainstate->CanFlushToDisk()); if (!is_coinsview_empty(chainstate)) { diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index 3b7a7c8d12..83916fd080 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -139,7 +139,7 @@ TestingSetup::TestingSetup(const std::string& chainName, const std::vectorInitCache(); } @@ -2228,7 +2228,7 @@ CoinsCacheSizeState CChainState::GetCoinsCacheSizeState(const CTxMemPool& tx_poo { return this->GetCoinsCacheSizeState( tx_pool, - nCoinCacheUsage, + m_coinstip_cache_size_bytes, gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000); } @@ -4300,7 +4300,7 @@ bool CVerifyDB::VerifyDB(const CChainParams& chainparams, CCoinsView *coinsview, } } // check level 3: check for inconsistencies during memory-only disconnect of tip blocks - if (nCheckLevel >= 3 && (coins.DynamicMemoryUsage() + ::ChainstateActive().CoinsTip().DynamicMemoryUsage()) <= nCoinCacheUsage) { + if (nCheckLevel >= 3 && (coins.DynamicMemoryUsage() + ::ChainstateActive().CoinsTip().DynamicMemoryUsage()) <= ::ChainstateActive().m_coinstip_cache_size_bytes) { assert(coins.GetBestBlock() == pindex->GetBlockHash()); DisconnectResult res = ::ChainstateActive().DisconnectBlock(block, pindex, coins); if (res == DISCONNECT_FAILED) { @@ -4965,6 +4965,39 @@ std::string CChainState::ToString() tip ? tip->nHeight : -1, tip ? tip->GetBlockHash().ToString() : "null"); } +bool CChainState::ResizeCoinsCaches(size_t coinstip_size, size_t coinsdb_size) +{ + if (coinstip_size == m_coinstip_cache_size_bytes && + coinsdb_size == m_coinsdb_cache_size_bytes) { + // Cache sizes are unchanged, no need to continue. + return true; + } + size_t old_coinstip_size = m_coinstip_cache_size_bytes; + m_coinstip_cache_size_bytes = coinstip_size; + m_coinsdb_cache_size_bytes = coinsdb_size; + CoinsDB().ResizeCache(coinsdb_size); + + LogPrintf("[%s] resized coinsdb cache to %.1f MiB\n", + this->ToString(), coinsdb_size * (1.0 / 1024 / 1024)); + LogPrintf("[%s] resized coinstip cache to %.1f MiB\n", + this->ToString(), coinstip_size * (1.0 / 1024 / 1024)); + + BlockValidationState state; + const CChainParams& chainparams = Params(); + + bool ret; + + if (coinstip_size > old_coinstip_size) { + // Likely no need to flush if cache sizes have grown. + ret = FlushStateToDisk(chainparams, state, FlushStateMode::IF_NEEDED); + } else { + // Otherwise, flush state to disk and deallocate the in-memory coins map. + ret = FlushStateToDisk(chainparams, state, FlushStateMode::ALWAYS); + CoinsTip().ReallocateCache(); + } + return ret; +} + std::string CBlockFileInfo::ToString() const { return strprintf("CBlockFileInfo(blocks=%u, size=%u, heights=%u...%u, time=%s...%s)", nBlocks, nSize, nHeightFirst, nHeightLast, FormatISO8601Date(nTimeFirst), FormatISO8601Date(nTimeLast)); diff --git a/src/validation.h b/src/validation.h index 8112e38704..b201403585 100644 --- a/src/validation.h +++ b/src/validation.h @@ -127,7 +127,6 @@ extern bool g_parallel_script_checks; extern bool fRequireStandard; extern bool fCheckBlockIndex; extern bool fCheckpointsEnabled; -extern size_t nCoinCacheUsage; /** A fee rate smaller than this is considered zero fee (for relaying, mining and transaction creation) */ extern CFeeRate minRelayTxFee; /** If the tip is older than this (in seconds), the node is considered to be in initial block download. */ @@ -519,7 +518,7 @@ public: //! Initialize the in-memory coins cache (to be done after the health of the on-disk database //! is verified). - void InitCoinsCache() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + void InitCoinsCache(size_t cache_size_bytes) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); //! @returns whether or not the CoinsViews object has been fully initialized and we can //! safely flush this object to disk. @@ -568,6 +567,17 @@ public: //! Destructs all objects related to accessing the UTXO set. void ResetCoinsViews() { m_coins_views.reset(); } + //! The cache size of the on-disk coins view. + size_t m_coinsdb_cache_size_bytes{0}; + + //! The cache size of the in-memory coins view. + size_t m_coinstip_cache_size_bytes{0}; + + //! Resize the CoinsViews caches dynamically and flush state to disk. + //! @returns true unless an error occurred during the flush. + bool ResizeCoinsCaches(size_t coinstip_size, size_t coinsdb_size) + EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + /** * Update the on-disk chain state. * The caches and indexes are flushed depending on the mode we're called with