diff --git a/src/Makefile.am b/src/Makefile.am index 72dd942c40..36de5dd150 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -132,6 +132,7 @@ BITCOIN_CORE_H = \ chainparamsseeds.h \ checkqueue.h \ clientversion.h \ + cluster_linearize.h \ coins.h \ common/args.h \ common/bloom.h \ diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include index 7e3aa369c7..fe6333d8c0 100644 --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -25,6 +25,7 @@ bench_bench_bitcoin_SOURCES = \ bench/checkblock.cpp \ bench/checkblockindex.cpp \ bench/checkqueue.cpp \ + bench/cluster_linearize.cpp \ bench/crypto_hash.cpp \ bench/data.cpp \ bench/data.h \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 0993a65eff..3d04498369 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -83,6 +83,7 @@ BITCOIN_TESTS =\ test/bloom_tests.cpp \ test/bswap_tests.cpp \ test/checkqueue_tests.cpp \ + test/cluster_linearize_tests.cpp \ test/coins_tests.cpp \ test/coinstatsindex_tests.cpp \ test/common_url_tests.cpp \ @@ -302,6 +303,7 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/buffered_file.cpp \ test/fuzz/chain.cpp \ test/fuzz/checkqueue.cpp \ + test/fuzz/cluster_linearize.cpp \ test/fuzz/coins_view.cpp \ test/fuzz/coinscache_sim.cpp \ test/fuzz/connman.cpp \ diff --git a/src/Makefile.test_util.include b/src/Makefile.test_util.include index 960eb078c8..0c0e849fba 100644 --- a/src/Makefile.test_util.include +++ b/src/Makefile.test_util.include @@ -10,6 +10,7 @@ EXTRA_LIBRARIES += \ TEST_UTIL_H = \ test/util/blockfilter.h \ test/util/chainstate.h \ + test/util/cluster_linearize.h \ test/util/coins.h \ test/util/index.h \ test/util/json.h \ diff --git a/src/bench/cluster_linearize.cpp b/src/bench/cluster_linearize.cpp new file mode 100644 index 0000000000..9987d376a5 --- /dev/null +++ b/src/bench/cluster_linearize.cpp @@ -0,0 +1,214 @@ +// Copyright (c) The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include + +#include +#include + +using namespace cluster_linearize; + +namespace { + +/** Construct a linear graph. These are pessimal for AncestorCandidateFinder, as they maximize + * the number of ancestor set feerate updates. The best ancestor set is always the topmost + * remaining transaction, whose removal requires updating all remaining transactions' ancestor + * set feerates. */ +template +DepGraph MakeLinearGraph(ClusterIndex ntx) +{ + DepGraph depgraph; + for (ClusterIndex i = 0; i < ntx; ++i) { + depgraph.AddTransaction({-int32_t(i), 1}); + if (i > 0) depgraph.AddDependency(i - 1, i); + } + return depgraph; +} + +/** Construct a wide graph (one root, with N-1 children that are otherwise unrelated, with + * increasing feerates). These graphs are pessimal for the LIMO step in Linearize, because + * rechunking is needed after every candidate (the last transaction gets picked every time). + */ +template +DepGraph MakeWideGraph(ClusterIndex ntx) +{ + DepGraph depgraph; + for (ClusterIndex i = 0; i < ntx; ++i) { + depgraph.AddTransaction({int32_t(i) + 1, 1}); + if (i > 0) depgraph.AddDependency(0, i); + } + return depgraph; +} + +// Construct a difficult graph. These need at least sqrt(2^(n-1)) iterations in the best +// known algorithms (purely empirically determined). +template +DepGraph MakeHardGraph(ClusterIndex ntx) +{ + DepGraph depgraph; + for (ClusterIndex i = 0; i < ntx; ++i) { + if (ntx & 1) { + // Odd cluster size. + // + // Mermaid diagram code for the resulting cluster for 11 transactions: + // ```mermaid + // graph BT + // T0["T0: 1/2"];T1["T1: 14/2"];T2["T2: 6/1"];T3["T3: 5/1"];T4["T4: 7/1"]; + // T5["T5: 5/1"];T6["T6: 7/1"];T7["T7: 5/1"];T8["T8: 7/1"];T9["T9: 5/1"]; + // T10["T10: 7/1"]; + // T1-->T0;T1-->T2;T3-->T2;T4-->T3;T4-->T5;T6-->T5;T4-->T7;T8-->T7;T4-->T9;T10-->T9; + // ``` + if (i == 0) { + depgraph.AddTransaction({1, 2}); + } else if (i == 1) { + depgraph.AddTransaction({14, 2}); + depgraph.AddDependency(0, 1); + } else if (i == 2) { + depgraph.AddTransaction({6, 1}); + depgraph.AddDependency(2, 1); + } else if (i == 3) { + depgraph.AddTransaction({5, 1}); + depgraph.AddDependency(2, 3); + } else if ((i & 1) == 0) { + depgraph.AddTransaction({7, 1}); + depgraph.AddDependency(i - 1, i); + } else { + depgraph.AddTransaction({5, 1}); + depgraph.AddDependency(i, 4); + } + } else { + // Even cluster size. + // + // Mermaid diagram code for the resulting cluster for 10 transactions: + // ```mermaid + // graph BT + // T0["T0: 1"];T1["T1: 3"];T2["T2: 1"];T3["T3: 4"];T4["T4: 0"];T5["T5: 4"];T6["T6: 0"]; + // T7["T7: 4"];T8["T8: 0"];T9["T9: 4"]; + // T1-->T0;T2-->T0;T3-->T2;T3-->T4;T5-->T4;T3-->T6;T7-->T6;T3-->T8;T9-->T8; + // ``` + if (i == 0) { + depgraph.AddTransaction({1, 1}); + } else if (i == 1) { + depgraph.AddTransaction({3, 1}); + depgraph.AddDependency(0, 1); + } else if (i == 2) { + depgraph.AddTransaction({1, 1}); + depgraph.AddDependency(0, 2); + } else if (i & 1) { + depgraph.AddTransaction({4, 1}); + depgraph.AddDependency(i - 1, i); + } else { + depgraph.AddTransaction({0, 1}); + depgraph.AddDependency(i, 3); + } + } + } + return depgraph; +} + +/** Benchmark that does search-based candidate finding with 10000 iterations. + * + * Its goal is measuring how much time every additional search iteration in linearization costs. + */ +template +void BenchLinearizePerIterWorstCase(ClusterIndex ntx, benchmark::Bench& bench) +{ + const auto depgraph = MakeHardGraph(ntx); + const auto iter_limit = std::min(10000, uint64_t{1} << (ntx / 2 - 1)); + uint64_t rng_seed = 0; + bench.batch(iter_limit).unit("iters").run([&] { + SearchCandidateFinder finder(depgraph, rng_seed++); + auto [candidate, iters_performed] = finder.FindCandidateSet(iter_limit, {}); + assert(iters_performed == iter_limit); + }); +} + +/** Benchmark for linearization improvement of a trivial linear graph using just ancestor sort. + * + * Its goal is measuring how much time linearization may take without any search iterations. + * + * If P is the resulting time of BenchLinearizePerIterWorstCase, and N is the resulting time of + * BenchLinearizeNoItersWorstCase*, then an invocation of Linearize with max_iterations=m should + * take no more than roughly N+m*P time. This may however be an overestimate, as the worst cases + * do not coincide (the ones that are worst for linearization without any search happen to be ones + * that do not need many search iterations). + * + * This benchmark exercises a worst case for AncestorCandidateFinder, but for which improvement is + * cheap. + */ +template +void BenchLinearizeNoItersWorstCaseAnc(ClusterIndex ntx, benchmark::Bench& bench) +{ + const auto depgraph = MakeLinearGraph(ntx); + uint64_t rng_seed = 0; + std::vector old_lin(ntx); + for (ClusterIndex i = 0; i < ntx; ++i) old_lin[i] = i; + bench.run([&] { + Linearize(depgraph, /*max_iterations=*/0, rng_seed++, old_lin); + }); +} + +/** Benchmark for linearization improvement of a trivial wide graph using just ancestor sort. + * + * Its goal is measuring how much time improving a linearization may take without any search + * iterations, similar to the previous function. + * + * This benchmark exercises a worst case for improving an existing linearization, but for which + * AncestorCandidateFinder is cheap. + */ +template +void BenchLinearizeNoItersWorstCaseLIMO(ClusterIndex ntx, benchmark::Bench& bench) +{ + const auto depgraph = MakeWideGraph(ntx); + uint64_t rng_seed = 0; + std::vector old_lin(ntx); + for (ClusterIndex i = 0; i < ntx; ++i) old_lin[i] = i; + bench.run([&] { + Linearize(depgraph, /*max_iterations=*/0, rng_seed++, old_lin); + }); +} + +} // namespace + +static void LinearizePerIter16TxWorstCase(benchmark::Bench& bench) { BenchLinearizePerIterWorstCase>(16, bench); } +static void LinearizePerIter32TxWorstCase(benchmark::Bench& bench) { BenchLinearizePerIterWorstCase>(32, bench); } +static void LinearizePerIter48TxWorstCase(benchmark::Bench& bench) { BenchLinearizePerIterWorstCase>(48, bench); } +static void LinearizePerIter64TxWorstCase(benchmark::Bench& bench) { BenchLinearizePerIterWorstCase>(64, bench); } +static void LinearizePerIter75TxWorstCase(benchmark::Bench& bench) { BenchLinearizePerIterWorstCase>(75, bench); } +static void LinearizePerIter99TxWorstCase(benchmark::Bench& bench) { BenchLinearizePerIterWorstCase>(99, bench); } + +static void LinearizeNoIters16TxWorstCaseAnc(benchmark::Bench& bench) { BenchLinearizeNoItersWorstCaseAnc>(16, bench); } +static void LinearizeNoIters32TxWorstCaseAnc(benchmark::Bench& bench) { BenchLinearizeNoItersWorstCaseAnc>(32, bench); } +static void LinearizeNoIters48TxWorstCaseAnc(benchmark::Bench& bench) { BenchLinearizeNoItersWorstCaseAnc>(48, bench); } +static void LinearizeNoIters64TxWorstCaseAnc(benchmark::Bench& bench) { BenchLinearizeNoItersWorstCaseAnc>(64, bench); } +static void LinearizeNoIters75TxWorstCaseAnc(benchmark::Bench& bench) { BenchLinearizeNoItersWorstCaseAnc>(75, bench); } +static void LinearizeNoIters99TxWorstCaseAnc(benchmark::Bench& bench) { BenchLinearizeNoItersWorstCaseAnc>(99, bench); } + +static void LinearizeNoIters16TxWorstCaseLIMO(benchmark::Bench& bench) { BenchLinearizeNoItersWorstCaseLIMO>(16, bench); } +static void LinearizeNoIters32TxWorstCaseLIMO(benchmark::Bench& bench) { BenchLinearizeNoItersWorstCaseLIMO>(32, bench); } +static void LinearizeNoIters48TxWorstCaseLIMO(benchmark::Bench& bench) { BenchLinearizeNoItersWorstCaseLIMO>(48, bench); } +static void LinearizeNoIters64TxWorstCaseLIMO(benchmark::Bench& bench) { BenchLinearizeNoItersWorstCaseLIMO>(64, bench); } +static void LinearizeNoIters75TxWorstCaseLIMO(benchmark::Bench& bench) { BenchLinearizeNoItersWorstCaseLIMO>(75, bench); } +static void LinearizeNoIters99TxWorstCaseLIMO(benchmark::Bench& bench) { BenchLinearizeNoItersWorstCaseLIMO>(99, bench); } + +BENCHMARK(LinearizePerIter16TxWorstCase, benchmark::PriorityLevel::HIGH); +BENCHMARK(LinearizePerIter32TxWorstCase, benchmark::PriorityLevel::HIGH); +BENCHMARK(LinearizePerIter48TxWorstCase, benchmark::PriorityLevel::HIGH); +BENCHMARK(LinearizePerIter64TxWorstCase, benchmark::PriorityLevel::HIGH); +BENCHMARK(LinearizePerIter75TxWorstCase, benchmark::PriorityLevel::HIGH); +BENCHMARK(LinearizePerIter99TxWorstCase, benchmark::PriorityLevel::HIGH); + +BENCHMARK(LinearizeNoIters16TxWorstCaseAnc, benchmark::PriorityLevel::HIGH); +BENCHMARK(LinearizeNoIters32TxWorstCaseAnc, benchmark::PriorityLevel::HIGH); +BENCHMARK(LinearizeNoIters48TxWorstCaseAnc, benchmark::PriorityLevel::HIGH); +BENCHMARK(LinearizeNoIters64TxWorstCaseAnc, benchmark::PriorityLevel::HIGH); +BENCHMARK(LinearizeNoIters75TxWorstCaseAnc, benchmark::PriorityLevel::HIGH); +BENCHMARK(LinearizeNoIters99TxWorstCaseAnc, benchmark::PriorityLevel::HIGH); + +BENCHMARK(LinearizeNoIters16TxWorstCaseLIMO, benchmark::PriorityLevel::HIGH); +BENCHMARK(LinearizeNoIters32TxWorstCaseLIMO, benchmark::PriorityLevel::HIGH); +BENCHMARK(LinearizeNoIters48TxWorstCaseLIMO, benchmark::PriorityLevel::HIGH); +BENCHMARK(LinearizeNoIters64TxWorstCaseLIMO, benchmark::PriorityLevel::HIGH); +BENCHMARK(LinearizeNoIters75TxWorstCaseLIMO, benchmark::PriorityLevel::HIGH); +BENCHMARK(LinearizeNoIters99TxWorstCaseLIMO, benchmark::PriorityLevel::HIGH); diff --git a/src/cluster_linearize.h b/src/cluster_linearize.h new file mode 100644 index 0000000000..07d28a9aa5 --- /dev/null +++ b/src/cluster_linearize.h @@ -0,0 +1,743 @@ +// Copyright (c) The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_CLUSTER_LINEARIZE_H +#define BITCOIN_CLUSTER_LINEARIZE_H + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace cluster_linearize { + +/** Data type to represent cluster input. + * + * cluster[i].first is tx_i's fee and size. + * cluster[i].second[j] is true iff tx_i spends one or more of tx_j's outputs. + */ +template +using Cluster = std::vector>; + +/** Data type to represent transaction indices in clusters. */ +using ClusterIndex = uint32_t; + +/** Data structure that holds a transaction graph's preprocessed data (fee, size, ancestors, + * descendants). */ +template +class DepGraph +{ + /** Information about a single transaction. */ + struct Entry + { + /** Fee and size of transaction itself. */ + FeeFrac feerate; + /** All ancestors of the transaction (including itself). */ + SetType ancestors; + /** All descendants of the transaction (including itself). */ + SetType descendants; + + /** Equality operator (primarily for for testing purposes). */ + friend bool operator==(const Entry&, const Entry&) noexcept = default; + + /** Construct an empty entry. */ + Entry() noexcept = default; + /** Construct an entry with a given feerate, ancestor set, descendant set. */ + Entry(const FeeFrac& f, const SetType& a, const SetType& d) noexcept : feerate(f), ancestors(a), descendants(d) {} + }; + + /** Data for each transaction, in the same order as the Cluster it was constructed from. */ + std::vector entries; + +public: + /** Equality operator (primarily for testing purposes). */ + friend bool operator==(const DepGraph&, const DepGraph&) noexcept = default; + + // Default constructors. + DepGraph() noexcept = default; + DepGraph(const DepGraph&) noexcept = default; + DepGraph(DepGraph&&) noexcept = default; + DepGraph& operator=(const DepGraph&) noexcept = default; + DepGraph& operator=(DepGraph&&) noexcept = default; + + /** Construct a DepGraph object for ntx transactions, with no dependencies. + * + * Complexity: O(N) where N=ntx. + **/ + explicit DepGraph(ClusterIndex ntx) noexcept + { + Assume(ntx <= SetType::Size()); + entries.resize(ntx); + for (ClusterIndex i = 0; i < ntx; ++i) { + entries[i].ancestors = SetType::Singleton(i); + entries[i].descendants = SetType::Singleton(i); + } + } + + /** Construct a DepGraph object given a cluster. + * + * Complexity: O(N^2) where N=cluster.size(). + */ + explicit DepGraph(const Cluster& cluster) noexcept : entries(cluster.size()) + { + for (ClusterIndex i = 0; i < cluster.size(); ++i) { + // Fill in fee and size. + entries[i].feerate = cluster[i].first; + // Fill in direct parents as ancestors. + entries[i].ancestors = cluster[i].second; + // Make sure transactions are ancestors of themselves. + entries[i].ancestors.Set(i); + } + + // Propagate ancestor information. + for (ClusterIndex i = 0; i < entries.size(); ++i) { + // At this point, entries[a].ancestors[b] is true iff b is an ancestor of a and there + // is a path from a to b through the subgraph consisting of {a, b} union + // {0, 1, ..., (i-1)}. + SetType to_merge = entries[i].ancestors; + for (ClusterIndex j = 0; j < entries.size(); ++j) { + if (entries[j].ancestors[i]) { + entries[j].ancestors |= to_merge; + } + } + } + + // Fill in descendant information by transposing the ancestor information. + for (ClusterIndex i = 0; i < entries.size(); ++i) { + for (auto j : entries[i].ancestors) { + entries[j].descendants.Set(i); + } + } + } + + /** Get the number of transactions in the graph. Complexity: O(1). */ + auto TxCount() const noexcept { return entries.size(); } + /** Get the feerate of a given transaction i. Complexity: O(1). */ + const FeeFrac& FeeRate(ClusterIndex i) const noexcept { return entries[i].feerate; } + /** Get the ancestors of a given transaction i. Complexity: O(1). */ + const SetType& Ancestors(ClusterIndex i) const noexcept { return entries[i].ancestors; } + /** Get the descendants of a given transaction i. Complexity: O(1). */ + const SetType& Descendants(ClusterIndex i) const noexcept { return entries[i].descendants; } + + /** Add a new unconnected transaction to this transaction graph (at the end), and return its + * ClusterIndex. + * + * Complexity: O(1) (amortized, due to resizing of backing vector). + */ + ClusterIndex AddTransaction(const FeeFrac& feefrac) noexcept + { + Assume(TxCount() < SetType::Size()); + ClusterIndex new_idx = TxCount(); + entries.emplace_back(feefrac, SetType::Singleton(new_idx), SetType::Singleton(new_idx)); + return new_idx; + } + + /** Modify this transaction graph, adding a dependency between a specified parent and child. + * + * Complexity: O(N) where N=TxCount(). + **/ + void AddDependency(ClusterIndex parent, ClusterIndex child) noexcept + { + // Bail out if dependency is already implied. + if (entries[child].ancestors[parent]) return; + // To each ancestor of the parent, add as descendants the descendants of the child. + const auto& chl_des = entries[child].descendants; + for (auto anc_of_par : Ancestors(parent)) { + entries[anc_of_par].descendants |= chl_des; + } + // To each descendant of the child, add as ancestors the ancestors of the parent. + const auto& par_anc = entries[parent].ancestors; + for (auto dec_of_chl : Descendants(child)) { + entries[dec_of_chl].ancestors |= par_anc; + } + } + + /** Compute the aggregate feerate of a set of nodes in this graph. + * + * Complexity: O(N) where N=elems.Count(). + **/ + FeeFrac FeeRate(const SetType& elems) const noexcept + { + FeeFrac ret; + for (auto pos : elems) ret += entries[pos].feerate; + return ret; + } + + /** Append the entries of select to list in a topologically valid order. + * + * Complexity: O(select.Count() * log(select.Count())). + */ + void AppendTopo(std::vector& list, const SetType& select) const noexcept + { + ClusterIndex old_len = list.size(); + for (auto i : select) list.push_back(i); + std::sort(list.begin() + old_len, list.end(), [&](ClusterIndex a, ClusterIndex b) noexcept { + const auto a_anc_count = entries[a].ancestors.Count(); + const auto b_anc_count = entries[b].ancestors.Count(); + if (a_anc_count != b_anc_count) return a_anc_count < b_anc_count; + return a < b; + }); + } +}; + +/** A set of transactions together with their aggregate feerate. */ +template +struct SetInfo +{ + /** The transactions in the set. */ + SetType transactions; + /** Their combined fee and size. */ + FeeFrac feerate; + + /** Construct a SetInfo for the empty set. */ + SetInfo() noexcept = default; + + /** Construct a SetInfo for a specified set and feerate. */ + SetInfo(const SetType& txn, const FeeFrac& fr) noexcept : transactions(txn), feerate(fr) {} + + /** Construct a SetInfo for a given transaction in a depgraph. */ + explicit SetInfo(const DepGraph& depgraph, ClusterIndex pos) noexcept : + transactions(SetType::Singleton(pos)), feerate(depgraph.FeeRate(pos)) {} + + /** Construct a SetInfo for a set of transactions in a depgraph. */ + explicit SetInfo(const DepGraph& depgraph, const SetType& txn) noexcept : + transactions(txn), feerate(depgraph.FeeRate(txn)) {} + + /** Add the transactions of other to this SetInfo (no overlap allowed). */ + SetInfo& operator|=(const SetInfo& other) noexcept + { + Assume(!transactions.Overlaps(other.transactions)); + transactions |= other.transactions; + feerate += other.feerate; + return *this; + } + + /** Construct a new SetInfo equal to this, with more transactions added (which may overlap + * with the existing transactions in the SetInfo). */ + [[nodiscard]] SetInfo Add(const DepGraph& depgraph, const SetType& txn) const noexcept + { + return {transactions | txn, feerate + depgraph.FeeRate(txn - transactions)}; + } + + /** Swap two SetInfo objects. */ + friend void swap(SetInfo& a, SetInfo& b) noexcept + { + swap(a.transactions, b.transactions); + swap(a.feerate, b.feerate); + } + + /** Permit equality testing. */ + friend bool operator==(const SetInfo&, const SetInfo&) noexcept = default; +}; + +/** Compute the feerates of the chunks of linearization. */ +template +std::vector ChunkLinearization(const DepGraph& depgraph, Span linearization) noexcept +{ + std::vector ret; + for (ClusterIndex i : linearization) { + /** The new chunk to be added, initially a singleton. */ + auto new_chunk = depgraph.FeeRate(i); + // As long as the new chunk has a higher feerate than the last chunk so far, absorb it. + while (!ret.empty() && new_chunk >> ret.back()) { + new_chunk += ret.back(); + ret.pop_back(); + } + // Actually move that new chunk into the chunking. + ret.push_back(std::move(new_chunk)); + } + return ret; +} + +/** Data structure encapsulating the chunking of a linearization, permitting removal of subsets. */ +template +class LinearizationChunking +{ + /** The depgraph this linearization is for. */ + const DepGraph& m_depgraph; + + /** The linearization we started from. */ + Span m_linearization; + + /** Chunk sets and their feerates, of what remains of the linearization. */ + std::vector> m_chunks; + + /** Which transactions remain in the linearization. */ + SetType m_todo; + + /** Fill the m_chunks variable. */ + void BuildChunks() noexcept + { + // Caller must clear m_chunks. + Assume(m_chunks.empty()); + + // Iterate over the entries in m_linearization. This is effectively the same + // algorithm as ChunkLinearization, but supports skipping parts of the linearization and + // keeps track of the sets themselves instead of just their feerates. + for (auto idx : m_linearization) { + if (!m_todo[idx]) continue; + // Start with an initial chunk containing just element idx. + SetInfo add(m_depgraph, idx); + // Absorb existing final chunks into add while they have lower feerate. + while (!m_chunks.empty() && add.feerate >> m_chunks.back().feerate) { + add |= m_chunks.back(); + m_chunks.pop_back(); + } + // Remember new chunk. + m_chunks.push_back(std::move(add)); + } + } + +public: + /** Initialize a LinearizationSubset object for a given length of linearization. */ + explicit LinearizationChunking(const DepGraph& depgraph LIFETIMEBOUND, Span lin LIFETIMEBOUND) noexcept : + m_depgraph(depgraph), m_linearization(lin) + { + // Mark everything in lin as todo still. + for (auto i : m_linearization) m_todo.Set(i); + // Compute the initial chunking. + m_chunks.reserve(depgraph.TxCount()); + BuildChunks(); + } + + /** Determine how many chunks remain in the linearization. */ + ClusterIndex NumChunksLeft() const noexcept { return m_chunks.size(); } + + /** Access a chunk. Chunk 0 is the highest-feerate prefix of what remains. */ + const SetInfo& GetChunk(ClusterIndex n) const noexcept + { + Assume(n < m_chunks.size()); + return m_chunks[n]; + } + + /** Remove some subset of transactions from the linearization. */ + void MarkDone(SetType subset) noexcept + { + Assume(subset.Any()); + Assume(subset.IsSubsetOf(m_todo)); + m_todo -= subset; + // Rechunk what remains of m_linearization. + m_chunks.clear(); + BuildChunks(); + } + + /** Find the shortest intersection between subset and the prefixes of remaining chunks + * of the linearization that has a feerate not below subset's. + * + * This is a crucial operation in guaranteeing improvements to linearizations. If subset has + * a feerate not below GetChunk(0)'s, then moving Intersect(subset) to the front of (what + * remains of) the linearization is guaranteed not to make it worse at any point. + * + * See https://delvingbitcoin.org/t/introduction-to-cluster-linearization/1032 for background. + */ + SetInfo Intersect(const SetInfo& subset) const noexcept + { + Assume(subset.transactions.IsSubsetOf(m_todo)); + SetInfo accumulator; + // Iterate over all chunks of the remaining linearization. + for (ClusterIndex i = 0; i < NumChunksLeft(); ++i) { + // Find what (if any) intersection the chunk has with subset. + const SetType to_add = GetChunk(i).transactions & subset.transactions; + if (to_add.Any()) { + // If adding that to accumulator makes us hit all of subset, we are done as no + // shorter intersection with higher/equal feerate exists. + accumulator.transactions |= to_add; + if (accumulator.transactions == subset.transactions) break; + // Otherwise update the accumulator feerate. + accumulator.feerate += m_depgraph.FeeRate(to_add); + // If that does result in something better, or something with the same feerate but + // smaller, return that. Even if a longer, higher-feerate intersection exists, it + // does not hurt to return the shorter one (the remainder of the longer intersection + // will generally be found in the next call to Intersect, but even if not, it is not + // required for the improvement guarantee this function makes). + if (!(accumulator.feerate << subset.feerate)) return accumulator; + } + } + return subset; + } +}; + +/** Class encapsulating the state needed to find the best remaining ancestor set. + * + * It is initialized for an entire DepGraph, and parts of the graph can be dropped by calling + * MarkDone. + * + * As long as any part of the graph remains, FindCandidateSet() can be called which will return a + * SetInfo with the highest-feerate ancestor set that remains (an ancestor set is a single + * transaction together with all its remaining ancestors). + */ +template +class AncestorCandidateFinder +{ + /** Internal dependency graph. */ + const DepGraph& m_depgraph; + /** Which transaction are left to include. */ + SetType m_todo; + /** Precomputed ancestor-set feerates (only kept up-to-date for indices in m_todo). */ + std::vector m_ancestor_set_feerates; + +public: + /** Construct an AncestorCandidateFinder for a given cluster. + * + * Complexity: O(N^2) where N=depgraph.TxCount(). + */ + AncestorCandidateFinder(const DepGraph& depgraph LIFETIMEBOUND) noexcept : + m_depgraph(depgraph), + m_todo{SetType::Fill(depgraph.TxCount())}, + m_ancestor_set_feerates(depgraph.TxCount()) + { + // Precompute ancestor-set feerates. + for (ClusterIndex i = 0; i < depgraph.TxCount(); ++i) { + /** The remaining ancestors for transaction i. */ + SetType anc_to_add = m_depgraph.Ancestors(i); + FeeFrac anc_feerate; + // Reuse accumulated feerate from first ancestor, if usable. + Assume(anc_to_add.Any()); + ClusterIndex first = anc_to_add.First(); + if (first < i) { + anc_feerate = m_ancestor_set_feerates[first]; + Assume(!anc_feerate.IsEmpty()); + anc_to_add -= m_depgraph.Ancestors(first); + } + // Add in other ancestors (which necessarily include i itself). + Assume(anc_to_add[i]); + anc_feerate += m_depgraph.FeeRate(anc_to_add); + // Store the result. + m_ancestor_set_feerates[i] = anc_feerate; + } + } + + /** Remove a set of transactions from the set of to-be-linearized ones. + * + * The same transaction may not be MarkDone()'d twice. + * + * Complexity: O(N*M) where N=depgraph.TxCount(), M=select.Count(). + */ + void MarkDone(SetType select) noexcept + { + Assume(select.Any()); + Assume(select.IsSubsetOf(m_todo)); + m_todo -= select; + for (auto i : select) { + auto feerate = m_depgraph.FeeRate(i); + for (auto j : m_depgraph.Descendants(i) & m_todo) { + m_ancestor_set_feerates[j] -= feerate; + } + } + } + + /** Check whether any unlinearized transactions remain. */ + bool AllDone() const noexcept + { + return m_todo.None(); + } + + /** Find the best (highest-feerate, smallest among those in case of a tie) ancestor set + * among the remaining transactions. Requires !AllDone(). + * + * Complexity: O(N) where N=depgraph.TxCount(); + */ + SetInfo FindCandidateSet() const noexcept + { + Assume(!AllDone()); + std::optional best; + for (auto i : m_todo) { + if (best.has_value()) { + Assume(!m_ancestor_set_feerates[i].IsEmpty()); + if (!(m_ancestor_set_feerates[i] > m_ancestor_set_feerates[*best])) continue; + } + best = i; + } + Assume(best.has_value()); + return {m_depgraph.Ancestors(*best) & m_todo, m_ancestor_set_feerates[*best]}; + } +}; + +/** Class encapsulating the state needed to perform search for good candidate sets. + * + * It is initialized for an entire DepGraph, and parts of the graph can be dropped by calling + * MarkDone(). + * + * As long as any part of the graph remains, FindCandidateSet() can be called to perform a search + * over the set of topologically-valid subsets of that remainder, with a limit on how many + * combinations are tried. + */ +template +class SearchCandidateFinder +{ + /** Internal RNG. */ + InsecureRandomContext m_rng; + /** Internal dependency graph for the cluster. */ + const DepGraph& m_depgraph; + /** Which transactions are left to do (sorted indices). */ + SetType m_todo; + +public: + /** Construct a candidate finder for a graph. + * + * @param[in] depgraph Dependency graph for the to-be-linearized cluster. + * @param[in] rng_seed A random seed to control the search order. + * + * Complexity: O(1). + */ + SearchCandidateFinder(const DepGraph& depgraph LIFETIMEBOUND, uint64_t rng_seed) noexcept : + m_rng(rng_seed), + m_depgraph(depgraph), + m_todo(SetType::Fill(depgraph.TxCount())) {} + + /** Check whether any unlinearized transactions remain. */ + bool AllDone() const noexcept + { + return m_todo.None(); + } + + /** Find a high-feerate topologically-valid subset of what remains of the cluster. + * Requires !AllDone(). + * + * @param[in] max_iterations The maximum number of optimization steps that will be performed. + * @param[in] best A set/feerate pair with an already-known good candidate. This may + * be empty. + * @return A pair of: + * - The best (highest feerate, smallest size as tiebreaker) + * topologically valid subset (and its feerate) that was + * encountered during search. It will be at least as good as the + * best passed in (if not empty). + * - The number of optimization steps that were performed. This will + * be <= max_iterations. If strictly < max_iterations, the + * returned subset is optimal. + * + * Complexity: O(N * min(max_iterations, 2^N)) where N=depgraph.TxCount(). + */ + std::pair, uint64_t> FindCandidateSet(uint64_t max_iterations, SetInfo best) noexcept + { + Assume(!AllDone()); + + /** Type for work queue items. */ + struct WorkItem + { + /** Set of transactions definitely included (and its feerate). This must be a subset + * of m_todo, and be topologically valid (includes all in-m_todo ancestors of + * itself). */ + SetInfo inc; + /** Set of undecided transactions. This must be a subset of m_todo, and have no overlap + * with inc. The set (inc | und) must be topologically valid. */ + SetType und; + + /** Construct a new work item. */ + WorkItem(SetInfo&& i, SetType&& u) noexcept : + inc(std::move(i)), und(std::move(u)) {} + + /** Swap two WorkItems. */ + void Swap(WorkItem& other) noexcept + { + swap(inc, other.inc); + swap(und, other.und); + } + }; + + /** The queue of work items. */ + VecDeque queue; + queue.reserve(std::max(256, 2 * m_todo.Count())); + + // Create an initial entry with m_todo as undecided. Also use it as best if not provided, + // so that during the work processing loop below, and during the add_fn/split_fn calls, we + // do not need to deal with the best=empty case. + if (best.feerate.IsEmpty()) best = SetInfo(m_depgraph, m_todo); + queue.emplace_back(SetInfo{}, SetType{m_todo}); + + /** Local copy of the iteration limit. */ + uint64_t iterations_left = max_iterations; + + /** Internal function to add an item to the queue of elements to explore if there are any + * transactions left to split on, and to update best. + * + * - inc: the "inc" value for the new work item (must be topological). + * - und: the "und" value for the new work item ((inc | und) must be topological). + */ + auto add_fn = [&](SetInfo inc, SetType und) noexcept { + if (!inc.feerate.IsEmpty()) { + // If inc's feerate is better than best's, remember it as our new best. + if (inc.feerate > best.feerate) { + best = inc; + } + } else { + Assume(inc.transactions.None()); + } + + // Make sure there are undecided transactions left to split on. + if (und.None()) return; + + // Actually construct a new work item on the queue. Due to the switch to DFS when queue + // space runs out (see below), we know that no reallocation of the queue should ever + // occur. + Assume(queue.size() < queue.capacity()); + queue.emplace_back(std::move(inc), std::move(und)); + }; + + /** Internal process function. It takes an existing work item, and splits it in two: one + * with a particular transaction (and its ancestors) included, and one with that + * transaction (and its descendants) excluded. */ + auto split_fn = [&](WorkItem&& elem) noexcept { + // Any queue element must have undecided transactions left, otherwise there is nothing + // to explore anymore. + Assume(elem.und.Any()); + // The included and undecided set are all subsets of m_todo. + Assume(elem.inc.transactions.IsSubsetOf(m_todo) && elem.und.IsSubsetOf(m_todo)); + // Included transactions cannot be undecided. + Assume(!elem.inc.transactions.Overlaps(elem.und)); + + // Pick the first undecided transaction as the one to split on. + const ClusterIndex split = elem.und.First(); + + // Add a work item corresponding to exclusion of the split transaction. + const auto& desc = m_depgraph.Descendants(split); + add_fn(/*inc=*/elem.inc, + /*und=*/elem.und - desc); + + // Add a work item corresponding to inclusion of the split transaction. + const auto anc = m_depgraph.Ancestors(split) & m_todo; + add_fn(/*inc=*/elem.inc.Add(m_depgraph, anc), + /*und=*/elem.und - anc); + + // Account for the performed split. + --iterations_left; + }; + + // Work processing loop. + // + // New work items are always added at the back of the queue, but items to process use a + // hybrid approach where they can be taken from the front or the back. + // + // Depth-first search (DFS) corresponds to always taking from the back of the queue. This + // is very memory-efficient (linear in the number of transactions). Breadth-first search + // (BFS) corresponds to always taking from the front, which potentially uses more memory + // (up to exponential in the transaction count), but seems to work better in practice. + // + // The approach here combines the two: use BFS (plus random swapping) until the queue grows + // too large, at which point we temporarily switch to DFS until the size shrinks again. + while (!queue.empty()) { + // Randomly swap the first two items to randomize the search order. + if (queue.size() > 1 && m_rng.randbool()) { + queue[0].Swap(queue[1]); + } + + // Processing the first queue item, and then using DFS for everything it gives rise to, + // may increase the queue size by the number of undecided elements in there, minus 1 + // for the first queue item being removed. Thus, only when that pushes the queue over + // its capacity can we not process from the front (BFS), and should we use DFS. + while (queue.size() - 1 + queue.front().und.Count() > queue.capacity()) { + if (!iterations_left) break; + auto elem = queue.back(); + queue.pop_back(); + split_fn(std::move(elem)); + } + + // Process one entry from the front of the queue (BFS exploration) + if (!iterations_left) break; + auto elem = queue.front(); + queue.pop_front(); + split_fn(std::move(elem)); + } + + // Return the found best set and the number of iterations performed. + return {std::move(best), max_iterations - iterations_left}; + } + + /** Remove a subset of transactions from the cluster being linearized. + * + * Complexity: O(N) where N=done.Count(). + */ + void MarkDone(const SetType& done) noexcept + { + Assume(done.Any()); + Assume(done.IsSubsetOf(m_todo)); + m_todo -= done; + } +}; + +/** Find or improve a linearization for a cluster. + * + * @param[in] depgraph Dependency graph of the cluster to be linearized. + * @param[in] max_iterations Upper bound on the number of optimization steps that will be done. + * @param[in] rng_seed A random number seed to control search order. This prevents peers + * from predicting exactly which clusters would be hard for us to + * linearize. + * @param[in] old_linearization An existing linearization for the cluster (which must be + * topologically valid), or empty. + * @return A pair of: + * - The resulting linearization. It is guaranteed to be at least as + * good (in the feerate diagram sense) as old_linearization. + * - A boolean indicating whether the result is guaranteed to be + * optimal. + * + * Complexity: O(N * min(max_iterations + N, 2^N)) where N=depgraph.TxCount(). + */ +template +std::pair, bool> Linearize(const DepGraph& depgraph, uint64_t max_iterations, uint64_t rng_seed, Span old_linearization = {}) noexcept +{ + Assume(old_linearization.empty() || old_linearization.size() == depgraph.TxCount()); + if (depgraph.TxCount() == 0) return {{}, true}; + + uint64_t iterations_left = max_iterations; + std::vector linearization; + + AncestorCandidateFinder anc_finder(depgraph); + SearchCandidateFinder src_finder(depgraph, rng_seed); + linearization.reserve(depgraph.TxCount()); + bool optimal = true; + + /** Chunking of what remains of the old linearization. */ + LinearizationChunking old_chunking(depgraph, old_linearization); + + while (true) { + // Find the highest-feerate prefix of the remainder of old_linearization. + SetInfo best_prefix; + if (old_chunking.NumChunksLeft()) best_prefix = old_chunking.GetChunk(0); + + // Then initialize best to be either the best remaining ancestor set, or the first chunk. + auto best = anc_finder.FindCandidateSet(); + if (!best_prefix.feerate.IsEmpty() && best_prefix.feerate >= best.feerate) best = best_prefix; + + // Invoke bounded search to update best, with up to half of our remaining iterations as + // limit. + uint64_t max_iterations_now = (iterations_left + 1) / 2; + uint64_t iterations_done_now = 0; + std::tie(best, iterations_done_now) = src_finder.FindCandidateSet(max_iterations_now, best); + iterations_left -= iterations_done_now; + + if (iterations_done_now == max_iterations_now) { + optimal = false; + // If the search result is not (guaranteed to be) optimal, run intersections to make + // sure we don't pick something that makes us unable to reach further diagram points + // of the old linearization. + if (old_chunking.NumChunksLeft() > 0) { + best = old_chunking.Intersect(best); + } + } + + // Add to output in topological order. + depgraph.AppendTopo(linearization, best.transactions); + + // Update state to reflect best is no longer to be linearized. + anc_finder.MarkDone(best.transactions); + if (anc_finder.AllDone()) break; + src_finder.MarkDone(best.transactions); + if (old_chunking.NumChunksLeft() > 0) { + old_chunking.MarkDone(best.transactions); + } + } + + return {std::move(linearization), optimal}; +} + +} // namespace cluster_linearize + +#endif // BITCOIN_CLUSTER_LINEARIZE_H diff --git a/src/test/cluster_linearize_tests.cpp b/src/test/cluster_linearize_tests.cpp new file mode 100644 index 0000000000..d15e783ea1 --- /dev/null +++ b/src/test/cluster_linearize_tests.cpp @@ -0,0 +1,138 @@ +// Copyright (c) The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include +#include +#include + +#include + +#include + +BOOST_FIXTURE_TEST_SUITE(cluster_linearize_tests, BasicTestingSetup) + +using namespace cluster_linearize; + +namespace { + +template +void TestDepGraphSerialization(const Cluster& cluster, const std::string& hexenc) +{ + DepGraph depgraph(cluster); + + // Run normal sanity and correspondence checks, which includes a round-trip test. + VerifyDepGraphFromCluster(cluster, depgraph); + + // There may be multiple serializations of the same graph, but DepGraphFormatter's serializer + // only produces one of those. Verify that hexenc matches that canonical serialization. + std::vector encoding; + VectorWriter writer(encoding, 0); + writer << Using(depgraph); + BOOST_CHECK_EQUAL(HexStr(encoding), hexenc); + + // Test that deserializing that encoding yields depgraph. This is effectively already implied + // by the round-trip test above (if depgraph is acyclic), but verify it explicitly again here. + SpanReader reader(encoding); + DepGraph depgraph_read; + reader >> Using(depgraph_read); + BOOST_CHECK(depgraph == depgraph_read); +} + +} // namespace + +BOOST_AUTO_TEST_CASE(depgraph_ser_tests) +{ + // Empty cluster. + TestDepGraphSerialization( + {}, + "00" /* end of graph */); + + // Transactions: A(fee=0,size=1). + TestDepGraphSerialization( + {{{0, 1}, {}}}, + "01" /* A size */ + "00" /* A fee */ + "00" /* A insertion position (no skips): A */ + "00" /* end of graph */); + + // Transactions: A(fee=42,size=11), B(fee=-13,size=7), B depends on A. + TestDepGraphSerialization( + {{{42, 11}, {}}, {{-13, 7}, {0}}}, + "0b" /* A size */ + "54" /* A fee */ + "00" /* A insertion position (no skips): A */ + "07" /* B size */ + "19" /* B fee */ + "00" /* B->A dependency (no skips) */ + "00" /* B insertion position (no skips): A,B */ + "00" /* end of graph */); + + // Transactions: A(64,128), B(128,256), C(1,1), C depends on A and B. + TestDepGraphSerialization( + {{{64, 128}, {}}, {{128, 256}, {}}, {{1, 1}, {0, 1}}}, + "8000" /* A size */ + "8000" /* A fee */ + "00" /* A insertion position (no skips): A */ + "8100" /* B size */ + "8100" /* B fee */ + "01" /* B insertion position (skip B->A dependency): A,B */ + "01" /* C size */ + "02" /* C fee */ + "00" /* C->B dependency (no skips) */ + "00" /* C->A dependency (no skips) */ + "00" /* C insertion position (no skips): A,B,C */ + "00" /* end of graph */); + + // Transactions: A(-57,113), B(57,114), C(-58,115), D(58,116). Deps: B->A, C->A, D->C, in order + // [B,A,C,D]. This exercises non-topological ordering (internally serialized as A,B,C,D). + TestDepGraphSerialization( + {{{57, 114}, {1}}, {{-57, 113}, {}}, {{-58, 115}, {1}}, {{58, 116}, {2}}}, + "71" /* A size */ + "71" /* A fee */ + "00" /* A insertion position (no skips): A */ + "72" /* B size */ + "72" /* B fee */ + "00" /* B->A dependency (no skips) */ + "01" /* B insertion position (skip A): B,A */ + "73" /* C size */ + "73" /* C fee */ + "01" /* C->A dependency (skip C->B dependency) */ + "00" /* C insertion position (no skips): B,A,C */ + "74" /* D size */ + "74" /* D fee */ + "00" /* D->C dependency (no skips) */ + "01" /* D insertion position (skip D->B dependency, D->A is implied): B,A,C,D */ + "00" /* end of graph */); + + // Transactions: A(1,2), B(3,1), C(2,1), D(1,3), E(1,1). Deps: C->A, D->A, D->B, E->D. + // In order: [D,A,B,E,C]. Internally serialized in order A,B,C,D,E. + TestDepGraphSerialization( + {{{1, 3}, {1, 2}}, {{1, 2}, {}}, {{3, 1}, {}}, {{1, 1}, {0}}, {{2, 1}, {1}}}, + "02" /* A size */ + "02" /* A fee */ + "00" /* A insertion position (no skips): A */ + "01" /* B size */ + "06" /* B fee */ + "01" /* B insertion position (skip B->A dependency): A,B */ + "01" /* C size */ + "04" /* C fee */ + "01" /* C->A dependency (skip C->B dependency) */ + "00" /* C insertion position (no skips): A,B,C */ + "03" /* D size */ + "02" /* D fee */ + "01" /* D->B dependency (skip D->C dependency) */ + "00" /* D->A dependency (no skips) */ + "03" /* D insertion position (skip C,B,A): D,A,B,C */ + "01" /* E size */ + "02" /* E fee */ + "00" /* E->D dependency (no skips) */ + "02" /* E insertion position (skip E->C dependency, E->B and E->A are implied, + skip insertion C): D,A,B,E,C */ + "00" /* end of graph */ + ); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/fuzz/cluster_linearize.cpp b/src/test/fuzz/cluster_linearize.cpp new file mode 100644 index 0000000000..031cb04559 --- /dev/null +++ b/src/test/fuzz/cluster_linearize.cpp @@ -0,0 +1,689 @@ +// Copyright (c) The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +using namespace cluster_linearize; + +namespace { + +/** A simple finder class for candidate sets. + * + * This class matches SearchCandidateFinder in interface and behavior, though with fewer + * optimizations. + */ +template +class SimpleCandidateFinder +{ + /** Internal dependency graph. */ + const DepGraph& m_depgraph; + /** Which transaction are left to include. */ + SetType m_todo; + +public: + /** Construct an SimpleCandidateFinder for a given graph. */ + SimpleCandidateFinder(const DepGraph& depgraph LIFETIMEBOUND) noexcept : + m_depgraph(depgraph), m_todo{SetType::Fill(depgraph.TxCount())} {} + + /** Remove a set of transactions from the set of to-be-linearized ones. */ + void MarkDone(SetType select) noexcept { m_todo -= select; } + + /** Determine whether unlinearized transactions remain. */ + bool AllDone() const noexcept { return m_todo.None(); } + + /** Find a candidate set using at most max_iterations iterations, and the number of iterations + * actually performed. If that number is less than max_iterations, then the result is optimal. + * + * Complexity: O(N * M), where M is the number of connected topological subsets of the cluster. + * That number is bounded by M <= 2^(N-1). + */ + std::pair, uint64_t> FindCandidateSet(uint64_t max_iterations) const noexcept + { + uint64_t iterations_left = max_iterations; + // Queue of work units. Each consists of: + // - inc: set of transactions definitely included + // - und: set of transactions that can be added to inc still + std::vector> queue; + // Initially we have just one queue element, with the entire graph in und. + queue.emplace_back(SetType{}, m_todo); + // Best solution so far. + SetInfo best(m_depgraph, m_todo); + // Process the queue. + while (!queue.empty() && iterations_left) { + --iterations_left; + // Pop top element of the queue. + auto [inc, und] = queue.back(); + queue.pop_back(); + // Look for a transaction to consider adding/removing. + bool inc_none = inc.None(); + for (auto split : und) { + // If inc is empty, consider any split transaction. Otherwise only consider + // transactions that share ancestry with inc so far (which means only connected + // sets will be considered). + if (inc_none || inc.Overlaps(m_depgraph.Ancestors(split))) { + // Add a queue entry with split included. + SetInfo new_inc(m_depgraph, inc | (m_todo & m_depgraph.Ancestors(split))); + queue.emplace_back(new_inc.transactions, und - new_inc.transactions); + // Add a queue entry with split excluded. + queue.emplace_back(inc, und - m_depgraph.Descendants(split)); + // Update statistics to account for the candidate new_inc. + if (new_inc.feerate > best.feerate) best = new_inc; + break; + } + } + } + return {std::move(best), max_iterations - iterations_left}; + } +}; + +/** A very simple finder class for optimal candidate sets, which tries every subset. + * + * It is even simpler than SimpleCandidateFinder, and is primarily included here to test the + * correctness of SimpleCandidateFinder, which is then used to test the correctness of + * SearchCandidateFinder. + */ +template +class ExhaustiveCandidateFinder +{ + /** Internal dependency graph. */ + const DepGraph& m_depgraph; + /** Which transaction are left to include. */ + SetType m_todo; + +public: + /** Construct an ExhaustiveCandidateFinder for a given graph. */ + ExhaustiveCandidateFinder(const DepGraph& depgraph LIFETIMEBOUND) noexcept : + m_depgraph(depgraph), m_todo{SetType::Fill(depgraph.TxCount())} {} + + /** Remove a set of transactions from the set of to-be-linearized ones. */ + void MarkDone(SetType select) noexcept { m_todo -= select; } + + /** Determine whether unlinearized transactions remain. */ + bool AllDone() const noexcept { return m_todo.None(); } + + /** Find the optimal remaining candidate set. + * + * Complexity: O(N * 2^N). + */ + SetInfo FindCandidateSet() const noexcept + { + // Best solution so far. + SetInfo best{m_todo, m_depgraph.FeeRate(m_todo)}; + // The number of combinations to try. + uint64_t limit = (uint64_t{1} << m_todo.Count()) - 1; + // Try the transitive closure of every non-empty subset of m_todo. + for (uint64_t x = 1; x < limit; ++x) { + // If bit number b is set in x, then the remaining ancestors of the b'th remaining + // transaction in m_todo are included. + SetType txn; + auto x_shifted{x}; + for (auto i : m_todo) { + if (x_shifted & 1) txn |= m_depgraph.Ancestors(i); + x_shifted >>= 1; + } + SetInfo cur(m_depgraph, txn & m_todo); + if (cur.feerate > best.feerate) best = cur; + } + return best; + } +}; + +/** A simple linearization algorithm. + * + * This matches Linearize() in interface and behavior, though with fewer optimizations, lacking + * the ability to pass in an existing linearization, and using just SimpleCandidateFinder rather + * than AncestorCandidateFinder and SearchCandidateFinder. + */ +template +std::pair, bool> SimpleLinearize(const DepGraph& depgraph, uint64_t max_iterations) +{ + std::vector linearization; + SimpleCandidateFinder finder(depgraph); + SetType todo = SetType::Fill(depgraph.TxCount()); + bool optimal = true; + while (todo.Any()) { + auto [candidate, iterations_done] = finder.FindCandidateSet(max_iterations); + if (iterations_done == max_iterations) optimal = false; + depgraph.AppendTopo(linearization, candidate.transactions); + todo -= candidate.transactions; + finder.MarkDone(candidate.transactions); + max_iterations -= iterations_done; + } + return {std::move(linearization), optimal}; +} + +/** Given a dependency graph, and a todo set, read a topological subset of todo from reader. */ +template +SetType ReadTopologicalSet(const DepGraph& depgraph, const SetType& todo, SpanReader& reader) +{ + uint64_t mask{0}; + try { + reader >> VARINT(mask); + } catch(const std::ios_base::failure&) {} + SetType ret; + for (auto i : todo) { + if (!ret[i]) { + if (mask & 1) ret |= depgraph.Ancestors(i); + mask >>= 1; + } + } + return ret & todo; +} + +/** Given a dependency graph, construct any valid linearization for it, reading from a SpanReader. */ +template +std::vector ReadLinearization(const DepGraph& depgraph, SpanReader& reader) +{ + std::vector linearization; + TestBitSet todo = TestBitSet::Fill(depgraph.TxCount()); + // In every iteration one topologically-valid transaction is appended to linearization. + while (todo.Any()) { + // Compute the set of transactions with no not-yet-included ancestors. + TestBitSet potential_next; + for (auto j : todo) { + if ((depgraph.Ancestors(j) & todo) == TestBitSet::Singleton(j)) { + potential_next.Set(j); + } + } + // There must always be one (otherwise there is a cycle in the graph). + assert(potential_next.Any()); + // Read a number from reader, and interpret it as index into potential_next. + uint64_t idx{0}; + try { + reader >> VARINT(idx); + } catch (const std::ios_base::failure&) {} + idx %= potential_next.Count(); + // Find out which transaction that corresponds to. + for (auto j : potential_next) { + if (idx == 0) { + // When found, add it to linearization and remove it from todo. + linearization.push_back(j); + assert(todo[j]); + todo.Reset(j); + break; + } + --idx; + } + } + return linearization; +} + +} // namespace + +FUZZ_TARGET(clusterlin_add_dependency) +{ + // Verify that computing a DepGraph from a cluster, or building it step by step using AddDependency + // have the same effect. + + // Construct a cluster of a certain length, with no dependencies. + FuzzedDataProvider provider(buffer.data(), buffer.size()); + auto num_tx = provider.ConsumeIntegralInRange(2, 32); + Cluster cluster(num_tx, std::pair{FeeFrac{0, 1}, TestBitSet{}}); + // Construct the corresponding DepGraph object (also no dependencies). + DepGraph depgraph(cluster); + SanityCheck(depgraph); + // Read (parent, child) pairs, and add them to the cluster and depgraph. + LIMITED_WHILE(provider.remaining_bytes() > 0, TestBitSet::Size() * TestBitSet::Size()) { + auto parent = provider.ConsumeIntegralInRange(0, num_tx - 1); + auto child = provider.ConsumeIntegralInRange(0, num_tx - 2); + child += (child >= parent); + cluster[child].second.Set(parent); + depgraph.AddDependency(parent, child); + assert(depgraph.Ancestors(child)[parent]); + assert(depgraph.Descendants(parent)[child]); + } + // Sanity check the result. + SanityCheck(depgraph); + // Verify that the resulting DepGraph matches one recomputed from the cluster. + assert(DepGraph(cluster) == depgraph); +} + +FUZZ_TARGET(clusterlin_cluster_serialization) +{ + // Verify that any graph of transactions has its ancestry correctly computed by DepGraph, and + // if it is a DAG, that it can be serialized as a DepGraph in a way that roundtrips. This + // guarantees that any acyclic cluster has a corresponding DepGraph serialization. + + FuzzedDataProvider provider(buffer.data(), buffer.size()); + + // Construct a cluster in a naive way (using a FuzzedDataProvider-based serialization). + Cluster cluster; + auto num_tx = provider.ConsumeIntegralInRange(1, 32); + cluster.resize(num_tx); + for (ClusterIndex i = 0; i < num_tx; ++i) { + cluster[i].first.size = provider.ConsumeIntegralInRange(1, 0x3fffff); + cluster[i].first.fee = provider.ConsumeIntegralInRange(-0x8000000000000, 0x7ffffffffffff); + for (ClusterIndex j = 0; j < num_tx; ++j) { + if (i == j) continue; + if (provider.ConsumeBool()) cluster[i].second.Set(j); + } + } + + // Construct dependency graph, and verify it matches the cluster (which includes a round-trip + // check for the serialization). + DepGraph depgraph(cluster); + VerifyDepGraphFromCluster(cluster, depgraph); +} + +FUZZ_TARGET(clusterlin_depgraph_serialization) +{ + // Verify that any deserialized depgraph is acyclic and roundtrips to an identical depgraph. + + // Construct a graph by deserializing. + SpanReader reader(buffer); + DepGraph depgraph; + try { + reader >> Using(depgraph); + } catch (const std::ios_base::failure&) {} + SanityCheck(depgraph); + + // Verify the graph is a DAG. + assert(IsAcyclic(depgraph)); +} + +FUZZ_TARGET(clusterlin_chunking) +{ + // Verify the correctness of the ChunkLinearization function. + + // Construct a graph by deserializing. + SpanReader reader(buffer); + DepGraph depgraph; + try { + reader >> Using(depgraph); + } catch (const std::ios_base::failure&) {} + + // Read a valid linearization for depgraph. + auto linearization = ReadLinearization(depgraph, reader); + + // Invoke the chunking function. + auto chunking = ChunkLinearization(depgraph, linearization); + + // Verify that chunk feerates are monotonically non-increasing. + for (size_t i = 1; i < chunking.size(); ++i) { + assert(!(chunking[i] >> chunking[i - 1])); + } + + // Naively recompute the chunks (each is the highest-feerate prefix of what remains). + auto todo = TestBitSet::Fill(depgraph.TxCount()); + for (const auto& chunk_feerate : chunking) { + assert(todo.Any()); + SetInfo accumulator, best; + for (ClusterIndex idx : linearization) { + if (todo[idx]) { + accumulator |= SetInfo(depgraph, idx); + if (best.feerate.IsEmpty() || accumulator.feerate >> best.feerate) { + best = accumulator; + } + } + } + assert(chunk_feerate == best.feerate); + assert(best.transactions.IsSubsetOf(todo)); + todo -= best.transactions; + } + assert(todo.None()); +} + +FUZZ_TARGET(clusterlin_ancestor_finder) +{ + // Verify that AncestorCandidateFinder works as expected. + + // Retrieve a depgraph from the fuzz input. + SpanReader reader(buffer); + DepGraph depgraph; + try { + reader >> Using(depgraph); + } catch (const std::ios_base::failure&) {} + + AncestorCandidateFinder anc_finder(depgraph); + auto todo = TestBitSet::Fill(depgraph.TxCount()); + while (todo.Any()) { + // Call the ancestor finder's FindCandidateSet for what remains of the graph. + assert(!anc_finder.AllDone()); + auto best_anc = anc_finder.FindCandidateSet(); + // Sanity check the result. + assert(best_anc.transactions.Any()); + assert(best_anc.transactions.IsSubsetOf(todo)); + assert(depgraph.FeeRate(best_anc.transactions) == best_anc.feerate); + // Check that it is topologically valid. + for (auto i : best_anc.transactions) { + assert((depgraph.Ancestors(i) & todo).IsSubsetOf(best_anc.transactions)); + } + + // Compute all remaining ancestor sets. + std::optional> real_best_anc; + for (auto i : todo) { + SetInfo info(depgraph, todo & depgraph.Ancestors(i)); + if (!real_best_anc.has_value() || info.feerate > real_best_anc->feerate) { + real_best_anc = info; + } + } + // The set returned by anc_finder must equal the real best ancestor sets. + assert(real_best_anc.has_value()); + assert(*real_best_anc == best_anc); + + // Find a topologically valid subset of transactions to remove from the graph. + auto del_set = ReadTopologicalSet(depgraph, todo, reader); + // If we did not find anything, use best_anc itself, because we should remove something. + if (del_set.None()) del_set = best_anc.transactions; + todo -= del_set; + anc_finder.MarkDone(del_set); + } + assert(anc_finder.AllDone()); +} + +static constexpr auto MAX_SIMPLE_ITERATIONS = 300000; + +FUZZ_TARGET(clusterlin_search_finder) +{ + // Verify that SearchCandidateFinder works as expected by sanity checking the results + // and comparing with the results from SimpleCandidateFinder, ExhaustiveCandidateFinder, and + // AncestorCandidateFinder. + + // Retrieve an RNG seed and a depgraph from the fuzz input. + SpanReader reader(buffer); + DepGraph depgraph; + uint64_t rng_seed{0}; + try { + reader >> Using(depgraph) >> rng_seed; + } catch (const std::ios_base::failure&) {} + + // Instantiate ALL the candidate finders. + SearchCandidateFinder src_finder(depgraph, rng_seed); + SimpleCandidateFinder smp_finder(depgraph); + ExhaustiveCandidateFinder exh_finder(depgraph); + AncestorCandidateFinder anc_finder(depgraph); + + auto todo = TestBitSet::Fill(depgraph.TxCount()); + while (todo.Any()) { + assert(!src_finder.AllDone()); + assert(!smp_finder.AllDone()); + assert(!exh_finder.AllDone()); + assert(!anc_finder.AllDone()); + + // For each iteration, read an iteration count limit from the fuzz input. + uint64_t max_iterations = 1; + try { + reader >> VARINT(max_iterations); + } catch (const std::ios_base::failure&) {} + max_iterations &= 0xfffff; + + // Read an initial subset from the fuzz input. + SetInfo init_best(depgraph, ReadTopologicalSet(depgraph, todo, reader)); + + // Call the search finder's FindCandidateSet for what remains of the graph. + auto [found, iterations_done] = src_finder.FindCandidateSet(max_iterations, init_best); + + // Sanity check the result. + assert(iterations_done <= max_iterations); + assert(found.transactions.Any()); + assert(found.transactions.IsSubsetOf(todo)); + assert(depgraph.FeeRate(found.transactions) == found.feerate); + if (!init_best.feerate.IsEmpty()) assert(found.feerate >= init_best.feerate); + // Check that it is topologically valid. + for (auto i : found.transactions) { + assert(found.transactions.IsSupersetOf(depgraph.Ancestors(i) & todo)); + } + + // At most 2^N-1 iterations can be required: the number of non-empty subsets a graph with N + // transactions has. + assert(iterations_done <= ((uint64_t{1} << todo.Count()) - 1)); + + // Perform quality checks only if SearchCandidateFinder claims an optimal result. + if (iterations_done < max_iterations) { + // Compare with SimpleCandidateFinder. + auto [simple, simple_iters] = smp_finder.FindCandidateSet(MAX_SIMPLE_ITERATIONS); + assert(found.feerate >= simple.feerate); + if (simple_iters < MAX_SIMPLE_ITERATIONS) { + assert(found.feerate == simple.feerate); + } + + // Compare with AncestorCandidateFinder; + auto anc = anc_finder.FindCandidateSet(); + assert(found.feerate >= anc.feerate); + + // Compare with ExhaustiveCandidateFinder. This quickly gets computationally expensive + // for large clusters (O(2^n)), so only do it for sufficiently small ones. + if (todo.Count() <= 12) { + auto exhaustive = exh_finder.FindCandidateSet(); + assert(exhaustive.feerate == found.feerate); + // Also compare ExhaustiveCandidateFinder with SimpleCandidateFinder (this is + // primarily a test for SimpleCandidateFinder's correctness). + assert(exhaustive.feerate >= simple.feerate); + if (simple_iters < MAX_SIMPLE_ITERATIONS) { + assert(exhaustive.feerate == simple.feerate); + } + } + } + + // Find a topologically valid subset of transactions to remove from the graph. + auto del_set = ReadTopologicalSet(depgraph, todo, reader); + // If we did not find anything, use found itself, because we should remove something. + if (del_set.None()) del_set = found.transactions; + todo -= del_set; + src_finder.MarkDone(del_set); + smp_finder.MarkDone(del_set); + exh_finder.MarkDone(del_set); + anc_finder.MarkDone(del_set); + } + + assert(src_finder.AllDone()); + assert(smp_finder.AllDone()); + assert(exh_finder.AllDone()); + assert(anc_finder.AllDone()); +} + +FUZZ_TARGET(clusterlin_linearization_chunking) +{ + // Verify the behavior of LinearizationChunking. + + // Retrieve a depgraph from the fuzz input. + SpanReader reader(buffer); + DepGraph depgraph; + try { + reader >> Using(depgraph); + } catch (const std::ios_base::failure&) {} + + // Retrieve a topologically-valid subset of depgraph. + auto todo = TestBitSet::Fill(depgraph.TxCount()); + auto subset = SetInfo(depgraph, ReadTopologicalSet(depgraph, todo, reader)); + + // Retrieve a valid linearization for depgraph. + auto linearization = ReadLinearization(depgraph, reader); + + // Construct a LinearizationChunking object, initially for the whole linearization. + LinearizationChunking chunking(depgraph, linearization); + + // Incrementally remove transactions from the chunking object, and check various properties at + // every step. + while (todo.Any()) { + assert(chunking.NumChunksLeft() > 0); + + // Construct linearization with just todo. + std::vector linearization_left; + for (auto i : linearization) { + if (todo[i]) linearization_left.push_back(i); + } + + // Compute the chunking for linearization_left. + auto chunking_left = ChunkLinearization(depgraph, linearization_left); + + // Verify that it matches the feerates of the chunks of chunking. + assert(chunking.NumChunksLeft() == chunking_left.size()); + for (ClusterIndex i = 0; i < chunking.NumChunksLeft(); ++i) { + assert(chunking.GetChunk(i).feerate == chunking_left[i]); + } + + // Check consistency of chunking. + TestBitSet combined; + for (ClusterIndex i = 0; i < chunking.NumChunksLeft(); ++i) { + const auto& chunk_info = chunking.GetChunk(i); + // Chunks must be non-empty. + assert(chunk_info.transactions.Any()); + // Chunk feerates must be monotonically non-increasing. + if (i > 0) assert(!(chunk_info.feerate >> chunking.GetChunk(i - 1).feerate)); + // Chunks must be a subset of what is left of the linearization. + assert(chunk_info.transactions.IsSubsetOf(todo)); + // Chunks' claimed feerates must match their transactions' aggregate feerate. + assert(depgraph.FeeRate(chunk_info.transactions) == chunk_info.feerate); + // Chunks must be the highest-feerate remaining prefix. + SetInfo accumulator, best; + for (auto j : linearization) { + if (todo[j] && !combined[j]) { + accumulator |= SetInfo(depgraph, j); + if (best.feerate.IsEmpty() || accumulator.feerate > best.feerate) { + best = accumulator; + } + } + } + assert(best.transactions == chunk_info.transactions); + assert(best.feerate == chunk_info.feerate); + // Chunks cannot overlap. + assert(!chunk_info.transactions.Overlaps(combined)); + combined |= chunk_info.transactions; + // Chunks must be topological. + for (auto idx : chunk_info.transactions) { + assert((depgraph.Ancestors(idx) & todo).IsSubsetOf(combined)); + } + } + assert(combined == todo); + + // Verify the expected properties of LinearizationChunking::Intersect: + auto intersect = chunking.Intersect(subset); + // - Intersecting again doesn't change the result. + assert(chunking.Intersect(intersect) == intersect); + // - The intersection is topological. + TestBitSet intersect_anc; + for (auto idx : intersect.transactions) { + intersect_anc |= (depgraph.Ancestors(idx) & todo); + } + assert(intersect.transactions == intersect_anc); + // - The claimed intersection feerate matches its transactions. + assert(intersect.feerate == depgraph.FeeRate(intersect.transactions)); + // - The intersection may only be empty if its input is empty. + assert(intersect.transactions.Any() == subset.transactions.Any()); + // - The intersection feerate must be as high as the input. + assert(intersect.feerate >= subset.feerate); + // - No non-empty intersection between the intersection and a prefix of the chunks of the + // remainder of the linearization may be better than the intersection. + TestBitSet prefix; + for (ClusterIndex i = 0; i < chunking.NumChunksLeft(); ++i) { + prefix |= chunking.GetChunk(i).transactions; + auto reintersect = SetInfo(depgraph, prefix & intersect.transactions); + if (!reintersect.feerate.IsEmpty()) { + assert(reintersect.feerate <= intersect.feerate); + } + } + + // Find a subset to remove from linearization. + auto done = ReadTopologicalSet(depgraph, todo, reader); + if (done.None()) { + // We need to remove a non-empty subset, so fall back to the unlinearized ancestors of + // the first transaction in todo if done is empty. + done = depgraph.Ancestors(todo.First()) & todo; + } + todo -= done; + chunking.MarkDone(done); + subset = SetInfo(depgraph, subset.transactions - done); + } + + assert(chunking.NumChunksLeft() == 0); +} + +FUZZ_TARGET(clusterlin_linearize) +{ + // Verify the behavior of Linearize(). + + // Retrieve an RNG seed, an iteration count, and a depgraph from the fuzz input. + SpanReader reader(buffer); + DepGraph depgraph; + uint64_t rng_seed{0}; + uint64_t iter_count{0}; + try { + reader >> VARINT(iter_count) >> Using(depgraph) >> rng_seed; + } catch (const std::ios_base::failure&) {} + + // Optionally construct an old linearization for it. + std::vector old_linearization; + { + uint8_t have_old_linearization{0}; + try { + reader >> have_old_linearization; + } catch(const std::ios_base::failure&) {} + if (have_old_linearization & 1) { + old_linearization = ReadLinearization(depgraph, reader); + SanityCheck(depgraph, old_linearization); + } + } + + // Invoke Linearize(). + iter_count &= 0x7ffff; + auto [linearization, optimal] = Linearize(depgraph, iter_count, rng_seed, old_linearization); + SanityCheck(depgraph, linearization); + auto chunking = ChunkLinearization(depgraph, linearization); + + // Linearization must always be as good as the old one, if provided. + if (!old_linearization.empty()) { + auto old_chunking = ChunkLinearization(depgraph, old_linearization); + auto cmp = CompareChunks(chunking, old_chunking); + assert(cmp >= 0); + } + + // If the iteration count is sufficiently high, an optimal linearization must be found. + // Each linearization step can use up to 2^k iterations, with steps k=1..n. That sum is + // 2 * (2^n - 1) + const uint64_t n = depgraph.TxCount(); + if (n <= 18 && iter_count > 2U * ((uint64_t{1} << n) - 1U)) { + assert(optimal); + } + + // If Linearize claims optimal result, run quality tests. + if (optimal) { + // It must be as good as SimpleLinearize. + auto [simple_linearization, simple_optimal] = SimpleLinearize(depgraph, MAX_SIMPLE_ITERATIONS); + SanityCheck(depgraph, simple_linearization); + auto simple_chunking = ChunkLinearization(depgraph, simple_linearization); + auto cmp = CompareChunks(chunking, simple_chunking); + assert(cmp >= 0); + // If SimpleLinearize finds the optimal result too, they must be equal (if not, + // SimpleLinearize is broken). + if (simple_optimal) assert(cmp == 0); + + // Only for very small clusters, test every topologically-valid permutation. + if (depgraph.TxCount() <= 7) { + std::vector perm_linearization(depgraph.TxCount()); + for (ClusterIndex i = 0; i < depgraph.TxCount(); ++i) perm_linearization[i] = i; + // Iterate over all valid permutations. + do { + // Determine whether perm_linearization is topological. + TestBitSet perm_done; + bool perm_is_topo{true}; + for (auto i : perm_linearization) { + perm_done.Set(i); + if (!depgraph.Ancestors(i).IsSubsetOf(perm_done)) { + perm_is_topo = false; + break; + } + } + // If so, verify that the obtained linearization is as good as the permutation. + if (perm_is_topo) { + auto perm_chunking = ChunkLinearization(depgraph, perm_linearization); + auto cmp = CompareChunks(chunking, perm_chunking); + assert(cmp >= 0); + } + } while(std::next_permutation(perm_linearization.begin(), perm_linearization.end())); + } + } +} diff --git a/src/test/util/cluster_linearize.h b/src/test/util/cluster_linearize.h new file mode 100644 index 0000000000..508a08133c --- /dev/null +++ b/src/test/util/cluster_linearize.h @@ -0,0 +1,353 @@ +// Copyright (c) The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_TEST_UTIL_CLUSTER_LINEARIZE_H +#define BITCOIN_TEST_UTIL_CLUSTER_LINEARIZE_H + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace { + +using namespace cluster_linearize; + +using TestBitSet = BitSet<32>; + +/** Check if a graph is acyclic. */ +template +bool IsAcyclic(const DepGraph& depgraph) noexcept +{ + for (ClusterIndex i = 0; i < depgraph.TxCount(); ++i) { + if ((depgraph.Ancestors(i) & depgraph.Descendants(i)) != SetType::Singleton(i)) { + return false; + } + } + return true; +} + +/** A formatter for a bespoke serialization for acyclic DepGraph objects. + * + * The serialization format outputs information about transactions in a topological order (parents + * before children), together with position information so transactions can be moved back to their + * correct position on deserialization. + * + * - For each transaction t in the DepGraph (in some topological order); + * - The size: VARINT(t.size), which cannot be 0. + * - The fee: VARINT(SignedToUnsigned(t.fee)), see below for SignedToUnsigned. + * - For each direct dependency: + * - VARINT(skip) + * - The position of t in the cluster: VARINT(skip) + * - The end of the graph: VARINT(0) + * + * The list of skip values encodes the dependencies of t, as well as its position in the cluster. + * Each skip value is the number of possibilities that were available, but were not taken. These + * possibilities are, in order: + * - For each previous transaction in the graph, in reverse serialization order, whether it is a + * direct parent of t (but excluding transactions which are already implied to be dependencies + * by parent relations that were serialized before it). + * - The various insertion positions in the cluster, from the very end of the cluster, to the + * front. + * + * Let's say you have a 7-transaction cluster, consisting of transactions F,A,C,B,G,E,D, but + * serialized in order A,B,C,D,E,F,G, because that happens to be a topological ordering. By the + * time G gets serialized, what has been serialized already represents the cluster F,A,C,B,E,D (in + * that order). G has B and E as direct parents, and E depends on C. + * + * In this case, the possibilities are, in order: + * - [ ] the dependency G->F + * - [X] the dependency G->E + * - [ ] the dependency G->D + * - [X] the dependency G->B + * - [ ] the dependency G->A + * - [ ] put G at the end of the cluster + * - [ ] put G before D + * - [X] put G before E + * - [ ] put G before B + * - [ ] put G before C + * - [ ] put G before A + * - [ ] put G before F + * + * The skip values in this case are 1 (G->F), 1 (G->D), 3 (G->A, G at end, G before D). No skip + * after 3 is needed (or permitted), because there can only be one position for G. Also note that + * G->C is not included in the list of possibilities, as it is implied by the included G->E and + * E->C that came before it. On deserialization, if the last skip value was 8 or larger (putting + * G before the beginning of the cluster), it is interpreted as wrapping around back to the end. + * + * + * Rationale: + * - Why VARINTs? They are flexible enough to represent large numbers where needed, but more + * compact for smaller numbers. The serialization format is designed so that simple structures + * involve smaller numbers, so smaller size maps to simpler graphs. + * - Why use SignedToUnsigned? It results in small unsigned values for signed values with small + * absolute value. This way we can encode negative fees in graphs, but still let small negative + * numbers have small encodings. + * - Why are the parents emitted in reverse order compared to the transactions themselves? This + * naturally lets us skip parents-of-parents, as they will be reflected as implied dependencies. + * - Why encode skip values and not a bitmask to convey the list positions? It turns out that the + * most complex graphs (in terms of linearization complexity) are ones with ~1 dependency per + * transaction. The current encoding uses ~1 byte per transaction for dependencies in this case, + * while a bitmask would require ~N/2 bits per transaction. + */ + +struct DepGraphFormatter +{ + /** Convert x>=0 to 2x (even), x<0 to -2x-1 (odd). */ + static uint64_t SignedToUnsigned(int64_t x) noexcept + { + if (x < 0) { + return 2 * uint64_t(-(x + 1)) + 1; + } else { + return 2 * uint64_t(x); + } + } + + /** Convert even x to x/2 (>=0), odd x to -(x/2)-1 (<0). */ + static int64_t UnsignedToSigned(uint64_t x) noexcept + { + if (x & 1) { + return -int64_t(x / 2) - 1; + } else { + return int64_t(x / 2); + } + } + + template + static void Ser(Stream& s, const DepGraph& depgraph) + { + /** Construct a topological order to serialize the transactions in. */ + std::vector topo_order(depgraph.TxCount()); + std::iota(topo_order.begin(), topo_order.end(), ClusterIndex{0}); + std::sort(topo_order.begin(), topo_order.end(), [&](ClusterIndex a, ClusterIndex b) { + auto anc_a = depgraph.Ancestors(a).Count(), anc_b = depgraph.Ancestors(b).Count(); + if (anc_a != anc_b) return anc_a < anc_b; + return a < b; + }); + + /** Which transactions the deserializer already knows when it has deserialized what has + * been serialized here so far, and in what order. */ + std::vector rebuilt_order; + rebuilt_order.reserve(depgraph.TxCount()); + + // Loop over the transactions in topological order. + for (ClusterIndex topo_idx = 0; topo_idx < topo_order.size(); ++topo_idx) { + /** Which depgraph index we are currently writing. */ + ClusterIndex idx = topo_order[topo_idx]; + // Write size, which must be larger than 0. + s << VARINT_MODE(depgraph.FeeRate(idx).size, VarIntMode::NONNEGATIVE_SIGNED); + // Write fee, encoded as an unsigned varint (odd=negative, even=non-negative). + s << VARINT(SignedToUnsigned(depgraph.FeeRate(idx).fee)); + // Write dependency information. + SetType written_parents; + uint64_t diff = 0; //!< How many potential parent/child relations we have skipped over. + for (ClusterIndex dep_dist = 0; dep_dist < topo_idx; ++dep_dist) { + /** Which depgraph index we are currently considering as parent of idx. */ + ClusterIndex dep_idx = topo_order[topo_idx - 1 - dep_dist]; + // Ignore transactions which are already known to be ancestors. + if (depgraph.Descendants(dep_idx).Overlaps(written_parents)) continue; + if (depgraph.Ancestors(idx)[dep_idx]) { + // When an actual parent is encounted, encode how many non-parents were skipped + // before it. + s << VARINT(diff); + diff = 0; + written_parents.Set(dep_idx); + } else { + // When a non-parent is encountered, increment the skip counter. + ++diff; + } + } + // Write position information. + ClusterIndex insert_distance = 0; + while (insert_distance < rebuilt_order.size()) { + // Loop to find how far from the end in rebuilt_order to insert. + if (idx > *(rebuilt_order.end() - 1 - insert_distance)) break; + ++insert_distance; + } + rebuilt_order.insert(rebuilt_order.end() - insert_distance, idx); + s << VARINT(diff + insert_distance); + } + + // Output a final 0 to denote the end of the graph. + s << uint8_t{0}; + } + + template + void Unser(Stream& s, DepGraph& depgraph) + { + /** The dependency graph which we deserialize into first, with transactions in + * topological serialization order, not original cluster order. */ + DepGraph topo_depgraph; + /** Mapping from cluster order to serialization order, used later to reconstruct the + * cluster order. */ + std::vector reordering; + + // Read transactions in topological order. + try { + while (true) { + // Read size. Size 0 signifies the end of the DepGraph. + int32_t size; + s >> VARINT_MODE(size, VarIntMode::NONNEGATIVE_SIGNED); + size &= 0x3FFFFF; // Enough for size up to 4M. + static_assert(0x3FFFFF >= 4000000); + if (size == 0 || topo_depgraph.TxCount() == SetType::Size()) break; + // Read fee, encoded as an unsigned varint (odd=negative, even=non-negative). + uint64_t coded_fee; + s >> VARINT(coded_fee); + coded_fee &= 0xFFFFFFFFFFFFF; // Enough for fee between -21M...21M BTC. + static_assert(0xFFFFFFFFFFFFF > uint64_t{2} * 21000000 * 100000000); + auto fee = UnsignedToSigned(coded_fee); + // Extend topo_depgraph with the new transaction (at the end). + auto topo_idx = topo_depgraph.AddTransaction({fee, size}); + reordering.push_back(topo_idx); + // Read dependency information. + uint64_t diff = 0; //!< How many potential parents we have to skip. + s >> VARINT(diff); + for (ClusterIndex dep_dist = 0; dep_dist < topo_idx; ++dep_dist) { + /** Which topo_depgraph index we are currently considering as parent of topo_idx. */ + ClusterIndex dep_topo_idx = topo_idx - 1 - dep_dist; + // Ignore transactions which are already known ancestors of topo_idx. + if (topo_depgraph.Descendants(dep_topo_idx)[topo_idx]) continue; + if (diff == 0) { + // When the skip counter has reached 0, add an actual dependency. + topo_depgraph.AddDependency(dep_topo_idx, topo_idx); + // And read the number of skips after it. + s >> VARINT(diff); + } else { + // Otherwise, dep_topo_idx is not a parent. Decrement and continue. + --diff; + } + } + // If we reach this point, we can interpret the remaining skip value as how far from the + // end of reordering topo_idx should be placed (wrapping around), so move it to its + // correct location. The preliminary reordering.push_back(topo_idx) above was to make + // sure that if a deserialization exception occurs, topo_idx still appears somewhere. + reordering.pop_back(); + reordering.insert(reordering.end() - (diff % (reordering.size() + 1)), topo_idx); + } + } catch (const std::ios_base::failure&) {} + + // Construct the original cluster order depgraph. + depgraph = {}; + // Add transactions to depgraph in the original cluster order. + for (auto topo_idx : reordering) { + depgraph.AddTransaction(topo_depgraph.FeeRate(topo_idx)); + } + // Translate dependencies from topological to cluster order. + for (ClusterIndex idx = 0; idx < reordering.size(); ++idx) { + ClusterIndex topo_idx = reordering[idx]; + for (ClusterIndex dep_idx = 0; dep_idx < reordering.size(); ++dep_idx) { + ClusterIndex dep_topo_idx = reordering[dep_idx]; + if (topo_depgraph.Ancestors(topo_idx)[dep_topo_idx]) { + depgraph.AddDependency(dep_idx, idx); + } + } + } + } +}; + +/** Perform a sanity/consistency check on a DepGraph. */ +template +void SanityCheck(const DepGraph& depgraph) +{ + // Consistency check between ancestors internally. + for (ClusterIndex i = 0; i < depgraph.TxCount(); ++i) { + // Transactions include themselves as ancestors. + assert(depgraph.Ancestors(i)[i]); + // If a is an ancestor of b, then b's ancestors must include all of a's ancestors. + for (auto a : depgraph.Ancestors(i)) { + assert(depgraph.Ancestors(i).IsSupersetOf(depgraph.Ancestors(a))); + } + } + // Consistency check between ancestors and descendants. + for (ClusterIndex i = 0; i < depgraph.TxCount(); ++i) { + for (ClusterIndex j = 0; j < depgraph.TxCount(); ++j) { + assert(depgraph.Ancestors(i)[j] == depgraph.Descendants(j)[i]); + } + } + // If DepGraph is acyclic, serialize + deserialize must roundtrip. + if (IsAcyclic(depgraph)) { + std::vector ser; + VectorWriter writer(ser, 0); + writer << Using(depgraph); + SpanReader reader(ser); + DepGraph decoded_depgraph; + reader >> Using(decoded_depgraph); + assert(depgraph == decoded_depgraph); + assert(reader.empty()); + // It must also deserialize correctly without the terminal 0 byte (as the deserializer + // will upon EOF still return what it read so far). + assert(ser.size() >= 1 && ser.back() == 0); + ser.pop_back(); + reader = SpanReader{ser}; + decoded_depgraph = {}; + reader >> Using(decoded_depgraph); + assert(depgraph == decoded_depgraph); + assert(reader.empty()); + } +} + +/** Verify that a DepGraph corresponds to the information in a cluster. */ +template +void VerifyDepGraphFromCluster(const Cluster& cluster, const DepGraph& depgraph) +{ + // Sanity check the depgraph, which includes a check for correspondence between ancestors and + // descendants, so it suffices to check just ancestors below. + SanityCheck(depgraph); + // Verify transaction count. + assert(cluster.size() == depgraph.TxCount()); + // Verify feerates. + for (ClusterIndex i = 0; i < depgraph.TxCount(); ++i) { + assert(depgraph.FeeRate(i) == cluster[i].first); + } + // Verify ancestors. + for (ClusterIndex i = 0; i < depgraph.TxCount(); ++i) { + // Start with the transaction having itself as ancestor. + auto ancestors = SetType::Singleton(i); + // Add parents of ancestors to the set of ancestors until it stops changing. + while (true) { + const auto old_ancestors = ancestors; + for (auto ancestor : ancestors) { + ancestors |= cluster[ancestor].second; + } + if (old_ancestors == ancestors) break; + } + // Compare against depgraph. + assert(depgraph.Ancestors(i) == ancestors); + // Some additional sanity tests: + // - Every transaction has itself as ancestor. + assert(ancestors[i]); + // - Every transaction has its direct parents as ancestors. + for (auto parent : cluster[i].second) { + assert(ancestors[parent]); + } + } +} + +/** Perform a sanity check on a linearization. */ +template +void SanityCheck(const DepGraph& depgraph, Span linearization) +{ + // Check completeness. + assert(linearization.size() == depgraph.TxCount()); + TestBitSet done; + for (auto i : linearization) { + // Check transaction position is in range. + assert(i < depgraph.TxCount()); + // Check topology and lack of duplicates. + assert((depgraph.Ancestors(i) - done) == TestBitSet::Singleton(i)); + done.Set(i); + } +} + +} // namespace + +#endif // BITCOIN_TEST_UTIL_CLUSTER_LINEARIZE_H