2022-12-24 23:49:50 +00:00
// Copyright (c) 2016-2022 The Bitcoin Core developers
2016-04-15 12:23:57 -07:00
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
2018-03-22 15:19:44 +01:00
# ifndef BITCOIN_BLOCKENCODINGS_H
# define BITCOIN_BLOCKENCODINGS_H
2016-04-15 12:23:57 -07:00
2017-11-10 13:57:53 +13:00
# include <primitives/block.h>
2016-04-15 12:23:57 -07:00
2023-01-16 16:13:52 +01:00
# include <functional>
2016-04-15 12:23:57 -07:00
class CTxMemPool ;
2023-01-16 16:13:52 +01:00
class BlockValidationState ;
namespace Consensus {
struct Params ;
} ;
2016-04-15 12:23:57 -07:00
2020-02-03 19:41:42 -08:00
// Transaction compression schemes for compact block relay can be introduced by writing
// an actual formatter here.
using TransactionCompression = DefaultFormatter ;
2016-04-15 12:23:57 -07:00
2020-02-15 19:05:11 -08:00
class DifferenceFormatter
{
uint64_t m_shift = 0 ;
public :
template < typename Stream , typename I >
void Ser ( Stream & s , I v )
{
if ( v < m_shift | | v > = std : : numeric_limits < uint64_t > : : max ( ) ) throw std : : ios_base : : failure ( " differential value overflow " ) ;
WriteCompactSize ( s , v - m_shift ) ;
m_shift = uint64_t ( v ) + 1 ;
}
template < typename Stream , typename I >
void Unser ( Stream & s , I & v )
{
uint64_t n = ReadCompactSize ( s ) ;
m_shift + = n ;
if ( m_shift < n | | m_shift > = std : : numeric_limits < uint64_t > : : max ( ) | | m_shift < std : : numeric_limits < I > : : min ( ) | | m_shift > std : : numeric_limits < I > : : max ( ) ) throw std : : ios_base : : failure ( " differential value overflow " ) ;
v = I ( m_shift + + ) ;
}
} ;
2016-04-15 12:23:57 -07:00
class BlockTransactionsRequest {
public :
// A BlockTransactionsRequest message
uint256 blockhash ;
std : : vector < uint16_t > indexes ;
2020-02-03 19:41:42 -08:00
SERIALIZE_METHODS ( BlockTransactionsRequest , obj )
{
READWRITE ( obj . blockhash , Using < VectorFormatter < DifferenceFormatter > > ( obj . indexes ) ) ;
2016-04-15 12:23:57 -07:00
}
} ;
class BlockTransactions {
public :
// A BlockTransactions message
uint256 blockhash ;
2016-11-10 17:34:17 -08:00
std : : vector < CTransactionRef > txn ;
2016-04-15 12:23:57 -07:00
2024-07-08 11:11:58 +02:00
BlockTransactions ( ) = default ;
2017-08-01 12:22:41 +02:00
explicit BlockTransactions ( const BlockTransactionsRequest & req ) :
2016-04-15 12:23:57 -07:00
blockhash ( req . blockhash ) , txn ( req . indexes . size ( ) ) { }
2020-02-03 19:41:42 -08:00
SERIALIZE_METHODS ( BlockTransactions , obj )
{
2023-09-07 19:16:57 +10:00
READWRITE ( obj . blockhash , TX_WITH_WITNESS ( Using < VectorFormatter < TransactionCompression > > ( obj . txn ) ) ) ;
2016-04-15 12:23:57 -07:00
}
} ;
2017-01-18 16:15:37 +01:00
// Dumb serialization/storage-helper for CBlockHeaderAndShortTxIDs and PartiallyDownloadedBlock
2016-04-15 12:23:57 -07:00
struct PrefilledTransaction {
// Used as an offset since last prefilled tx in CBlockHeaderAndShortTxIDs,
// as a proper transaction-in-block-index in PartiallyDownloadedBlock
uint16_t index ;
2016-11-10 17:34:17 -08:00
CTransactionRef tx ;
2016-04-15 12:23:57 -07:00
2023-09-07 19:16:57 +10:00
SERIALIZE_METHODS ( PrefilledTransaction , obj ) { READWRITE ( COMPACTSIZE ( obj . index ) , TX_WITH_WITNESS ( Using < TransactionCompression > ( obj . tx ) ) ) ; }
2016-04-15 12:23:57 -07:00
} ;
typedef enum ReadStatus_t
{
READ_STATUS_OK ,
READ_STATUS_INVALID , // Invalid object, peer is sending bogus crap
READ_STATUS_FAILED , // Failed to process object
2016-10-31 10:03:49 -04:00
READ_STATUS_CHECKBLOCK_FAILED , // Used only by FillBlock to indicate a
// failure in CheckBlock.
2016-04-15 12:23:57 -07:00
} ReadStatus ;
class CBlockHeaderAndShortTxIDs {
private :
mutable uint64_t shorttxidk0 , shorttxidk1 ;
uint64_t nonce ;
void FillShortTxIDSelector ( ) const ;
friend class PartiallyDownloadedBlock ;
protected :
std : : vector < uint64_t > shorttxids ;
std : : vector < PrefilledTransaction > prefilledtxn ;
public :
2020-03-11 09:35:39 -07:00
static constexpr int SHORTTXIDS_LENGTH = 6 ;
2016-04-15 12:23:57 -07:00
CBlockHeader header ;
test: Make blockencodings_tests deterministic
refactor: CBlockHeaderAndShortTxIDs constructor now always takes an explicit nonce.
test: Make blockencodings_tests deterministic using fixed seed providing deterministic
CBlockHeaderAndShortTxID nonces and dummy transaction IDs.
Fixes very rare flaky test failures, where the ShortIDs of test transactions collide, leading to
`READ_STATUS_FAILED` from PartiallyDownloadedBlock::InitData and/or `IsTxAvailable` giving `false`
when the transaction should actually be available.
* Use a new `FastRandomContext` with a fixed seed in each test, to ensure 'random' uint256s
used as fake prevouts are deterministic, so in-turn test txids and short IDs are deterministic
and don't collide causing very rare but flaky test failures.
* Add new test-only/internal initializer for `CBlockHeaderAndShortTxIDs` that takes a specified
nonce to further ensure determinism and avoid rare but undesireable short ID collisions.
In a test context this nonce is set to a fixed known-good value. Normally it is random, as
previously.
Flaky test failures can be reproduced with:
```patch
diff --git a/src/blockencodings.cpp b/src/blockencodings.cpp
index 695e8d806a..64d635a97a 100644
--- a/src/blockencodings.cpp
+++ b/src/blockencodings.cpp
@@ -44,7 +44,8 @@ void CBlockHeaderAndShortTxIDs::FillShortTxIDSelector() const {
uint64_t CBlockHeaderAndShortTxIDs::GetShortID(const Wtxid& wtxid) const {
static_assert(SHORTTXIDS_LENGTH == 6, "shorttxids calculation assumes 6-byte shorttxids");
- return SipHashUint256(shorttxidk0, shorttxidk1, wtxid) & 0xffffffffffffL;
+ // return SipHashUint256(shorttxidk0, shorttxidk1, wtxid) & 0xffffffffffffL;
+ return SipHashUint256(shorttxidk0, shorttxidk1, wtxid) & 0x0f;
}
```
to increase the likelihood of a short ID collision; and running
```shell
set -e;
n=0;
while (( n++ < 5000 )); do
src/test/test_bitcoin --run_test=blockencodings_tests;
done
```
2024-06-12 22:09:15 +01:00
/**
* Dummy for deserialization
*/
2024-07-08 11:11:58 +02:00
CBlockHeaderAndShortTxIDs ( ) = default ;
2016-04-15 12:23:57 -07:00
test: Make blockencodings_tests deterministic
refactor: CBlockHeaderAndShortTxIDs constructor now always takes an explicit nonce.
test: Make blockencodings_tests deterministic using fixed seed providing deterministic
CBlockHeaderAndShortTxID nonces and dummy transaction IDs.
Fixes very rare flaky test failures, where the ShortIDs of test transactions collide, leading to
`READ_STATUS_FAILED` from PartiallyDownloadedBlock::InitData and/or `IsTxAvailable` giving `false`
when the transaction should actually be available.
* Use a new `FastRandomContext` with a fixed seed in each test, to ensure 'random' uint256s
used as fake prevouts are deterministic, so in-turn test txids and short IDs are deterministic
and don't collide causing very rare but flaky test failures.
* Add new test-only/internal initializer for `CBlockHeaderAndShortTxIDs` that takes a specified
nonce to further ensure determinism and avoid rare but undesireable short ID collisions.
In a test context this nonce is set to a fixed known-good value. Normally it is random, as
previously.
Flaky test failures can be reproduced with:
```patch
diff --git a/src/blockencodings.cpp b/src/blockencodings.cpp
index 695e8d806a..64d635a97a 100644
--- a/src/blockencodings.cpp
+++ b/src/blockencodings.cpp
@@ -44,7 +44,8 @@ void CBlockHeaderAndShortTxIDs::FillShortTxIDSelector() const {
uint64_t CBlockHeaderAndShortTxIDs::GetShortID(const Wtxid& wtxid) const {
static_assert(SHORTTXIDS_LENGTH == 6, "shorttxids calculation assumes 6-byte shorttxids");
- return SipHashUint256(shorttxidk0, shorttxidk1, wtxid) & 0xffffffffffffL;
+ // return SipHashUint256(shorttxidk0, shorttxidk1, wtxid) & 0xffffffffffffL;
+ return SipHashUint256(shorttxidk0, shorttxidk1, wtxid) & 0x0f;
}
```
to increase the likelihood of a short ID collision; and running
```shell
set -e;
n=0;
while (( n++ < 5000 )); do
src/test/test_bitcoin --run_test=blockencodings_tests;
done
```
2024-06-12 22:09:15 +01:00
/**
* @ param [ in ] nonce This should be randomly generated , and is used for the siphash secret key
*/
CBlockHeaderAndShortTxIDs ( const CBlock & block , const uint64_t nonce ) ;
2016-04-15 12:23:57 -07:00
2024-03-22 21:53:41 +00:00
uint64_t GetShortID ( const Wtxid & wtxid ) const ;
2016-04-15 12:23:57 -07:00
size_t BlockTxCount ( ) const { return shorttxids . size ( ) + prefilledtxn . size ( ) ; }
2020-02-03 19:41:42 -08:00
SERIALIZE_METHODS ( CBlockHeaderAndShortTxIDs , obj )
{
READWRITE ( obj . header , obj . nonce , Using < VectorFormatter < CustomUintFormatter < SHORTTXIDS_LENGTH > > > ( obj . shorttxids ) , obj . prefilledtxn ) ;
2016-04-15 12:23:57 -07:00
if ( ser_action . ForRead ( ) ) {
2020-02-03 19:41:42 -08:00
if ( obj . BlockTxCount ( ) > std : : numeric_limits < uint16_t > : : max ( ) ) {
throw std : : ios_base : : failure ( " indexes overflowed 16 bits " ) ;
2016-04-15 12:23:57 -07:00
}
2020-02-03 19:41:42 -08:00
obj . FillShortTxIDSelector ( ) ;
2016-04-15 12:23:57 -07:00
}
}
} ;
class PartiallyDownloadedBlock {
protected :
2016-11-10 17:34:17 -08:00
std : : vector < CTransactionRef > txn_available ;
2017-01-12 12:19:14 -08:00
size_t prefilled_count = 0 , mempool_count = 0 , extra_count = 0 ;
2020-05-28 09:55:39 +03:00
const CTxMemPool * pool ;
2016-04-15 12:23:57 -07:00
public :
CBlockHeader header ;
2023-01-16 16:13:52 +01:00
2023-02-15 14:03:37 -08:00
// Can be overridden for testing
2023-01-16 16:13:52 +01:00
using CheckBlockFn = std : : function < bool ( const CBlock & , BlockValidationState & , const Consensus : : Params & , bool , bool ) > ;
CheckBlockFn m_check_block_mock { nullptr } ;
2017-08-01 12:22:41 +02:00
explicit PartiallyDownloadedBlock ( CTxMemPool * poolIn ) : pool ( poolIn ) { }
2016-04-15 12:23:57 -07:00
2024-06-04 19:27:30 +01:00
// extra_txn is a list of extra orphan/conflicted/etc transactions to look at
2024-03-25 20:13:35 +01:00
ReadStatus InitData ( const CBlockHeaderAndShortTxIDs & cmpctblock , const std : : vector < CTransactionRef > & extra_txn ) ;
2016-04-15 12:23:57 -07:00
bool IsTxAvailable ( size_t index ) const ;
2016-11-11 13:01:27 -08:00
ReadStatus FillBlock ( CBlock & block , const std : : vector < CTransactionRef > & vtx_missing ) ;
2016-04-15 12:23:57 -07:00
} ;
2018-03-22 15:19:44 +01:00
# endif // BITCOIN_BLOCKENCODINGS_H