From 053930ffc41ba33fe7ce26bde7097951fe0b8462 Mon Sep 17 00:00:00 2001 From: Patrick Strateman Date: Mon, 23 May 2016 00:21:05 -0700 Subject: [PATCH 1/5] Avoid recalculating vchKeyedNetGroup in eviction logic. Lazy calculate vchKeyedNetGroup in CNode::GetKeyedNetGroup. --- src/net.cpp | 43 +++++++++---------------------------------- src/net.h | 20 ++++++++++++++++++++ 2 files changed, 29 insertions(+), 34 deletions(-) diff --git a/src/net.cpp b/src/net.cpp index c09e3aedb6b..eb62ee8a06e 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -14,6 +14,7 @@ #include "clientversion.h" #include "consensus/consensus.h" #include "crypto/common.h" +#include "crypto/sha256.h" #include "hash.h" #include "primitives/transaction.h" #include "scheduler.h" @@ -838,6 +839,7 @@ struct NodeEvictionCandidate int64_t nTimeConnected; int64_t nMinPingUsecTime; CAddress addr; + std::vector vchKeyedNetGroup; }; static bool ReverseCompareNodeMinPingTime(const NodeEvictionCandidate &a, const NodeEvictionCandidate &b) @@ -850,36 +852,8 @@ static bool ReverseCompareNodeTimeConnected(const NodeEvictionCandidate &a, cons return a.nTimeConnected > b.nTimeConnected; } -class CompareNetGroupKeyed -{ - std::vector vchSecretKey; -public: - CompareNetGroupKeyed() - { - vchSecretKey.resize(32, 0); - GetRandBytes(vchSecretKey.data(), vchSecretKey.size()); - } - - bool operator()(const NodeEvictionCandidate &a, const NodeEvictionCandidate &b) - { - std::vector vchGroupA, vchGroupB; - CSHA256 hashA, hashB; - std::vector vchA(32), vchB(32); - - vchGroupA = a.addr.GetGroup(); - vchGroupB = b.addr.GetGroup(); - - hashA.Write(begin_ptr(vchGroupA), vchGroupA.size()); - hashB.Write(begin_ptr(vchGroupB), vchGroupB.size()); - - hashA.Write(begin_ptr(vchSecretKey), vchSecretKey.size()); - hashB.Write(begin_ptr(vchSecretKey), vchSecretKey.size()); - - hashA.Finalize(begin_ptr(vchA)); - hashB.Finalize(begin_ptr(vchB)); - - return vchA < vchB; - } +static bool CompareNetGroupKeyed(const NodeEvictionCandidate &a, const NodeEvictionCandidate &b) { + return a.vchKeyedNetGroup < b.vchKeyedNetGroup; }; /** Try to find a connection to evict when the node is full. @@ -902,7 +876,7 @@ static bool AttemptToEvictConnection(bool fPreferNewConnection) { continue; if (node->fDisconnect) continue; - NodeEvictionCandidate candidate = {node->id, node->nTimeConnected, node->nMinPingUsecTime, node->addr}; + NodeEvictionCandidate candidate = {node->id, node->nTimeConnected, node->nMinPingUsecTime, node->addr, node->vchKeyedNetGroup}; vEvictionCandidates.push_back(candidate); } } @@ -912,9 +886,8 @@ static bool AttemptToEvictConnection(bool fPreferNewConnection) { // Protect connections with certain characteristics // Deterministically select 4 peers to protect by netgroup. - // An attacker cannot predict which netgroups will be protected. - static CompareNetGroupKeyed comparerNetGroupKeyed; - std::sort(vEvictionCandidates.begin(), vEvictionCandidates.end(), comparerNetGroupKeyed); + // An attacker cannot predict which netgroups will be protected + std::sort(vEvictionCandidates.begin(), vEvictionCandidates.end(), CompareNetGroupKeyed); vEvictionCandidates.erase(vEvictionCandidates.end() - std::min(4, static_cast(vEvictionCandidates.size())), vEvictionCandidates.end()); if (vEvictionCandidates.empty()) return false; @@ -2392,6 +2365,8 @@ CNode::CNode(SOCKET hSocketIn, const CAddress& addrIn, const std::string& addrNa lastSentFeeFilter = 0; nextSendTimeFeeFilter = 0; + CalculateKeyedNetGroup(); + BOOST_FOREACH(const std::string &msg, getAllNetMessageTypes()) mapRecvBytesPerMsgCmd[msg] = 0; mapRecvBytesPerMsgCmd[NET_MESSAGE_COMMAND_OTHER] = 0; diff --git a/src/net.h b/src/net.h index 403653e8c83..019a3f7ee3e 100644 --- a/src/net.h +++ b/src/net.h @@ -9,6 +9,8 @@ #include "amount.h" #include "bloom.h" #include "compat.h" +#include "crypto/common.h" +#include "crypto/sha256.h" #include "limitedmap.h" #include "netbase.h" #include "protocol.h" @@ -362,6 +364,8 @@ public: CBloomFilter* pfilter; int nRefCount; NodeId id; + + std::vector vchKeyedNetGroup; protected: // Denial-of-service detection/prevention @@ -450,6 +454,22 @@ private: CNode(const CNode&); void operator=(const CNode&); + void CalculateKeyedNetGroup() { + static std::vector vchSecretKey; + if (vchSecretKey.empty()) { + vchSecretKey.resize(32, 0); + GetRandBytes(vchSecretKey.data(), vchSecretKey.size()); + } + + std::vector vchNetGroup(this->addr.GetGroup()); + + CSHA256 hash; + hash.Write(begin_ptr(vchNetGroup), vchNetGroup.size()); + hash.Write(begin_ptr(vchSecretKey), vchSecretKey.size()); + + vchKeyedNetGroup.resize(32, 0); + hash.Finalize(begin_ptr(vchKeyedNetGroup)); + } public: NodeId GetId() const { From 9bf156bb9ee4915fa7e8f84589d65f6598563c97 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Sun, 22 May 2016 11:06:18 +0200 Subject: [PATCH 2/5] Support SipHash with arbitrary byte writes --- src/hash.cpp | 39 ++++++++++++++++++++++++++++++++++++--- src/hash.h | 21 ++++++++++++++++++++- src/test/hash_tests.cpp | 17 +++++++++++++---- 3 files changed, 69 insertions(+), 8 deletions(-) diff --git a/src/hash.cpp b/src/hash.cpp index a518314a537..20a83342db4 100644 --- a/src/hash.cpp +++ b/src/hash.cpp @@ -100,12 +100,15 @@ CSipHasher::CSipHasher(uint64_t k0, uint64_t k1) v[2] = 0x6c7967656e657261ULL ^ k0; v[3] = 0x7465646279746573ULL ^ k1; count = 0; + tmp = 0; } CSipHasher& CSipHasher::Write(uint64_t data) { uint64_t v0 = v[0], v1 = v[1], v2 = v[2], v3 = v[3]; + assert(count % 8 == 0); + v3 ^= data; SIPROUND; SIPROUND; @@ -116,7 +119,35 @@ CSipHasher& CSipHasher::Write(uint64_t data) v[2] = v2; v[3] = v3; - count++; + count += 8; + return *this; +} + +CSipHasher& CSipHasher::Write(const unsigned char* data, size_t size) +{ + uint64_t v0 = v[0], v1 = v[1], v2 = v[2], v3 = v[3]; + uint64_t t = tmp; + int c = count; + + while (size--) { + t |= ((uint64_t)(*(data++))) << (8 * (c % 8)); + c++; + if ((c & 7) == 0) { + v3 ^= t; + SIPROUND; + SIPROUND; + v0 ^= t; + t = 0; + } + } + + v[0] = v0; + v[1] = v1; + v[2] = v2; + v[3] = v3; + count = c; + tmp = t; + return *this; } @@ -124,10 +155,12 @@ uint64_t CSipHasher::Finalize() const { uint64_t v0 = v[0], v1 = v[1], v2 = v[2], v3 = v[3]; - v3 ^= ((uint64_t)count) << 59; + uint64_t t = tmp | (((uint64_t)count) << 56); + + v3 ^= t; SIPROUND; SIPROUND; - v0 ^= ((uint64_t)count) << 59; + v0 ^= t; v2 ^= 0xFF; SIPROUND; SIPROUND; diff --git a/src/hash.h b/src/hash.h index 600dabec560..db4e130ae7f 100644 --- a/src/hash.h +++ b/src/hash.h @@ -171,19 +171,38 @@ unsigned int MurmurHash3(unsigned int nHashSeed, const std::vector Date: Wed, 25 May 2016 15:38:32 +0200 Subject: [PATCH 3/5] Use 64-bit SipHash of netgroups in eviction --- src/net.cpp | 39 ++++++++++++++++++++++++++------------- src/net.h | 22 +++------------------- 2 files changed, 29 insertions(+), 32 deletions(-) diff --git a/src/net.cpp b/src/net.cpp index eb62ee8a06e..0bc501601dd 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -839,7 +839,7 @@ struct NodeEvictionCandidate int64_t nTimeConnected; int64_t nMinPingUsecTime; CAddress addr; - std::vector vchKeyedNetGroup; + uint64_t nKeyedNetGroup; }; static bool ReverseCompareNodeMinPingTime(const NodeEvictionCandidate &a, const NodeEvictionCandidate &b) @@ -853,7 +853,7 @@ static bool ReverseCompareNodeTimeConnected(const NodeEvictionCandidate &a, cons } static bool CompareNetGroupKeyed(const NodeEvictionCandidate &a, const NodeEvictionCandidate &b) { - return a.vchKeyedNetGroup < b.vchKeyedNetGroup; + return a.nKeyedNetGroup < b.nKeyedNetGroup; }; /** Try to find a connection to evict when the node is full. @@ -876,7 +876,7 @@ static bool AttemptToEvictConnection(bool fPreferNewConnection) { continue; if (node->fDisconnect) continue; - NodeEvictionCandidate candidate = {node->id, node->nTimeConnected, node->nMinPingUsecTime, node->addr, node->vchKeyedNetGroup}; + NodeEvictionCandidate candidate = {node->id, node->nTimeConnected, node->nMinPingUsecTime, node->addr, node->nKeyedNetGroup}; vEvictionCandidates.push_back(candidate); } } @@ -908,24 +908,24 @@ static bool AttemptToEvictConnection(bool fPreferNewConnection) { // Identify the network group with the most connections and youngest member. // (vEvictionCandidates is already sorted by reverse connect time) - std::vector naMostConnections; + uint64_t naMostConnections; unsigned int nMostConnections = 0; int64_t nMostConnectionsTime = 0; - std::map, std::vector > mapAddrCounts; + std::map > mapAddrCounts; BOOST_FOREACH(const NodeEvictionCandidate &node, vEvictionCandidates) { - mapAddrCounts[node.addr.GetGroup()].push_back(node); - int64_t grouptime = mapAddrCounts[node.addr.GetGroup()][0].nTimeConnected; - size_t groupsize = mapAddrCounts[node.addr.GetGroup()].size(); + mapAddrCounts[node.nKeyedNetGroup].push_back(node); + int64_t grouptime = mapAddrCounts[node.nKeyedNetGroup][0].nTimeConnected; + size_t groupsize = mapAddrCounts[node.nKeyedNetGroup].size(); if (groupsize > nMostConnections || (groupsize == nMostConnections && grouptime > nMostConnectionsTime)) { nMostConnections = groupsize; nMostConnectionsTime = grouptime; - naMostConnections = node.addr.GetGroup(); + naMostConnections = node.nKeyedNetGroup; } } // Reduce to the network group with the most connections - vEvictionCandidates = mapAddrCounts[naMostConnections]; + vEvictionCandidates = std::move(mapAddrCounts[naMostConnections]); // Do not disconnect peers if there is only one unprotected connection from their network group. // This step excessively favors netgroup diversity, and should be removed once more protective criteria are established. @@ -2318,6 +2318,8 @@ unsigned int SendBufferSize() { return 1000*GetArg("-maxsendbuffer", DEFAULT_MAX CNode::CNode(SOCKET hSocketIn, const CAddress& addrIn, const std::string& addrNameIn, bool fInboundIn) : ssSend(SER_NETWORK, INIT_PROTO_VERSION), + addr(addrIn), + nKeyedNetGroup(CalculateKeyedNetGroup(addrIn)), addrKnown(5000, 0.001), filterInventoryKnown(50000, 0.000001) { @@ -2330,7 +2332,6 @@ CNode::CNode(SOCKET hSocketIn, const CAddress& addrIn, const std::string& addrNa nRecvBytes = 0; nTimeConnected = GetTime(); nTimeOffset = 0; - addr = addrIn; addrName = addrNameIn == "" ? addr.ToStringIPPort() : addrNameIn; nVersion = 0; strSubVer = ""; @@ -2365,8 +2366,6 @@ CNode::CNode(SOCKET hSocketIn, const CAddress& addrIn, const std::string& addrNa lastSentFeeFilter = 0; nextSendTimeFeeFilter = 0; - CalculateKeyedNetGroup(); - BOOST_FOREACH(const std::string &msg, getAllNetMessageTypes()) mapRecvBytesPerMsgCmd[msg] = 0; mapRecvBytesPerMsgCmd[NET_MESSAGE_COMMAND_OTHER] = 0; @@ -2599,3 +2598,17 @@ bool CBanDB::Read(banmap_t& banSet) int64_t PoissonNextSend(int64_t nNow, int average_interval_seconds) { return nNow + (int64_t)(log1p(GetRand(1ULL << 48) * -0.0000000000000035527136788 /* -1/2^48 */) * average_interval_seconds * -1000000.0 + 0.5); } + +/* static */ uint64_t CNode::CalculateKeyedNetGroup(const CAddress& ad) +{ + static uint64_t k0 = 0, k1 = 0; + while (k0 == 0 && k1 == 0) { + // Make sure this only runs on the first invocation. + GetRandBytes((unsigned char*)&k0, sizeof(k0)); + GetRandBytes((unsigned char*)&k1, sizeof(k1)); + } + + std::vector vchNetGroup(ad.GetGroup()); + + return CSipHasher(k0, k1).Write(&vchNetGroup[0], vchNetGroup.size()).Finalize(); +} diff --git a/src/net.h b/src/net.h index 019a3f7ee3e..8df3cb421fe 100644 --- a/src/net.h +++ b/src/net.h @@ -9,8 +9,6 @@ #include "amount.h" #include "bloom.h" #include "compat.h" -#include "crypto/common.h" -#include "crypto/sha256.h" #include "limitedmap.h" #include "netbase.h" #include "protocol.h" @@ -337,7 +335,7 @@ public: int64_t nLastRecv; int64_t nTimeConnected; int64_t nTimeOffset; - CAddress addr; + const CAddress addr; std::string addrName; CService addrLocal; int nVersion; @@ -365,7 +363,7 @@ public: int nRefCount; NodeId id; - std::vector vchKeyedNetGroup; + const uint64_t nKeyedNetGroup; protected: // Denial-of-service detection/prevention @@ -454,22 +452,8 @@ private: CNode(const CNode&); void operator=(const CNode&); - void CalculateKeyedNetGroup() { - static std::vector vchSecretKey; - if (vchSecretKey.empty()) { - vchSecretKey.resize(32, 0); - GetRandBytes(vchSecretKey.data(), vchSecretKey.size()); - } + static uint64_t CalculateKeyedNetGroup(const CAddress& ad); - std::vector vchNetGroup(this->addr.GetGroup()); - - CSHA256 hash; - hash.Write(begin_ptr(vchNetGroup), vchNetGroup.size()); - hash.Write(begin_ptr(vchSecretKey), vchSecretKey.size()); - - vchKeyedNetGroup.resize(32, 0); - hash.Finalize(begin_ptr(vchKeyedNetGroup)); - } public: NodeId GetId() const { From 888483098e60f2a944f1d246bbfec4d14a2975f8 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Tue, 7 Jun 2016 16:29:03 +0200 Subject: [PATCH 4/5] Use C++11 thread-safe static initializers --- src/coins.cpp | 6 +----- src/coins.h | 2 +- src/main.cpp | 7 ++----- src/net.cpp | 8 ++------ 4 files changed, 6 insertions(+), 17 deletions(-) diff --git a/src/coins.cpp b/src/coins.cpp index b7dd293d695..39db7dedfbb 100644 --- a/src/coins.cpp +++ b/src/coins.cpp @@ -56,11 +56,7 @@ void CCoinsViewBacked::SetBackend(CCoinsView &viewIn) { base = &viewIn; } bool CCoinsViewBacked::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) { return base->BatchWrite(mapCoins, hashBlock); } CCoinsViewCursor *CCoinsViewBacked::Cursor() const { return base->Cursor(); } -SaltedTxidHasher::SaltedTxidHasher() -{ - GetRandBytes((unsigned char*)&k0, sizeof(k0)); - GetRandBytes((unsigned char*)&k1, sizeof(k1)); -} +SaltedTxidHasher::SaltedTxidHasher() : k0(GetRand(std::numeric_limits::max())), k1(GetRand(std::numeric_limits::max())) {} CCoinsViewCache::CCoinsViewCache(CCoinsView *baseIn) : CCoinsViewBacked(baseIn), hasModifier(false), cachedCoinsUsage(0) { } diff --git a/src/coins.h b/src/coins.h index 1dd908700b1..033651a435c 100644 --- a/src/coins.h +++ b/src/coins.h @@ -269,7 +269,7 @@ class SaltedTxidHasher { private: /** Salt */ - uint64_t k0, k1; + const uint64_t k0, k1; public: SaltedTxidHasher(); diff --git a/src/main.cpp b/src/main.cpp index bf6e6d04b51..da140cffacf 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -4784,11 +4784,8 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, LOCK(cs_vNodes); // Use deterministic randomness to send to the same nodes for 24 hours // at a time so the addrKnowns of the chosen nodes prevent repeats - static uint64_t salt0 = 0, salt1 = 0; - while (salt0 == 0 && salt1 == 0) { - GetRandBytes((unsigned char*)&salt0, sizeof(salt0)); - GetRandBytes((unsigned char*)&salt1, sizeof(salt1)); - } + static const uint64_t salt0 = GetRand(std::numeric_limits::max()); + static const uint64_t salt1 = GetRand(std::numeric_limits::max()); uint64_t hashAddr = addr.GetHash(); multimap mapMix; const CSipHasher hasher = CSipHasher(salt0, salt1).Write(hashAddr << 32).Write((GetTime() + hashAddr) / (24*60*60)); diff --git a/src/net.cpp b/src/net.cpp index 0bc501601dd..e29053cf548 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -2601,12 +2601,8 @@ int64_t PoissonNextSend(int64_t nNow, int average_interval_seconds) { /* static */ uint64_t CNode::CalculateKeyedNetGroup(const CAddress& ad) { - static uint64_t k0 = 0, k1 = 0; - while (k0 == 0 && k1 == 0) { - // Make sure this only runs on the first invocation. - GetRandBytes((unsigned char*)&k0, sizeof(k0)); - GetRandBytes((unsigned char*)&k1, sizeof(k1)); - } + static const uint64_t k0 = GetRand(std::numeric_limits::max()); + static const uint64_t k1 = GetRand(std::numeric_limits::max()); std::vector vchNetGroup(ad.GetGroup()); From eebc23218758d89bce0b7cb0eced9fd654cd3d15 Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Wed, 8 Jun 2016 15:49:27 +0200 Subject: [PATCH 5/5] test: Add more test vectors for siphash Add full test vectors from spec, test per byte and per 8 bytes. Builds on #8086. --- src/test/hash_tests.cpp | 48 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/src/test/hash_tests.cpp b/src/test/hash_tests.cpp index f62080c1d24..82d61209b5e 100644 --- a/src/test/hash_tests.cpp +++ b/src/test/hash_tests.cpp @@ -47,6 +47,38 @@ BOOST_AUTO_TEST_CASE(murmurhash3) #undef T } +/* + SipHash-2-4 output with + k = 00 01 02 ... + and + in = (empty string) + in = 00 (1 byte) + in = 00 01 (2 bytes) + in = 00 01 02 (3 bytes) + ... + in = 00 01 02 ... 3e (63 bytes) + + from: https://131002.net/siphash/siphash24.c +*/ +uint64_t siphash_4_2_testvec[] = { + 0x726fdb47dd0e0e31, 0x74f839c593dc67fd, 0x0d6c8009d9a94f5a, 0x85676696d7fb7e2d, + 0xcf2794e0277187b7, 0x18765564cd99a68d, 0xcbc9466e58fee3ce, 0xab0200f58b01d137, + 0x93f5f5799a932462, 0x9e0082df0ba9e4b0, 0x7a5dbbc594ddb9f3, 0xf4b32f46226bada7, + 0x751e8fbc860ee5fb, 0x14ea5627c0843d90, 0xf723ca908e7af2ee, 0xa129ca6149be45e5, + 0x3f2acc7f57c29bdb, 0x699ae9f52cbe4794, 0x4bc1b3f0968dd39c, 0xbb6dc91da77961bd, + 0xbed65cf21aa2ee98, 0xd0f2cbb02e3b67c7, 0x93536795e3a33e88, 0xa80c038ccd5ccec8, + 0xb8ad50c6f649af94, 0xbce192de8a85b8ea, 0x17d835b85bbb15f3, 0x2f2e6163076bcfad, + 0xde4daaaca71dc9a5, 0xa6a2506687956571, 0xad87a3535c49ef28, 0x32d892fad841c342, + 0x7127512f72f27cce, 0xa7f32346f95978e3, 0x12e0b01abb051238, 0x15e034d40fa197ae, + 0x314dffbe0815a3b4, 0x027990f029623981, 0xcadcd4e59ef40c4d, 0x9abfd8766a33735c, + 0x0e3ea96b5304a7d0, 0xad0c42d6fc585992, 0x187306c89bc215a9, 0xd4a60abcf3792b95, + 0xf935451de4f21df2, 0xa9538f0419755787, 0xdb9acddff56ca510, 0xd06c98cd5c0975eb, + 0xe612a3cb9ecba951, 0xc766e62cfcadaf96, 0xee64435a9752fe72, 0xa192d576b245165a, + 0x0a8787bf8ecb74b2, 0x81b3e73d20b49b6f, 0x7fa8220ba3b2ecea, 0x245731c13ca42499, + 0xb78dbfaf3a8d83bd, 0xea1ad565322a1a0b, 0x60e61c23a3795013, 0x6606d7e446282b93, + 0x6ca4ecb15c5f91e1, 0x9f626da15c9625f3, 0xe51b38608ef25f57, 0x958a324ceb064572 +}; + BOOST_AUTO_TEST_CASE(siphash) { CSipHasher hasher(0x0706050403020100ULL, 0x0F0E0D0C0B0A0908ULL); @@ -74,6 +106,22 @@ BOOST_AUTO_TEST_CASE(siphash) BOOST_CHECK_EQUAL(hasher.Finalize(), 0xe612a3cb9ecba951ull); BOOST_CHECK_EQUAL(SipHashUint256(0x0706050403020100ULL, 0x0F0E0D0C0B0A0908ULL, uint256S("1f1e1d1c1b1a191817161514131211100f0e0d0c0b0a09080706050403020100")), 0x7127512f72f27cceull); + + // Check test vectors from spec, one byte at a time + CSipHasher hasher2(0x0706050403020100ULL, 0x0F0E0D0C0B0A0908ULL); + for (uint8_t x=0; x