From 65c589e45e8b8914698a0fd25cd5aafdda30869c Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Wed, 20 May 2020 10:16:41 -0700 Subject: [PATCH 1/4] Convert Qt to new serialization --- src/qt/recentrequeststablemodel.h | 18 +++++------------ src/qt/sendcoinsrecipient.h | 33 +++++++++++-------------------- 2 files changed, 17 insertions(+), 34 deletions(-) diff --git a/src/qt/recentrequeststablemodel.h b/src/qt/recentrequeststablemodel.h index addf5ad0ae7..c0bd3461bb4 100644 --- a/src/qt/recentrequeststablemodel.h +++ b/src/qt/recentrequeststablemodel.h @@ -24,19 +24,11 @@ public: QDateTime date; SendCoinsRecipient recipient; - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { - unsigned int nDate = date.toTime_t(); - - READWRITE(this->nVersion); - READWRITE(id); - READWRITE(nDate); - READWRITE(recipient); - - if (ser_action.ForRead()) - date = QDateTime::fromTime_t(nDate); + SERIALIZE_METHODS(RecentRequestEntry, obj) { + unsigned int date_timet; + SER_WRITE(obj, date_timet = obj.date.toTime_t()); + READWRITE(obj.nVersion, obj.id, date_timet, obj.recipient); + SER_READ(obj, obj.date = QDateTime::fromTime_t(date_timet)); } }; diff --git a/src/qt/sendcoinsrecipient.h b/src/qt/sendcoinsrecipient.h index 12279fab64f..6619faf417f 100644 --- a/src/qt/sendcoinsrecipient.h +++ b/src/qt/sendcoinsrecipient.h @@ -44,30 +44,21 @@ public: static const int CURRENT_VERSION = 1; int nVersion; - ADD_SERIALIZE_METHODS; + SERIALIZE_METHODS(SendCoinsRecipient, obj) + { + std::string address_str, label_str, message_str, auth_merchant_str; - template - inline void SerializationOp(Stream& s, Operation ser_action) { - std::string sAddress = address.toStdString(); - std::string sLabel = label.toStdString(); - std::string sMessage = message.toStdString(); - std::string sAuthenticatedMerchant = authenticatedMerchant.toStdString(); + SER_WRITE(obj, address_str = obj.address.toStdString()); + SER_WRITE(obj, label_str = obj.label.toStdString()); + SER_WRITE(obj, message_str = obj.message.toStdString()); + SER_WRITE(obj, auth_merchant_str = obj.authenticatedMerchant.toStdString()); - READWRITE(this->nVersion); - READWRITE(sAddress); - READWRITE(sLabel); - READWRITE(amount); - READWRITE(sMessage); - READWRITE(sPaymentRequest); - READWRITE(sAuthenticatedMerchant); + READWRITE(obj.nVersion, address_str, label_str, obj.amount, message_str, obj.sPaymentRequest, auth_merchant_str); - if (ser_action.ForRead()) - { - address = QString::fromStdString(sAddress); - label = QString::fromStdString(sLabel); - message = QString::fromStdString(sMessage); - authenticatedMerchant = QString::fromStdString(sAuthenticatedMerchant); - } + SER_READ(obj, obj.address = QString::fromStdString(address_str)); + SER_READ(obj, obj.label = QString::fromStdString(label_str)); + SER_READ(obj, obj.message = QString::fromStdString(message_str)); + SER_READ(obj, obj.authenticatedMerchant = QString::fromStdString(auth_merchant_str)); } }; From ef17c03e074b6c3f185afa4eff572ba687c2a171 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Sun, 24 May 2020 10:34:52 -0700 Subject: [PATCH 2/4] Convert wallet to new serialization --- src/wallet/crypter.h | 12 ++------ src/wallet/scriptpubkeyman.h | 57 ++++++++++++++++++------------------ src/wallet/walletdb.h | 35 +++++++++------------- src/wallet/walletutil.h | 34 ++++++++++----------- 4 files changed, 61 insertions(+), 77 deletions(-) diff --git a/src/wallet/crypter.h b/src/wallet/crypter.h index f59c63260ee..f2df786e2e7 100644 --- a/src/wallet/crypter.h +++ b/src/wallet/crypter.h @@ -43,15 +43,9 @@ public: //! such as the various parameters to scrypt std::vector vchOtherDerivationParameters; - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(vchCryptedKey); - READWRITE(vchSalt); - READWRITE(nDerivationMethod); - READWRITE(nDeriveIterations); - READWRITE(vchOtherDerivationParameters); + SERIALIZE_METHODS(CMasterKey, obj) + { + READWRITE(obj.vchCryptedKey, obj.vchSalt, obj.nDerivationMethod, obj.nDeriveIterations, obj.vchOtherDerivationParameters); } CMasterKey() diff --git a/src/wallet/scriptpubkeyman.h b/src/wallet/scriptpubkeyman.h index 4c002edf2d7..be3116170fa 100644 --- a/src/wallet/scriptpubkeyman.h +++ b/src/wallet/scriptpubkeyman.h @@ -110,36 +110,37 @@ public: CKeyPool(); CKeyPool(const CPubKey& vchPubKeyIn, bool internalIn); - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { + template + void Serialize(Stream& s) const + { int nVersion = s.GetVersion(); - if (!(s.GetType() & SER_GETHASH)) - READWRITE(nVersion); - READWRITE(nTime); - READWRITE(vchPubKey); - if (ser_action.ForRead()) { - try { - READWRITE(fInternal); - } - catch (std::ios_base::failure&) { - /* flag as external address if we can't read the internal boolean - (this will be the case for any wallet before the HD chain split version) */ - fInternal = false; - } - try { - READWRITE(m_pre_split); - } - catch (std::ios_base::failure&) { - /* flag as postsplit address if we can't read the m_pre_split boolean - (this will be the case for any wallet that upgrades to HD chain split)*/ - m_pre_split = false; - } + if (!(s.GetType() & SER_GETHASH)) { + s << nVersion; } - else { - READWRITE(fInternal); - READWRITE(m_pre_split); + s << nTime << vchPubKey << fInternal << m_pre_split; + } + + template + void Unserialize(Stream& s) + { + int nVersion = s.GetVersion(); + if (!(s.GetType() & SER_GETHASH)) { + s >> nVersion; + } + s >> nTime >> vchPubKey; + try { + s >> fInternal; + } catch (std::ios_base::failure&) { + /* flag as external address if we can't read the internal boolean + (this will be the case for any wallet before the HD chain split version) */ + fInternal = false; + } + try { + s >> m_pre_split; + } catch (std::ios_base::failure&) { + /* flag as postsplit address if we can't read the m_pre_split boolean + (this will be the case for any wallet that upgrades to HD chain split) */ + m_pre_split = false; } } }; diff --git a/src/wallet/walletdb.h b/src/wallet/walletdb.h index e2bf229c687..dfec3dba60d 100644 --- a/src/wallet/walletdb.h +++ b/src/wallet/walletdb.h @@ -98,15 +98,13 @@ public: int nVersion; CHDChain() { SetNull(); } - ADD_SERIALIZE_METHODS; - template - inline void SerializationOp(Stream& s, Operation ser_action) + + SERIALIZE_METHODS(CHDChain, obj) { - READWRITE(this->nVersion); - READWRITE(nExternalChainCounter); - READWRITE(seed_id); - if (this->nVersion >= VERSION_HD_CHAIN_SPLIT) - READWRITE(nInternalChainCounter); + READWRITE(obj.nVersion, obj.nExternalChainCounter, obj.seed_id); + if (obj.nVersion >= VERSION_HD_CHAIN_SPLIT) { + READWRITE(obj.nInternalChainCounter); + } } void SetNull() @@ -142,21 +140,16 @@ public: nCreateTime = nCreateTime_; } - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(this->nVersion); - READWRITE(nCreateTime); - if (this->nVersion >= VERSION_WITH_HDDATA) - { - READWRITE(hdKeypath); - READWRITE(hd_seed_id); + SERIALIZE_METHODS(CKeyMetadata, obj) + { + READWRITE(obj.nVersion, obj.nCreateTime); + if (obj.nVersion >= VERSION_WITH_HDDATA) { + READWRITE(obj.hdKeypath, obj.hd_seed_id); } - if (this->nVersion >= VERSION_WITH_KEY_ORIGIN) + if (obj.nVersion >= VERSION_WITH_KEY_ORIGIN) { - READWRITE(key_origin); - READWRITE(has_key_origin); + READWRITE(obj.key_origin); + READWRITE(obj.has_key_origin); } } diff --git a/src/wallet/walletutil.h b/src/wallet/walletutil.h index 599b1a9f5a2..a4e4fda8a1f 100644 --- a/src/wallet/walletutil.h +++ b/src/wallet/walletutil.h @@ -98,26 +98,22 @@ public: int32_t next_index = 0; // Position of the next item to generate DescriptorCache cache; - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { - if (ser_action.ForRead()) { - std::string desc; - std::string error; - READWRITE(desc); - FlatSigningProvider keys; - descriptor = Parse(desc, keys, error, true); - if (!descriptor) { - throw std::ios_base::failure("Invalid descriptor: " + error); - } - } else { - READWRITE(descriptor->ToString()); + void DeserializeDescriptor(const std::string& str) + { + std::string error; + FlatSigningProvider keys; + descriptor = Parse(str, keys, error, true); + if (!descriptor) { + throw std::ios_base::failure("Invalid descriptor: " + error); } - READWRITE(creation_time); - READWRITE(next_index); - READWRITE(range_start); - READWRITE(range_end); + } + + SERIALIZE_METHODS(WalletDescriptor, obj) + { + std::string descriptor_str; + SER_WRITE(obj, descriptor_str = obj.descriptor->ToString()); + READWRITE(descriptor_str, obj.creation_time, obj.next_index, obj.range_start, obj.range_end); + SER_READ(obj, obj.DeserializeDescriptor(descriptor_str)); } WalletDescriptor() {} From 92beff15d3ae2646c00bd78146d7592a7097ce9c Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Wed, 20 May 2020 11:13:06 -0700 Subject: [PATCH 3/4] Convert LimitedString to formatter --- src/serialize.h | 22 +++++++--------------- src/test/fuzz/string.cpp | 4 ++-- 2 files changed, 9 insertions(+), 17 deletions(-) diff --git a/src/serialize.h b/src/serialize.h index af75c50ff95..c2e863d97a5 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -503,7 +503,7 @@ static inline Wrapper Using(T&& t) { return Wrapper>(obj) #define VARINT(obj) Using>(obj) #define COMPACTSIZE(obj) Using(obj) -#define LIMITED_STRING(obj,n) LimitedString< n >(REF(obj)) +#define LIMITED_STRING(obj,n) Using>(obj) /** Serialization wrapper class for integers in VarInt format. */ template @@ -588,31 +588,23 @@ struct CompactSizeFormatter }; template -class LimitedString +struct LimitedStringFormatter { -protected: - std::string& string; -public: - explicit LimitedString(std::string& _string) : string(_string) {} - template - void Unserialize(Stream& s) + void Unser(Stream& s, std::string& v) { size_t size = ReadCompactSize(s); if (size > Limit) { throw std::ios_base::failure("String length limit exceeded"); } - string.resize(size); - if (size != 0) - s.read((char*)string.data(), size); + v.resize(size); + if (size != 0) s.read((char*)v.data(), size); } template - void Serialize(Stream& s) const + void Ser(Stream& s, const std::string& v) { - WriteCompactSize(s, string.size()); - if (!string.empty()) - s.write((char*)string.data(), string.size()); + s << v; } }; diff --git a/src/test/fuzz/string.cpp b/src/test/fuzz/string.cpp index 3c1f911f7e8..50984b1aefb 100644 --- a/src/test/fuzz/string.cpp +++ b/src/test/fuzz/string.cpp @@ -93,7 +93,7 @@ void test_one_input(const std::vector& buffer) { CDataStream data_stream{SER_NETWORK, INIT_PROTO_VERSION}; std::string s; - LimitedString<10> limited_string = LIMITED_STRING(s, 10); + auto limited_string = LIMITED_STRING(s, 10); data_stream << random_string_1; try { data_stream >> limited_string; @@ -108,7 +108,7 @@ void test_one_input(const std::vector& buffer) } { CDataStream data_stream{SER_NETWORK, INIT_PROTO_VERSION}; - const LimitedString<10> limited_string = LIMITED_STRING(random_string_1, 10); + const auto limited_string = LIMITED_STRING(random_string_1, 10); data_stream << limited_string; std::string deserialized_string; data_stream >> deserialized_string; From 71f016c6eb42e1ac2c905e04ba4d20c2009e533f Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Wed, 20 May 2020 11:13:25 -0700 Subject: [PATCH 4/4] Remove old serialization primitives --- src/serialize.h | 38 +------------------------------------- 1 file changed, 1 insertion(+), 37 deletions(-) diff --git a/src/serialize.h b/src/serialize.h index c2e863d97a5..71c2cfa164f 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -43,26 +43,6 @@ static const unsigned int MAX_VECTOR_ALLOCATE = 5000000; struct deserialize_type {}; constexpr deserialize_type deserialize {}; -/** - * Used to bypass the rule against non-const reference to temporary - * where it makes sense with wrappers. - */ -template -inline T& REF(const T& val) -{ - return const_cast(val); -} - -/** - * Used to acquire a non-const pointer "this" to generate bodies - * of const serialization operations from a template - */ -template -inline T* NCONST_PTR(const T* val) -{ - return const_cast(val); -} - //! Safely convert odd char pointer types to standard ones. inline char* CharCast(char* c) { return c; } inline char* CharCast(unsigned char* c) { return (char*)c; } @@ -193,22 +173,6 @@ template const X& ReadWriteAsHelper(const X& x) { return x; } #define SER_READ(obj, code) ::SerRead(s, ser_action, obj, [&](Stream& s, typename std::remove_const::type& obj) { code; }) #define SER_WRITE(obj, code) ::SerWrite(s, ser_action, obj, [&](Stream& s, const Type& obj) { code; }) -/** - * Implement three methods for serializable objects. These are actually wrappers over - * "SerializationOp" template, which implements the body of each class' serialization - * code. Adding "ADD_SERIALIZE_METHODS" in the body of the class causes these wrappers to be - * added as members. - */ -#define ADD_SERIALIZE_METHODS \ - template \ - void Serialize(Stream& s) const { \ - NCONST_PTR(this)->SerializationOp(s, CSerActionSerialize()); \ - } \ - template \ - void Unserialize(Stream& s) { \ - SerializationOp(s, CSerActionUnserialize()); \ - } - /** * Implement the Ser and Unser methods needed for implementing a formatter (see Using below). * @@ -1004,7 +968,7 @@ void Unserialize(Stream& is, std::shared_ptr& p) /** - * Support for ADD_SERIALIZE_METHODS and READWRITE macro + * Support for SERIALIZE_METHODS and READWRITE macro. */ struct CSerActionSerialize {