diff --git a/src/chain.h b/src/chain.h index f51368f4d..1b6be6248 100644 --- a/src/chain.h +++ b/src/chain.h @@ -1,233 +1,233 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_CHAIN_H #define BITCOIN_CHAIN_H #include #include #include #include #include #include // for ReadLE64 #include #include #include #include #include #include #include /** * Maximum amount of time that a block timestamp is allowed to exceed the * current network-adjusted time before the block will be accepted. */ static constexpr int64_t MAX_FUTURE_BLOCK_TIME = 2 * 60 * 60; /** * Timestamp window used as a grace period by code that compares external * timestamps (such as timestamps passed to RPCs, or wallet key creation times) * to block timestamps. This should be set at least as high as * MAX_FUTURE_BLOCK_TIME. */ static constexpr int64_t TIMESTAMP_WINDOW = MAX_FUTURE_BLOCK_TIME; /** * Maximum gap between node time and block time used * for the "Catching up..." mode in GUI. * * Ref: https://github.com/bitcoin/bitcoin/pull/1026 */ static constexpr int64_t MAX_BLOCK_TIME_GAP = 90 * 60; /** * Maintain a map of CBlockIndex for all known headers. */ struct BlockHasher { // this used to call `GetCheapHash()` in uint256, which was later moved; the // cheap hash function simply calls ReadLE64() however, so the end result is // identical size_t operator()(const BlockHash &hash) const { return ReadLE64(hash.begin()); } }; extern RecursiveMutex cs_main; arith_uint256 GetBlockProof(const CBlockIndex &block); /** * Return the time it would take to redo the work difference between from and * to, assuming the current hashrate corresponds to the difficulty at tip, in * seconds. */ int64_t GetBlockProofEquivalentTime(const CBlockIndex &to, const CBlockIndex &from, const CBlockIndex &tip, const Consensus::Params &); /** * Find the forking point between two chain tips. */ const CBlockIndex *LastCommonAncestor(const CBlockIndex *pa, const CBlockIndex *pb); /** * Check if two block index are on the same fork. */ bool AreOnTheSameFork(const CBlockIndex *pa, const CBlockIndex *pb); /** Used to marshal pointers into hashes for db storage. */ class CDiskBlockIndex : public CBlockIndex { public: static constexpr int TRACK_SIZE_VERSION = 220800; BlockHash hashPrev; CDiskBlockIndex() : hashPrev() {} explicit CDiskBlockIndex(const CBlockIndex *pindex) : CBlockIndex(*pindex) { hashPrev = (pprev ? pprev->GetBlockHash() : BlockHash()); } SERIALIZE_METHODS(CDiskBlockIndex, obj) { int _nVersion = s.GetVersion(); if (!(s.GetType() & SER_GETHASH)) { - READWRITE(VARINT(_nVersion, VarIntMode::NONNEGATIVE_SIGNED)); + READWRITE(VARINT_MODE(_nVersion, VarIntMode::NONNEGATIVE_SIGNED)); } - READWRITE(VARINT(obj.nHeight, VarIntMode::NONNEGATIVE_SIGNED)); + READWRITE(VARINT_MODE(obj.nHeight, VarIntMode::NONNEGATIVE_SIGNED)); READWRITE(obj.nStatus); READWRITE(VARINT(obj.nTx)); // The size of the blocks are tracked starting at version 0.22.8 if (obj.nStatus.hasData() && _nVersion >= TRACK_SIZE_VERSION) { READWRITE(VARINT(obj.nSize)); } if (obj.nStatus.hasData() || obj.nStatus.hasUndo()) { - READWRITE(VARINT(obj.nFile, VarIntMode::NONNEGATIVE_SIGNED)); + READWRITE(VARINT_MODE(obj.nFile, VarIntMode::NONNEGATIVE_SIGNED)); } if (obj.nStatus.hasData()) { READWRITE(VARINT(obj.nDataPos)); } if (obj.nStatus.hasUndo()) { READWRITE(VARINT(obj.nUndoPos)); } // block header READWRITE(obj.nVersion); READWRITE(obj.hashPrev); READWRITE(obj.hashMerkleRoot); READWRITE(obj.nTime); READWRITE(obj.nBits); READWRITE(obj.nNonce); } BlockHash GetBlockHash() const { CBlockHeader block; block.nVersion = nVersion; block.hashPrevBlock = hashPrev; block.hashMerkleRoot = hashMerkleRoot; block.nTime = nTime; block.nBits = nBits; block.nNonce = nNonce; return block.GetHash(); } std::string ToString() const { std::string str = "CDiskBlockIndex("; str += CBlockIndex::ToString(); str += strprintf("\n hashBlock=%s, hashPrev=%s)", GetBlockHash().ToString(), hashPrev.ToString()); return str; } }; /** * An in-memory indexed chain of blocks. */ class CChain { private: std::vector vChain; public: /** * Returns the index entry for the genesis block of this chain, or nullptr * if none. */ CBlockIndex *Genesis() const { return vChain.size() > 0 ? vChain[0] : nullptr; } /** * Returns the index entry for the tip of this chain, or nullptr if none. */ CBlockIndex *Tip() const { return vChain.size() > 0 ? vChain[vChain.size() - 1] : nullptr; } /** * Returns the index entry at a particular height in this chain, or nullptr * if no such height exists. */ CBlockIndex *operator[](int nHeight) const { if (nHeight < 0 || nHeight >= (int)vChain.size()) { return nullptr; } return vChain[nHeight]; } /** Compare two chains efficiently. */ friend bool operator==(const CChain &a, const CChain &b) { return a.vChain.size() == b.vChain.size() && a.vChain[a.vChain.size() - 1] == b.vChain[b.vChain.size() - 1]; } /** Efficiently check whether a block is present in this chain. */ bool Contains(const CBlockIndex *pindex) const { return (*this)[pindex->nHeight] == pindex; } /** * Find the successor of a block in this chain, or nullptr if the given * index is not found or is the tip. */ CBlockIndex *Next(const CBlockIndex *pindex) const { if (!Contains(pindex)) { return nullptr; } return (*this)[pindex->nHeight + 1]; } /** * Return the maximal height in the chain. Is equal to chain.Tip() ? * chain.Tip()->nHeight : -1. */ int Height() const { return vChain.size() - 1; } /** Set/initialize a chain with a given tip. */ void SetTip(CBlockIndex *pindex); /** * Return a CBlockLocator that refers to a block in this chain (by default * the tip). */ CBlockLocator GetLocator(const CBlockIndex *pindex = nullptr) const; /** * Find the last common block between this chain and a block index entry. */ const CBlockIndex *FindFork(const CBlockIndex *pindex) const; /** * Find the earliest block with timestamp equal or greater than the given * time and height equal or greater than the given height. */ CBlockIndex *FindEarliestAtLeast(int64_t nTime, int height) const; }; #endif // BITCOIN_CHAIN_H diff --git a/src/flatfile.h b/src/flatfile.h index 2537a4365..4a1f9812b 100644 --- a/src/flatfile.h +++ b/src/flatfile.h @@ -1,100 +1,100 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_FLATFILE_H #define BITCOIN_FLATFILE_H #include #include #include struct FlatFilePos { int nFile; unsigned int nPos; ADD_SERIALIZE_METHODS; template inline void SerializationOp(Stream &s, Operation ser_action) { - READWRITE(VARINT(nFile, VarIntMode::NONNEGATIVE_SIGNED)); + READWRITE(VARINT_MODE(nFile, VarIntMode::NONNEGATIVE_SIGNED)); READWRITE(VARINT(nPos)); } FlatFilePos() : nFile(-1), nPos(0) {} FlatFilePos(int nFileIn, unsigned int nPosIn) : nFile(nFileIn), nPos(nPosIn) {} friend bool operator==(const FlatFilePos &a, const FlatFilePos &b) { return (a.nFile == b.nFile && a.nPos == b.nPos); } friend bool operator!=(const FlatFilePos &a, const FlatFilePos &b) { return !(a == b); } void SetNull() { nFile = -1; nPos = 0; } bool IsNull() const { return (nFile == -1); } std::string ToString() const; }; /** * FlatFileSeq represents a sequence of numbered files storing raw data. This * class facilitates access to and efficient management of these files. */ class FlatFileSeq { private: const fs::path m_dir; const char *const m_prefix; const size_t m_chunk_size; public: /** * Constructor * * @param dir The base directory that all files live in. * @param prefix A short prefix given to all file names. * @param chunk_size Disk space is pre-allocated in multiples of this * amount. */ FlatFileSeq(fs::path dir, const char *prefix, size_t chunk_size); /** Get the name of the file at the given position. */ fs::path FileName(const FlatFilePos &pos) const; /** Open a handle to the file at the given position. */ FILE *Open(const FlatFilePos &pos, bool read_only = false); /** * Allocate additional space in a file after the given starting position. * The amount allocated will be the minimum multiple of the sequence chunk * size greater than add_size. * * @param[in] pos The starting position that bytes will be allocated after. * @param[in] add_size The minimum number of bytes to be allocated. * @param[out] out_of_space Whether the allocation failed due to * insufficient disk space. * @return The number of bytes successfully allocated. */ size_t Allocate(const FlatFilePos &pos, size_t add_size, bool &out_of_space); /** * Commit a file to disk, and optionally truncate off extra pre-allocated * bytes if final. * * @param[in] pos The first unwritten position in the file to be flushed. * @param[in] finalize True if no more data will be written to this file. * @return true on success, false on failure. */ bool Flush(const FlatFilePos &pos, bool finalize = false); }; #endif // BITCOIN_FLATFILE_H diff --git a/src/node/coinstats.cpp b/src/node/coinstats.cpp index b01c00539..7b4ceae2a 100644 --- a/src/node/coinstats.cpp +++ b/src/node/coinstats.cpp @@ -1,77 +1,77 @@ // Copyright (c) 2010 Satoshi Nakamoto // Copyright (c) 2009-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include #include #include #include #include #include #include static void ApplyStats(CCoinsStats &stats, CHashWriter &ss, const uint256 &hash, const std::map &outputs) { assert(!outputs.empty()); ss << hash; ss << VARINT(outputs.begin()->second.GetHeight() * 2 + outputs.begin()->second.IsCoinBase()); stats.nTransactions++; for (const auto &output : outputs) { ss << VARINT(output.first + 1); ss << output.second.GetTxOut().scriptPubKey; - ss << VARINT(output.second.GetTxOut().nValue / SATOSHI, - VarIntMode::NONNEGATIVE_SIGNED); + ss << VARINT_MODE(output.second.GetTxOut().nValue / SATOSHI, + VarIntMode::NONNEGATIVE_SIGNED); stats.nTransactionOutputs++; stats.nTotalAmount += output.second.GetTxOut().nValue; stats.nBogoSize += 32 /* txid */ + 4 /* vout index */ + 4 /* height + coinbase */ + 8 /* amount */ + 2 /* scriptPubKey len */ + output.second.GetTxOut().scriptPubKey.size() /* scriptPubKey */; } ss << VARINT(0u); } //! Calculate statistics about the unspent transaction output set bool GetUTXOStats(CCoinsView *view, CCoinsStats &stats, const std::function &interruption_point) { stats = CCoinsStats(); std::unique_ptr pcursor(view->Cursor()); assert(pcursor); CHashWriter ss(SER_GETHASH, PROTOCOL_VERSION); stats.hashBlock = pcursor->GetBestBlock(); { LOCK(cs_main); stats.nHeight = LookupBlockIndex(stats.hashBlock)->nHeight; } ss << stats.hashBlock; uint256 prevkey; std::map outputs; while (pcursor->Valid()) { interruption_point(); COutPoint key; Coin coin; if (pcursor->GetKey(key) && pcursor->GetValue(coin)) { if (!outputs.empty() && key.GetTxId() != prevkey) { ApplyStats(stats, ss, prevkey, outputs); outputs.clear(); } prevkey = key.GetTxId(); outputs[key.GetN()] = std::move(coin); stats.coins_count++; } else { return error("%s: unable to read value", __func__); } pcursor->Next(); } if (!outputs.empty()) { ApplyStats(stats, ss, prevkey, outputs); } stats.hashSerialized = ss.GetHash(); stats.nDiskSize = view->EstimateSize(); return true; } diff --git a/src/serialize.h b/src/serialize.h index ccc71dde4..9b8df12c4 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -1,1176 +1,1177 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_SERIALIZE_H #define BITCOIN_SERIALIZE_H #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include static const uint64_t MAX_SIZE = 0x02000000; /** * Maximum amount of memory (in bytes) to allocate at once when deserializing * vectors. */ static const unsigned int MAX_VECTOR_ALLOCATE = 5000000; /** * Dummy data type to identify deserializing constructors. * * By convention, a constructor of a type T with signature * * template T::T(deserialize_type, Stream& s) * * is a deserializing constructor, which builds the type by deserializing it * from s. If T contains const fields, this is likely the only way to do so. */ struct deserialize_type {}; constexpr deserialize_type deserialize{}; /** * Used to bypass the rule against non-const reference to temporary * where it makes sense with wrappers. */ template inline T &REF(const T &val) { return const_cast(val); } /** * Used to acquire a non-const pointer "this" to generate bodies of const * serialization operations from a template */ template inline T *NCONST_PTR(const T *val) { return const_cast(val); } //! Safely convert odd char pointer types to standard ones. inline char *CharCast(char *c) { return c; } inline char *CharCast(uint8_t *c) { return (char *)c; } inline const char *CharCast(const char *c) { return c; } inline const char *CharCast(const uint8_t *c) { return (const char *)c; } /** * Lowest-level serialization and conversion. * @note Sizes of these types are verified in the tests */ template inline void ser_writedata8(Stream &s, uint8_t obj) { s.write((char *)&obj, 1); } template inline void ser_writedata16(Stream &s, uint16_t obj) { obj = htole16(obj); s.write((char *)&obj, 2); } template inline void ser_writedata16be(Stream &s, uint16_t obj) { obj = htobe16(obj); s.write((char *)&obj, 2); } template inline void ser_writedata32(Stream &s, uint32_t obj) { obj = htole32(obj); s.write((char *)&obj, 4); } template inline void ser_writedata32be(Stream &s, uint32_t obj) { obj = htobe32(obj); s.write((char *)&obj, 4); } template inline void ser_writedata64(Stream &s, uint64_t obj) { obj = htole64(obj); s.write((char *)&obj, 8); } template inline uint8_t ser_readdata8(Stream &s) { uint8_t obj; s.read((char *)&obj, 1); return obj; } template inline uint16_t ser_readdata16(Stream &s) { uint16_t obj; s.read((char *)&obj, 2); return le16toh(obj); } template inline uint16_t ser_readdata16be(Stream &s) { uint16_t obj; s.read((char *)&obj, 2); return be16toh(obj); } template inline uint32_t ser_readdata32(Stream &s) { uint32_t obj; s.read((char *)&obj, 4); return le32toh(obj); } template inline uint32_t ser_readdata32be(Stream &s) { uint32_t obj; s.read((char *)&obj, 4); return be32toh(obj); } template inline uint64_t ser_readdata64(Stream &s) { uint64_t obj; s.read((char *)&obj, 8); return le64toh(obj); } inline uint64_t ser_double_to_uint64(double x) { union { double x; uint64_t y; } tmp; tmp.x = x; return tmp.y; } inline uint32_t ser_float_to_uint32(float x) { union { float x; uint32_t y; } tmp; tmp.x = x; return tmp.y; } inline double ser_uint64_to_double(uint64_t y) { union { double x; uint64_t y; } tmp; tmp.y = y; return tmp.x; } inline float ser_uint32_to_float(uint32_t y) { union { float x; uint32_t y; } tmp; tmp.y = y; return tmp.x; } ///////////////////////////////////////////////////////////////// // // Templates for serializing to anything that looks like a stream, // i.e. anything that supports .read(char*, size_t) and .write(char*, size_t) // class CSizeComputer; enum { // primary actions SER_NETWORK = (1 << 0), SER_DISK = (1 << 1), SER_GETHASH = (1 << 2), }; //! Convert the reference base type to X, without changing constness or //! reference type. template X &ReadWriteAsHelper(X &x) { return x; } template const X &ReadWriteAsHelper(const X &x) { return x; } #define READWRITE(...) (::SerReadWriteMany(s, ser_action, __VA_ARGS__)) #define READWRITEAS(type, obj) \ (::SerReadWriteMany(s, ser_action, ReadWriteAsHelper(obj))) /** * Implement three methods for serializable objects. These are actually wrappers * over "SerializationOp" template, which implements the body of each class' * serialization code. Adding "ADD_SERIALIZE_METHODS" in the body of the class * causes these wrappers to be added as members. */ #define ADD_SERIALIZE_METHODS \ template void Serialize(Stream &s) const { \ NCONST_PTR(this)->SerializationOp(s, CSerActionSerialize()); \ } \ template void Unserialize(Stream &s) { \ SerializationOp(s, CSerActionUnserialize()); \ } /** * Implement the Ser and Unser methods needed for implementing a formatter * (see Using below). * * Both Ser and Unser are delegated to a single static method SerializationOps, * which is polymorphic in the serialized/deserialized type (allowing it to be * const when serializing, and non-const when deserializing). * * Example use: * struct FooFormatter { * FORMATTER_METHODS(Class, obj) { READWRITE(obj.val1, VARINT(obj.val2)); } * } * would define a class FooFormatter that defines a serialization of Class * objects consisting of serializing its val1 member using the default * serialization, and its val2 member using VARINT serialization. That * FooFormatter can then be used in statements like * READWRITE(Using(obj.bla)). */ #define FORMATTER_METHODS(cls, obj) \ template static void Ser(Stream &s, const cls &obj) { \ SerializationOps(obj, s, CSerActionSerialize()); \ } \ template static void Unser(Stream &s, cls &obj) { \ SerializationOps(obj, s, CSerActionUnserialize()); \ } \ template \ static inline void SerializationOps(Type &obj, Stream &s, \ Operation ser_action) /** * Implement the Serialize and Unserialize methods by delegating to a * single templated static method that takes the to-be-(de)serialized * object as a parameter. This approach has the advantage that the * constness of the object becomes a template parameter, and thus * allows a single implementation that sees the object as const for * serializing and non-const for deserializing, without casts. */ #define SERIALIZE_METHODS(cls, obj) \ template void Serialize(Stream &s) const { \ static_assert(std::is_same::value, \ "Serialize type mismatch"); \ Ser(s, *this); \ } \ template void Unserialize(Stream &s) { \ static_assert(std::is_same::value, \ "Unserialize type mismatch"); \ Unser(s, *this); \ } \ FORMATTER_METHODS(cls, obj) #ifndef CHAR_EQUALS_INT8 // TODO Get rid of bare char template inline void Serialize(Stream &s, char a) { ser_writedata8(s, a); } #endif template inline void Serialize(Stream &s, int8_t a) { ser_writedata8(s, a); } template inline void Serialize(Stream &s, uint8_t a) { ser_writedata8(s, a); } template inline void Serialize(Stream &s, int16_t a) { ser_writedata16(s, a); } template inline void Serialize(Stream &s, uint16_t a) { ser_writedata16(s, a); } template inline void Serialize(Stream &s, int32_t a) { ser_writedata32(s, a); } template inline void Serialize(Stream &s, uint32_t a) { ser_writedata32(s, a); } template inline void Serialize(Stream &s, int64_t a) { ser_writedata64(s, a); } template inline void Serialize(Stream &s, uint64_t a) { ser_writedata64(s, a); } template inline void Serialize(Stream &s, float a) { ser_writedata32(s, ser_float_to_uint32(a)); } template inline void Serialize(Stream &s, double a) { ser_writedata64(s, ser_double_to_uint64(a)); } template inline void Serialize(Stream &s, const int8_t (&a)[N]) { s.write(a, N); } template inline void Serialize(Stream &s, const uint8_t (&a)[N]) { s.write(CharCast(a), N); } template inline void Serialize(Stream &s, const std::array &a) { s.write(a.data(), N); } template inline void Serialize(Stream &s, const std::array &a) { s.write(CharCast(a.data()), N); } #ifndef CHAR_EQUALS_INT8 // TODO Get rid of bare char template inline void Unserialize(Stream &s, char &a) { a = ser_readdata8(s); } template inline void Serialize(Stream &s, const char (&a)[N]) { s.write(a, N); } template inline void Serialize(Stream &s, const std::array &a) { s.write(a.data(), N); } #endif template inline void Serialize(Stream &s, const Span &span) { s.write(CharCast(span.data()), span.size()); } template inline void Serialize(Stream &s, const Span &span) { s.write(CharCast(span.data()), span.size()); } template inline void Unserialize(Stream &s, int8_t &a) { a = ser_readdata8(s); } template inline void Unserialize(Stream &s, uint8_t &a) { a = ser_readdata8(s); } template inline void Unserialize(Stream &s, int16_t &a) { a = ser_readdata16(s); } template inline void Unserialize(Stream &s, uint16_t &a) { a = ser_readdata16(s); } template inline void Unserialize(Stream &s, int32_t &a) { a = ser_readdata32(s); } template inline void Unserialize(Stream &s, uint32_t &a) { a = ser_readdata32(s); } template inline void Unserialize(Stream &s, int64_t &a) { a = ser_readdata64(s); } template inline void Unserialize(Stream &s, uint64_t &a) { a = ser_readdata64(s); } template inline void Unserialize(Stream &s, float &a) { a = ser_uint32_to_float(ser_readdata32(s)); } template inline void Unserialize(Stream &s, double &a) { a = ser_uint64_to_double(ser_readdata64(s)); } template inline void Unserialize(Stream &s, int8_t (&a)[N]) { s.read(a, N); } template inline void Unserialize(Stream &s, uint8_t (&a)[N]) { s.read(CharCast(a), N); } template inline void Unserialize(Stream &s, std::array &a) { s.read(a.data(), N); } template inline void Unserialize(Stream &s, std::array &a) { s.read(CharCast(a.data()), N); } #ifndef CHAR_EQUALS_INT8 template inline void Unserialize(Stream &s, char (&a)[N]) { s.read(CharCast(a), N); } template inline void Unserialize(Stream &s, std::array &a) { s.read(CharCast(a.data()), N); } #endif template inline void Serialize(Stream &s, bool a) { char f = a; ser_writedata8(s, f); } template inline void Unserialize(Stream &s, bool &a) { char f = ser_readdata8(s); a = f; } template inline void Unserialize(Stream &s, Span &span) { s.read(CharCast(span.data()), span.size()); } /** * Compact Size * size < 253 -- 1 byte * size <= USHRT_MAX -- 3 bytes (253 + 2 bytes) * size <= UINT_MAX -- 5 bytes (254 + 4 bytes) * size > UINT_MAX -- 9 bytes (255 + 8 bytes) */ inline uint32_t GetSizeOfCompactSize(uint64_t nSize) { if (nSize < 253) { return sizeof(uint8_t); } if (nSize <= std::numeric_limits::max()) { return sizeof(uint8_t) + sizeof(uint16_t); } if (nSize <= std::numeric_limits::max()) { return sizeof(uint8_t) + sizeof(uint32_t); } return sizeof(uint8_t) + sizeof(uint64_t); } inline void WriteCompactSize(CSizeComputer &os, uint64_t nSize); template void WriteCompactSize(Stream &os, uint64_t nSize) { if (nSize < 253) { ser_writedata8(os, nSize); } else if (nSize <= std::numeric_limits::max()) { ser_writedata8(os, 253); ser_writedata16(os, nSize); } else if (nSize <= std::numeric_limits::max()) { ser_writedata8(os, 254); ser_writedata32(os, nSize); } else { ser_writedata8(os, 255); ser_writedata64(os, nSize); } return; } template uint64_t ReadCompactSize(Stream &is) { uint8_t chSize = ser_readdata8(is); uint64_t nSizeRet = 0; if (chSize < 253) { nSizeRet = chSize; } else if (chSize == 253) { nSizeRet = ser_readdata16(is); if (nSizeRet < 253) { throw std::ios_base::failure("non-canonical ReadCompactSize()"); } } else if (chSize == 254) { nSizeRet = ser_readdata32(is); if (nSizeRet < 0x10000u) { throw std::ios_base::failure("non-canonical ReadCompactSize()"); } } else { nSizeRet = ser_readdata64(is); if (nSizeRet < 0x100000000ULL) { throw std::ios_base::failure("non-canonical ReadCompactSize()"); } } if (nSizeRet > MAX_SIZE) { throw std::ios_base::failure("ReadCompactSize(): size too large"); } return nSizeRet; } /** * Variable-length integers: bytes are a MSB base-128 encoding of the number. * The high bit in each byte signifies whether another digit follows. To make * sure the encoding is one-to-one, one is subtracted from all but the last * digit. Thus, the byte sequence a[] with length len, where all but the last * byte has bit 128 set, encodes the number: * * (a[len-1] & 0x7F) + sum(i=1..len-1, 128^i*((a[len-i-1] & 0x7F)+1)) * * Properties: * * Very small (0-127: 1 byte, 128-16511: 2 bytes, 16512-2113663: 3 bytes) * * Every integer has exactly one encoding * * Encoding does not depend on size of original integer type * * No redundancy: every (infinite) byte sequence corresponds to a list * of encoded integers. * * 0: [0x00] 256: [0x81 0x00] * 1: [0x01] 16383: [0xFE 0x7F] * 127: [0x7F] 16384: [0xFF 0x00] * 128: [0x80 0x00] 16511: [0xFF 0x7F] * 255: [0x80 0x7F] 65535: [0x82 0xFE 0x7F] * 2^32: [0x8E 0xFE 0xFE 0xFF 0x00] */ /** * Mode for encoding VarInts. * * Currently there is no support for signed encodings. The default mode will not * compile with signed values, and the legacy "nonnegative signed" mode will * accept signed values, but improperly encode and decode them if they are * negative. In the future, the DEFAULT mode could be extended to support * negative numbers in a backwards compatible way, and additional modes could be * added to support different varint formats (e.g. zigzag encoding). */ enum class VarIntMode { DEFAULT, NONNEGATIVE_SIGNED }; template struct CheckVarIntMode { constexpr CheckVarIntMode() { static_assert(Mode != VarIntMode::DEFAULT || std::is_unsigned::value, "Unsigned type required with mode DEFAULT."); static_assert(Mode != VarIntMode::NONNEGATIVE_SIGNED || std::is_signed::value, "Signed type required with mode NONNEGATIVE_SIGNED."); } }; template inline unsigned int GetSizeOfVarInt(I n) { CheckVarIntMode(); int nRet = 0; while (true) { nRet++; if (n <= 0x7F) { return nRet; } n = (n >> 7) - 1; } } template inline void WriteVarInt(CSizeComputer &os, I n); template void WriteVarInt(Stream &os, I n) { CheckVarIntMode(); uint8_t tmp[(sizeof(n) * 8 + 6) / 7]; int len = 0; while (true) { tmp[len] = (n & 0x7F) | (len ? 0x80 : 0x00); if (n <= 0x7F) { break; } n = (n >> 7) - 1; len++; } do { ser_writedata8(os, tmp[len]); } while (len--); } template I ReadVarInt(Stream &is) { CheckVarIntMode(); I n = 0; while (true) { uint8_t chData = ser_readdata8(is); if (n > (std::numeric_limits::max() >> 7)) { throw std::ios_base::failure("ReadVarInt(): size too large"); } n = (n << 7) | (chData & 0x7F); if ((chData & 0x80) == 0) { return n; } if (n == std::numeric_limits::max()) { throw std::ios_base::failure("ReadVarInt(): size too large"); } n++; } } /** * Simple wrapper class to serialize objects using a formatter; used by * Using(). */ template class Wrapper { static_assert(std::is_lvalue_reference::value, "Wrapper needs an lvalue reference type T"); protected: T m_object; public: explicit Wrapper(T obj) : m_object(obj) {} template void Serialize(Stream &s) const { Formatter().Ser(s, m_object); } template void Unserialize(Stream &s) { Formatter().Unser(s, m_object); } }; /** * Cause serialization/deserialization of an object to be done using a * specified formatter class. * * To use this, you need a class Formatter that has public functions Ser(stream, * const object&) for serialization, and Unser(stream, object&) for * deserialization. Serialization routines (inside READWRITE, or directly with * << and >> operators), can then use Using(object). * * This works by constructing a Wrapper-wrapped version of object, * where T is const during serialization, and non-const during deserialization, * which maintains const correctness. */ template static inline Wrapper Using(T &&t) { return Wrapper(t); } -#define VARINT(obj, ...) Using>(obj) +#define VARINT_MODE(obj, mode) Using>(obj) +#define VARINT(obj) Using>(obj) #define COMPACTSIZE(obj) CCompactSize(REF(obj)) #define LIMITED_STRING(obj, n) LimitedString(REF(obj)) /** * Serialization wrapper class for integers in VarInt format. */ -template struct VarIntFormatter { +template struct VarIntFormatter { template void Ser(Stream &s, I v) { WriteVarInt::type>(s, v); } template void Unser(Stream &s, I &v) { v = ReadVarInt::type>(s); } }; /** Serialization wrapper class for big-endian integers. * * Use this wrapper around integer types that are stored in memory in native * byte order, but serialized in big endian notation. This is only intended * to implement serializers that are compatible with existing formats, and * its use is not recommended for new data structures. * * Only 16-bit types are supported for now. */ template class BigEndian { protected: I &m_val; public: explicit BigEndian(I &val) : m_val(val) { static_assert(std::is_unsigned::value, "BigEndian type must be unsigned integer"); static_assert(sizeof(I) == 2 && std::numeric_limits::min() == 0 && std::numeric_limits::max() == std::numeric_limits::max(), "Unsupported BigEndian size"); } template void Serialize(Stream &s) const { ser_writedata16be(s, m_val); } template void Unserialize(Stream &s) { m_val = ser_readdata16be(s); } }; class CCompactSize { protected: uint64_t &n; public: explicit CCompactSize(uint64_t &nIn) : n(nIn) {} template void Serialize(Stream &s) const { WriteCompactSize(s, n); } template void Unserialize(Stream &s) { n = ReadCompactSize(s); } }; template class LimitedString { protected: std::string &string; public: explicit LimitedString(std::string &_string) : string(_string) {} template void Unserialize(Stream &s) { size_t size = ReadCompactSize(s); if (size > Limit) { throw std::ios_base::failure("String length limit exceeded"); } string.resize(size); if (size != 0) { s.read((char *)string.data(), size); } } template void Serialize(Stream &s) const { WriteCompactSize(s, string.size()); if (!string.empty()) { s.write((char *)string.data(), string.size()); } } }; template BigEndian WrapBigEndian(I &n) { return BigEndian(n); } /** * Formatter to serialize/deserialize vector elements using another formatter * * Example: * struct X { * std::vector v; * SERIALIZE_METHODS(X, obj) { * READWRITE(Using>(obj.v)); * } * }; * will define a struct that contains a vector of uint64_t, which is serialized * as a vector of VarInt-encoded integers. * * V is not required to be an std::vector type. It works for any class that * exposes a value_type, size, reserve, push_back, and const iterators. */ template struct VectorFormatter { template void Ser(Stream &s, const V &v) { WriteCompactSize(s, v.size()); for (const typename V::value_type &elem : v) { s << Using(elem); } } template void Unser(Stream &s, V &v) { v.clear(); size_t size = ReadCompactSize(s); size_t allocated = 0; while (allocated < size) { // For DoS prevention, do not blindly allocate as much as the stream // claims to contain. Instead, allocate in 5MiB batches, so that an // attacker actually needs to provide X MiB of data to make us // allocate X+5 Mib. static_assert(sizeof(typename V::value_type) <= MAX_VECTOR_ALLOCATE, "Vector element size too large"); allocated = std::min(size, allocated + MAX_VECTOR_ALLOCATE / sizeof(typename V::value_type)); v.reserve(allocated); while (v.size() < allocated) { typename V::value_type val; s >> Using(val); v.push_back(std::move(val)); } } }; }; /** * Forward declarations */ /** * string */ template void Serialize(Stream &os, const std::basic_string &str); template void Unserialize(Stream &is, std::basic_string &str); /** * prevector * prevectors of uint8_t are a special case and are intended to be serialized as * a single opaque blob. */ template void Serialize_impl(Stream &os, const prevector &v, const uint8_t &); template void Serialize_impl(Stream &os, const prevector &v, const V &); template inline void Serialize(Stream &os, const prevector &v); template void Unserialize_impl(Stream &is, prevector &v, const uint8_t &); template void Unserialize_impl(Stream &is, prevector &v, const V &); template inline void Unserialize(Stream &is, prevector &v); /** * vector * vectors of uint8_t are a special case and are intended to be serialized as a * single opaque blob. */ template void Serialize_impl(Stream &os, const std::vector &v, const uint8_t &); template void Serialize_impl(Stream &os, const std::vector &v, const bool &); template void Serialize_impl(Stream &os, const std::vector &v, const V &); template inline void Serialize(Stream &os, const std::vector &v); template void Unserialize_impl(Stream &is, std::vector &v, const uint8_t &); template void Unserialize_impl(Stream &is, std::vector &v, const V &); template inline void Unserialize(Stream &is, std::vector &v); /** * pair */ template void Serialize(Stream &os, const std::pair &item); template void Unserialize(Stream &is, std::pair &item); /** * map */ template void Serialize(Stream &os, const std::map &m); template void Unserialize(Stream &is, std::map &m); /** * set */ template void Serialize(Stream &os, const std::set &m); template void Unserialize(Stream &is, std::set &m); /** * shared_ptr */ template void Serialize(Stream &os, const std::shared_ptr &p); template void Unserialize(Stream &os, std::shared_ptr &p); /** * unique_ptr */ template void Serialize(Stream &os, const std::unique_ptr &p); template void Unserialize(Stream &os, std::unique_ptr &p); /** * If none of the specialized versions above matched, default to calling member * function. */ template inline void Serialize(Stream &os, const T &a) { a.Serialize(os); } template inline void Unserialize(Stream &is, T &&a) { a.Unserialize(is); } /** * Default formatter. Serializes objects as themselves. * * The vector/prevector serialization code passes this to VectorFormatter * to enable reusing that logic. It shouldn't be needed elsewhere. */ struct DefaultFormatter { template static void Ser(Stream &s, const T &t) { Serialize(s, t); } template static void Unser(Stream &s, T &t) { Unserialize(s, t); } }; /** * string */ template void Serialize(Stream &os, const std::basic_string &str) { WriteCompactSize(os, str.size()); if (!str.empty()) { os.write((char *)str.data(), str.size() * sizeof(C)); } } template void Unserialize(Stream &is, std::basic_string &str) { size_t nSize = ReadCompactSize(is); str.resize(nSize); if (nSize != 0) { is.read((char *)str.data(), nSize * sizeof(C)); } } /** * prevector */ template void Serialize_impl(Stream &os, const prevector &v, const uint8_t &) { WriteCompactSize(os, v.size()); if (!v.empty()) { os.write((char *)v.data(), v.size() * sizeof(T)); } } template void Serialize_impl(Stream &os, const prevector &v, const V &) { Serialize(os, Using>(v)); } template inline void Serialize(Stream &os, const prevector &v) { Serialize_impl(os, v, T()); } template void Unserialize_impl(Stream &is, prevector &v, const uint8_t &) { // Limit size per read so bogus size value won't cause out of memory v.clear(); size_t nSize = ReadCompactSize(is); size_t i = 0; while (i < nSize) { size_t blk = std::min(nSize - i, size_t(1 + 4999999 / sizeof(T))); v.resize_uninitialized(i + blk); is.read((char *)&v[i], blk * sizeof(T)); i += blk; } } template void Unserialize_impl(Stream &is, prevector &v, const V &) { Unserialize(is, Using>(v)); } template inline void Unserialize(Stream &is, prevector &v) { Unserialize_impl(is, v, T()); } /** * vector */ template void Serialize_impl(Stream &os, const std::vector &v, const uint8_t &) { WriteCompactSize(os, v.size()); if (!v.empty()) { os.write((char *)v.data(), v.size() * sizeof(T)); } } template void Serialize_impl(Stream &os, const std::vector &v, const bool &) { // A special case for std::vector, as dereferencing // std::vector::const_iterator does not result in a const bool& // due to std::vector's special casing for bool arguments. WriteCompactSize(os, v.size()); for (bool elem : v) { ::Serialize(os, elem); } } template void Serialize_impl(Stream &os, const std::vector &v, const V &) { Serialize(os, Using>(v)); } template inline void Serialize(Stream &os, const std::vector &v) { Serialize_impl(os, v, T()); } template void Unserialize_impl(Stream &is, std::vector &v, const uint8_t &) { // Limit size per read so bogus size value won't cause out of memory v.clear(); size_t nSize = ReadCompactSize(is); size_t i = 0; while (i < nSize) { size_t blk = std::min(nSize - i, size_t(1 + 4999999 / sizeof(T))); v.resize(i + blk); is.read((char *)&v[i], blk * sizeof(T)); i += blk; } } template void Unserialize_impl(Stream &is, std::vector &v, const V &) { Unserialize(is, Using>(v)); } template inline void Unserialize(Stream &is, std::vector &v) { Unserialize_impl(is, v, T()); } /** * pair */ template void Serialize(Stream &os, const std::pair &item) { Serialize(os, item.first); Serialize(os, item.second); } template void Unserialize(Stream &is, std::pair &item) { Unserialize(is, item.first); Unserialize(is, item.second); } /** * map */ template void Serialize(Stream &os, const std::map &m) { WriteCompactSize(os, m.size()); for (const auto &entry : m) { Serialize(os, entry); } } template void Unserialize(Stream &is, std::map &m) { m.clear(); size_t nSize = ReadCompactSize(is); typename std::map::iterator mi = m.begin(); for (size_t i = 0; i < nSize; i++) { std::pair item; Unserialize(is, item); mi = m.insert(mi, item); } } /** * set */ template void Serialize(Stream &os, const std::set &m) { WriteCompactSize(os, m.size()); for (const K &i : m) { Serialize(os, i); } } template void Unserialize(Stream &is, std::set &m) { m.clear(); size_t nSize = ReadCompactSize(is); typename std::set::iterator it = m.begin(); for (size_t i = 0; i < nSize; i++) { K key; Unserialize(is, key); it = m.insert(it, key); } } /** * unique_ptr */ template void Serialize(Stream &os, const std::unique_ptr &p) { Serialize(os, *p); } template void Unserialize(Stream &is, std::unique_ptr &p) { p.reset(new T(deserialize, is)); } /** * shared_ptr */ template void Serialize(Stream &os, const std::shared_ptr &p) { Serialize(os, *p); } template void Unserialize(Stream &is, std::shared_ptr &p) { p = std::make_shared(deserialize, is); } /** * Support for ADD_SERIALIZE_METHODS and READWRITE macro */ struct CSerActionSerialize { constexpr bool ForRead() const { return false; } }; struct CSerActionUnserialize { constexpr bool ForRead() const { return true; } }; /** * ::GetSerializeSize implementations * * Computing the serialized size of objects is done through a special stream * object of type CSizeComputer, which only records the number of bytes written * to it. * * If your Serialize or SerializationOp method has non-trivial overhead for * serialization, it may be worthwhile to implement a specialized version for * CSizeComputer, which uses the s.seek() method to record bytes that would * be written instead. */ class CSizeComputer { protected: size_t nSize; const int nVersion; public: explicit CSizeComputer(int nVersionIn) : nSize(0), nVersion(nVersionIn) {} void write(const char *psz, size_t _nSize) { this->nSize += _nSize; } /** Pretend _nSize bytes are written, without specifying them. */ void seek(size_t _nSize) { this->nSize += _nSize; } template CSizeComputer &operator<<(const T &obj) { ::Serialize(*this, obj); return (*this); } size_t size() const { return nSize; } int GetVersion() const { return nVersion; } }; template void SerializeMany(Stream &s) {} template void SerializeMany(Stream &s, const Arg &arg, const Args &... args) { ::Serialize(s, arg); ::SerializeMany(s, args...); } template inline void UnserializeMany(Stream &s) {} template inline void UnserializeMany(Stream &s, Arg &&arg, Args &&... args) { ::Unserialize(s, arg); ::UnserializeMany(s, args...); } template inline void SerReadWriteMany(Stream &s, CSerActionSerialize ser_action, const Args &... args) { ::SerializeMany(s, args...); } template inline void SerReadWriteMany(Stream &s, CSerActionUnserialize ser_action, Args &&... args) { ::UnserializeMany(s, args...); } template inline void WriteVarInt(CSizeComputer &s, I n) { s.seek(GetSizeOfVarInt(n)); } inline void WriteCompactSize(CSizeComputer &s, uint64_t nSize) { s.seek(GetSizeOfCompactSize(nSize)); } template size_t GetSerializeSize(const T &t, int nVersion = 0) { return (CSizeComputer(nVersion) << t).size(); } template size_t GetSerializeSizeMany(int nVersion, const T &... t) { CSizeComputer sc(nVersion); SerializeMany(sc, t...); return sc.size(); } #endif // BITCOIN_SERIALIZE_H diff --git a/src/test/serialize_tests.cpp b/src/test/serialize_tests.cpp index 2f24836d0..c12003bac 100644 --- a/src/test/serialize_tests.cpp +++ b/src/test/serialize_tests.cpp @@ -1,448 +1,449 @@ // Copyright (c) 2012-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include #include #include #include #include #include #include #include BOOST_FIXTURE_TEST_SUITE(serialize_tests, BasicTestingSetup) class CSerializeMethodsTestSingle { protected: int intval; bool boolval; std::string stringval; char charstrval[16]; CTransactionRef txval; public: CSerializeMethodsTestSingle() = default; CSerializeMethodsTestSingle(int intvalin, bool boolvalin, std::string stringvalin, const char *charstrvalin, const CTransactionRef &txvalin) : intval(intvalin), boolval(boolvalin), stringval(std::move(stringvalin)), txval(txvalin) { memcpy(charstrval, charstrvalin, sizeof(charstrval)); } ADD_SERIALIZE_METHODS; template inline void SerializationOp(Stream &s, Operation ser_action) { READWRITE(intval); READWRITE(boolval); READWRITE(stringval); READWRITE(charstrval); READWRITE(txval); } bool operator==(const CSerializeMethodsTestSingle &rhs) { return intval == rhs.intval && boolval == rhs.boolval && stringval == rhs.stringval && strcmp(charstrval, rhs.charstrval) == 0 && *txval == *rhs.txval; } }; class CSerializeMethodsTestMany : public CSerializeMethodsTestSingle { public: using CSerializeMethodsTestSingle::CSerializeMethodsTestSingle; ADD_SERIALIZE_METHODS; template inline void SerializationOp(Stream &s, Operation ser_action) { READWRITE(intval, boolval, stringval, charstrval, txval); } }; BOOST_AUTO_TEST_CASE(sizes) { BOOST_CHECK_EQUAL(sizeof(char), GetSerializeSize(char(0))); BOOST_CHECK_EQUAL(sizeof(int8_t), GetSerializeSize(int8_t(0))); BOOST_CHECK_EQUAL(sizeof(uint8_t), GetSerializeSize(uint8_t(0))); BOOST_CHECK_EQUAL(sizeof(int16_t), GetSerializeSize(int16_t(0))); BOOST_CHECK_EQUAL(sizeof(uint16_t), GetSerializeSize(uint16_t(0))); BOOST_CHECK_EQUAL(sizeof(int32_t), GetSerializeSize(int32_t(0))); BOOST_CHECK_EQUAL(sizeof(uint32_t), GetSerializeSize(uint32_t(0))); BOOST_CHECK_EQUAL(sizeof(int64_t), GetSerializeSize(int64_t(0))); BOOST_CHECK_EQUAL(sizeof(uint64_t), GetSerializeSize(uint64_t(0))); BOOST_CHECK_EQUAL(sizeof(float), GetSerializeSize(float(0))); BOOST_CHECK_EQUAL(sizeof(double), GetSerializeSize(double(0))); // Bool is serialized as char BOOST_CHECK_EQUAL(sizeof(char), GetSerializeSize(bool(0))); // Sanity-check GetSerializeSize and c++ type matching BOOST_CHECK_EQUAL(GetSerializeSize(char(0)), 1U); BOOST_CHECK_EQUAL(GetSerializeSize(int8_t(0)), 1U); BOOST_CHECK_EQUAL(GetSerializeSize(uint8_t(0)), 1U); BOOST_CHECK_EQUAL(GetSerializeSize(int16_t(0)), 2U); BOOST_CHECK_EQUAL(GetSerializeSize(uint16_t(0)), 2U); BOOST_CHECK_EQUAL(GetSerializeSize(int32_t(0)), 4U); BOOST_CHECK_EQUAL(GetSerializeSize(uint32_t(0)), 4U); BOOST_CHECK_EQUAL(GetSerializeSize(int64_t(0)), 8U); BOOST_CHECK_EQUAL(GetSerializeSize(uint64_t(0)), 8U); BOOST_CHECK_EQUAL(GetSerializeSize(float(0)), 4U); BOOST_CHECK_EQUAL(GetSerializeSize(double(0)), 8U); BOOST_CHECK_EQUAL(GetSerializeSize(bool(0)), 1U); } BOOST_AUTO_TEST_CASE(floats_conversion) { // Choose values that map unambiguously to binary floating point to avoid // rounding issues at the compiler side. BOOST_CHECK_EQUAL(ser_uint32_to_float(0x00000000), 0.0F); BOOST_CHECK_EQUAL(ser_uint32_to_float(0x3f000000), 0.5F); BOOST_CHECK_EQUAL(ser_uint32_to_float(0x3f800000), 1.0F); BOOST_CHECK_EQUAL(ser_uint32_to_float(0x40000000), 2.0F); BOOST_CHECK_EQUAL(ser_uint32_to_float(0x40800000), 4.0F); BOOST_CHECK_EQUAL(ser_uint32_to_float(0x44444444), 785.066650390625F); BOOST_CHECK_EQUAL(ser_float_to_uint32(0.0F), 0x00000000U); BOOST_CHECK_EQUAL(ser_float_to_uint32(0.5F), 0x3f000000U); BOOST_CHECK_EQUAL(ser_float_to_uint32(1.0F), 0x3f800000U); BOOST_CHECK_EQUAL(ser_float_to_uint32(2.0F), 0x40000000U); BOOST_CHECK_EQUAL(ser_float_to_uint32(4.0F), 0x40800000U); BOOST_CHECK_EQUAL(ser_float_to_uint32(785.066650390625F), 0x44444444U); } BOOST_AUTO_TEST_CASE(doubles_conversion) { // Choose values that map unambiguously to binary floating point to avoid // rounding issues at the compiler side. BOOST_CHECK_EQUAL(ser_uint64_to_double(0x0000000000000000ULL), 0.0); BOOST_CHECK_EQUAL(ser_uint64_to_double(0x3fe0000000000000ULL), 0.5); BOOST_CHECK_EQUAL(ser_uint64_to_double(0x3ff0000000000000ULL), 1.0); BOOST_CHECK_EQUAL(ser_uint64_to_double(0x4000000000000000ULL), 2.0); BOOST_CHECK_EQUAL(ser_uint64_to_double(0x4010000000000000ULL), 4.0); BOOST_CHECK_EQUAL(ser_uint64_to_double(0x4088888880000000ULL), 785.066650390625); BOOST_CHECK_EQUAL(ser_double_to_uint64(0.0), 0x0000000000000000ULL); BOOST_CHECK_EQUAL(ser_double_to_uint64(0.5), 0x3fe0000000000000ULL); BOOST_CHECK_EQUAL(ser_double_to_uint64(1.0), 0x3ff0000000000000ULL); BOOST_CHECK_EQUAL(ser_double_to_uint64(2.0), 0x4000000000000000ULL); BOOST_CHECK_EQUAL(ser_double_to_uint64(4.0), 0x4010000000000000ULL); BOOST_CHECK_EQUAL(ser_double_to_uint64(785.066650390625), 0x4088888880000000ULL); } /* Python code to generate the below hashes: def reversed_hex(x): return binascii.hexlify(''.join(reversed(x))) def dsha256(x): return hashlib.sha256(hashlib.sha256(x).digest()).digest() reversed_hex(dsha256(''.join(struct.pack('> j; BOOST_CHECK_MESSAGE(i == j, "decoded:" << j << " expected:" << i); } } BOOST_AUTO_TEST_CASE(doubles) { CDataStream ss(SER_DISK, 0); // encode for (int i = 0; i < 1000; i++) { ss << double(i); } BOOST_CHECK(Hash(ss.begin(), ss.end()) == uint256S("43d0c82591953c4eafe114590d392676a01585d25b25d433557f0" "d7878b23f96")); // decode for (int i = 0; i < 1000; i++) { double j; ss >> j; BOOST_CHECK_MESSAGE(i == j, "decoded:" << j << " expected:" << i); } } BOOST_AUTO_TEST_CASE(varints) { // encode CDataStream ss(SER_DISK, 0); CDataStream::size_type size = 0; for (int i = 0; i < 100000; i++) { - ss << VARINT(i, VarIntMode::NONNEGATIVE_SIGNED); - size += ::GetSerializeSize(VARINT(i, VarIntMode::NONNEGATIVE_SIGNED)); + ss << VARINT_MODE(i, VarIntMode::NONNEGATIVE_SIGNED); + size += + ::GetSerializeSize(VARINT_MODE(i, VarIntMode::NONNEGATIVE_SIGNED)); BOOST_CHECK(size == ss.size()); } for (uint64_t i = 0; i < 100000000000ULL; i += 999999937) { ss << VARINT(i); size += ::GetSerializeSize(VARINT(i)); BOOST_CHECK(size == ss.size()); } // decode for (int i = 0; i < 100000; i++) { int j = -1; - ss >> VARINT(j, VarIntMode::NONNEGATIVE_SIGNED); + ss >> VARINT_MODE(j, VarIntMode::NONNEGATIVE_SIGNED); BOOST_CHECK_MESSAGE(i == j, "decoded:" << j << " expected:" << i); } for (uint64_t i = 0; i < 100000000000ULL; i += 999999937) { uint64_t j = std::numeric_limits::max(); ss >> VARINT(j); BOOST_CHECK_MESSAGE(i == j, "decoded:" << j << " expected:" << i); } } BOOST_AUTO_TEST_CASE(varints_bitpatterns) { CDataStream ss(SER_DISK, 0); - ss << VARINT(0, VarIntMode::NONNEGATIVE_SIGNED); + ss << VARINT_MODE(0, VarIntMode::NONNEGATIVE_SIGNED); BOOST_CHECK_EQUAL(HexStr(ss), "00"); ss.clear(); - ss << VARINT(0x7f, VarIntMode::NONNEGATIVE_SIGNED); + ss << VARINT_MODE(0x7f, VarIntMode::NONNEGATIVE_SIGNED); BOOST_CHECK_EQUAL(HexStr(ss), "7f"); ss.clear(); - ss << VARINT((int8_t)0x7f, VarIntMode::NONNEGATIVE_SIGNED); + ss << VARINT_MODE((int8_t)0x7f, VarIntMode::NONNEGATIVE_SIGNED); BOOST_CHECK_EQUAL(HexStr(ss), "7f"); ss.clear(); - ss << VARINT(0x80, VarIntMode::NONNEGATIVE_SIGNED); + ss << VARINT_MODE(0x80, VarIntMode::NONNEGATIVE_SIGNED); BOOST_CHECK_EQUAL(HexStr(ss), "8000"); ss.clear(); ss << VARINT((uint8_t)0x80); BOOST_CHECK_EQUAL(HexStr(ss), "8000"); ss.clear(); - ss << VARINT(0x1234, VarIntMode::NONNEGATIVE_SIGNED); + ss << VARINT_MODE(0x1234, VarIntMode::NONNEGATIVE_SIGNED); BOOST_CHECK_EQUAL(HexStr(ss), "a334"); ss.clear(); - ss << VARINT((int16_t)0x1234, VarIntMode::NONNEGATIVE_SIGNED); + ss << VARINT_MODE((int16_t)0x1234, VarIntMode::NONNEGATIVE_SIGNED); BOOST_CHECK_EQUAL(HexStr(ss), "a334"); ss.clear(); - ss << VARINT(0xffff, VarIntMode::NONNEGATIVE_SIGNED); + ss << VARINT_MODE(0xffff, VarIntMode::NONNEGATIVE_SIGNED); BOOST_CHECK_EQUAL(HexStr(ss), "82fe7f"); ss.clear(); ss << VARINT((uint16_t)0xffff); BOOST_CHECK_EQUAL(HexStr(ss), "82fe7f"); ss.clear(); - ss << VARINT(0x123456, VarIntMode::NONNEGATIVE_SIGNED); + ss << VARINT_MODE(0x123456, VarIntMode::NONNEGATIVE_SIGNED); BOOST_CHECK_EQUAL(HexStr(ss), "c7e756"); ss.clear(); - ss << VARINT((int32_t)0x123456, VarIntMode::NONNEGATIVE_SIGNED); + ss << VARINT_MODE((int32_t)0x123456, VarIntMode::NONNEGATIVE_SIGNED); BOOST_CHECK_EQUAL(HexStr(ss), "c7e756"); ss.clear(); ss << VARINT(0x80123456U); BOOST_CHECK_EQUAL(HexStr(ss), "86ffc7e756"); ss.clear(); ss << VARINT((uint32_t)0x80123456U); BOOST_CHECK_EQUAL(HexStr(ss), "86ffc7e756"); ss.clear(); ss << VARINT(0xffffffff); BOOST_CHECK_EQUAL(HexStr(ss), "8efefefe7f"); ss.clear(); - ss << VARINT(0x7fffffffffffffffLL, VarIntMode::NONNEGATIVE_SIGNED); + ss << VARINT_MODE(0x7fffffffffffffffLL, VarIntMode::NONNEGATIVE_SIGNED); BOOST_CHECK_EQUAL(HexStr(ss), "fefefefefefefefe7f"); ss.clear(); ss << VARINT(0xffffffffffffffffULL); BOOST_CHECK_EQUAL(HexStr(ss), "80fefefefefefefefe7f"); ss.clear(); } static bool isTooLargeException(const std::ios_base::failure &ex) { std::ios_base::failure expectedException( "ReadCompactSize(): size too large"); // The string returned by what() can be different for different platforms. // Instead of directly comparing the ex.what() with an expected string, // create an instance of exception to see if ex.what() matches the expected // explanatory string returned by the exception instance. return strcmp(expectedException.what(), ex.what()) == 0; } BOOST_AUTO_TEST_CASE(compactsize) { CDataStream ss(SER_DISK, 0); std::vector::size_type i, j; for (i = 1; i <= MAX_SIZE; i *= 2) { WriteCompactSize(ss, i - 1); WriteCompactSize(ss, i); } for (i = 1; i <= MAX_SIZE; i *= 2) { j = ReadCompactSize(ss); BOOST_CHECK_MESSAGE((i - 1) == j, "decoded:" << j << " expected:" << (i - 1)); j = ReadCompactSize(ss); BOOST_CHECK_MESSAGE(i == j, "decoded:" << j << " expected:" << i); } WriteCompactSize(ss, MAX_SIZE); BOOST_CHECK_EQUAL(ReadCompactSize(ss), MAX_SIZE); WriteCompactSize(ss, MAX_SIZE + 1); BOOST_CHECK_EXCEPTION(ReadCompactSize(ss), std::ios_base::failure, isTooLargeException); WriteCompactSize(ss, std::numeric_limits::max()); BOOST_CHECK_EXCEPTION(ReadCompactSize(ss), std::ios_base::failure, isTooLargeException); WriteCompactSize(ss, std::numeric_limits::max()); BOOST_CHECK_EXCEPTION(ReadCompactSize(ss), std::ios_base::failure, isTooLargeException); } static bool isCanonicalException(const std::ios_base::failure &ex) { std::ios_base::failure expectedException("non-canonical ReadCompactSize()"); // The string returned by what() can be different for different platforms. // Instead of directly comparing the ex.what() with an expected string, // create an instance of exception to see if ex.what() matches the expected // explanatory string returned by the exception instance. return strcmp(expectedException.what(), ex.what()) == 0; } BOOST_AUTO_TEST_CASE(vector_bool) { std::vector vec1{1, 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 1, 1, 1, 0, 1, 0, 0, 1}; std::vector vec2{1, 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 1, 1, 1, 0, 1, 0, 0, 1}; BOOST_CHECK(vec1 == std::vector(vec2.begin(), vec2.end())); BOOST_CHECK(SerializeHash(vec1) == SerializeHash(vec2)); } BOOST_AUTO_TEST_CASE(noncanonical) { // Write some non-canonical CompactSize encodings, and make sure an // exception is thrown when read back. CDataStream ss(SER_DISK, 0); std::vector::size_type n; // zero encoded with three bytes: ss.write("\xfd\x00\x00", 3); BOOST_CHECK_EXCEPTION(ReadCompactSize(ss), std::ios_base::failure, isCanonicalException); // 0xfc encoded with three bytes: ss.write("\xfd\xfc\x00", 3); BOOST_CHECK_EXCEPTION(ReadCompactSize(ss), std::ios_base::failure, isCanonicalException); // 0xfd encoded with three bytes is OK: ss.write("\xfd\xfd\x00", 3); n = ReadCompactSize(ss); BOOST_CHECK(n == 0xfd); // zero encoded with five bytes: ss.write("\xfe\x00\x00\x00\x00", 5); BOOST_CHECK_EXCEPTION(ReadCompactSize(ss), std::ios_base::failure, isCanonicalException); // 0xffff encoded with five bytes: ss.write("\xfe\xff\xff\x00\x00", 5); BOOST_CHECK_EXCEPTION(ReadCompactSize(ss), std::ios_base::failure, isCanonicalException); // zero encoded with nine bytes: ss.write("\xff\x00\x00\x00\x00\x00\x00\x00\x00", 9); BOOST_CHECK_EXCEPTION(ReadCompactSize(ss), std::ios_base::failure, isCanonicalException); // 0x01ffffff encoded with nine bytes: ss.write("\xff\xff\xff\xff\x01\x00\x00\x00\x00", 9); BOOST_CHECK_EXCEPTION(ReadCompactSize(ss), std::ios_base::failure, isCanonicalException); } BOOST_AUTO_TEST_CASE(insert_delete) { // Test inserting/deleting bytes. CDataStream ss(SER_DISK, 0); BOOST_CHECK_EQUAL(ss.size(), 0U); ss.write("\x00\x01\x02\xff", 4); BOOST_CHECK_EQUAL(ss.size(), 4U); char c = (char)11; // Inserting at beginning/end/middle: ss.insert(ss.begin(), c); BOOST_CHECK_EQUAL(ss.size(), 5U); BOOST_CHECK_EQUAL(ss[0], c); BOOST_CHECK_EQUAL(ss[1], 0); ss.insert(ss.end(), c); BOOST_CHECK_EQUAL(ss.size(), 6U); BOOST_CHECK_EQUAL(ss[4], (char)0xff); BOOST_CHECK_EQUAL(ss[5], c); ss.insert(ss.begin() + 2, c); BOOST_CHECK_EQUAL(ss.size(), 7U); BOOST_CHECK_EQUAL(ss[2], c); // Delete at beginning/end/middle ss.erase(ss.begin()); BOOST_CHECK_EQUAL(ss.size(), 6U); BOOST_CHECK_EQUAL(ss[0], 0); ss.erase(ss.begin() + ss.size() - 1); BOOST_CHECK_EQUAL(ss.size(), 5U); BOOST_CHECK_EQUAL(ss[4], (char)0xff); ss.erase(ss.begin() + 1); BOOST_CHECK_EQUAL(ss.size(), 4U); BOOST_CHECK_EQUAL(ss[0], 0); BOOST_CHECK_EQUAL(ss[1], 1); BOOST_CHECK_EQUAL(ss[2], 2); BOOST_CHECK_EQUAL(ss[3], (char)0xff); // Make sure GetAndClear does the right thing: CSerializeData d; ss.GetAndClear(d); BOOST_CHECK_EQUAL(ss.size(), 0U); } BOOST_AUTO_TEST_CASE(class_methods) { int intval(100); bool boolval(true); std::string stringval("testing"); const char charstrval[16] = "testing charstr"; CMutableTransaction txval; CTransactionRef tx_ref{MakeTransactionRef(txval)}; CSerializeMethodsTestSingle methodtest1(intval, boolval, stringval, charstrval, tx_ref); CSerializeMethodsTestMany methodtest2(intval, boolval, stringval, charstrval, tx_ref); CSerializeMethodsTestSingle methodtest3; CSerializeMethodsTestMany methodtest4; CDataStream ss(SER_DISK, PROTOCOL_VERSION); BOOST_CHECK(methodtest1 == methodtest2); ss << methodtest1; ss >> methodtest4; ss << methodtest2; ss >> methodtest3; BOOST_CHECK(methodtest1 == methodtest2); BOOST_CHECK(methodtest2 == methodtest3); BOOST_CHECK(methodtest3 == methodtest4); CDataStream ss2(SER_DISK, PROTOCOL_VERSION, intval, boolval, stringval, charstrval, txval); ss2 >> methodtest3; BOOST_CHECK(methodtest3 == methodtest4); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/txdb.cpp b/src/txdb.cpp index 2de409ed8..ef5622b9f 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -1,564 +1,564 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2018 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include #include #include #include #include #include #include #include #include #include #include #include // boost::this_thread::interruption_point() (mingw) #include static const char DB_COIN = 'C'; static const char DB_COINS = 'c'; static const char DB_BLOCK_FILES = 'f'; static const char DB_BLOCK_INDEX = 'b'; static const char DB_BEST_BLOCK = 'B'; static const char DB_HEAD_BLOCKS = 'H'; static const char DB_FLAG = 'F'; static const char DB_REINDEX_FLAG = 'R'; static const char DB_LAST_BLOCK = 'l'; namespace { struct CoinEntry { COutPoint *outpoint; char key; explicit CoinEntry(const COutPoint *ptr) : outpoint(const_cast(ptr)), key(DB_COIN) {} template void Serialize(Stream &s) const { s << key; s << outpoint->GetTxId(); s << VARINT(outpoint->GetN()); } template void Unserialize(Stream &s) { s >> key; TxId id; s >> id; uint32_t n = 0; s >> VARINT(n); *outpoint = COutPoint(id, n); } }; } // namespace CCoinsViewDB::CCoinsViewDB(fs::path ldb_path, size_t nCacheSize, bool fMemory, bool fWipe) : db(ldb_path, nCacheSize, fMemory, fWipe, true) {} bool CCoinsViewDB::GetCoin(const COutPoint &outpoint, Coin &coin) const { return db.Read(CoinEntry(&outpoint), coin); } bool CCoinsViewDB::HaveCoin(const COutPoint &outpoint) const { return db.Exists(CoinEntry(&outpoint)); } BlockHash CCoinsViewDB::GetBestBlock() const { BlockHash hashBestChain; if (!db.Read(DB_BEST_BLOCK, hashBestChain)) { return BlockHash(); } return hashBestChain; } std::vector CCoinsViewDB::GetHeadBlocks() const { std::vector vhashHeadBlocks; if (!db.Read(DB_HEAD_BLOCKS, vhashHeadBlocks)) { return std::vector(); } return vhashHeadBlocks; } bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const BlockHash &hashBlock) { CDBBatch batch(db); size_t count = 0; size_t changed = 0; size_t batch_size = (size_t)gArgs.GetArg("-dbbatchsize", DEFAULT_DB_BATCH_SIZE); int crash_simulate = gArgs.GetArg("-dbcrashratio", 0); assert(!hashBlock.IsNull()); BlockHash old_tip = GetBestBlock(); if (old_tip.IsNull()) { // We may be in the middle of replaying. std::vector old_heads = GetHeadBlocks(); if (old_heads.size() == 2) { assert(old_heads[0] == hashBlock); old_tip = old_heads[1]; } } // In the first batch, mark the database as being in the middle of a // transition from old_tip to hashBlock. // A vector is used for future extensibility, as we may want to support // interrupting after partial writes from multiple independent reorgs. batch.Erase(DB_BEST_BLOCK); batch.Write(DB_HEAD_BLOCKS, Vector(hashBlock, old_tip)); for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end();) { if (it->second.flags & CCoinsCacheEntry::DIRTY) { CoinEntry entry(&it->first); if (it->second.coin.IsSpent()) { batch.Erase(entry); } else { batch.Write(entry, it->second.coin); } changed++; } count++; CCoinsMap::iterator itOld = it++; mapCoins.erase(itOld); if (batch.SizeEstimate() > batch_size) { LogPrint(BCLog::COINDB, "Writing partial batch of %.2f MiB\n", batch.SizeEstimate() * (1.0 / 1048576.0)); db.WriteBatch(batch); batch.Clear(); if (crash_simulate) { static FastRandomContext rng; if (rng.randrange(crash_simulate) == 0) { LogPrintf("Simulating a crash. Goodbye.\n"); _Exit(0); } } } } // In the last batch, mark the database as consistent with hashBlock again. batch.Erase(DB_HEAD_BLOCKS); batch.Write(DB_BEST_BLOCK, hashBlock); LogPrint(BCLog::COINDB, "Writing final batch of %.2f MiB\n", batch.SizeEstimate() * (1.0 / 1048576.0)); bool ret = db.WriteBatch(batch); LogPrint(BCLog::COINDB, "Committed %u changed transaction outputs (out of " "%u) to coin database...\n", (unsigned int)changed, (unsigned int)count); return ret; } size_t CCoinsViewDB::EstimateSize() const { return db.EstimateSize(DB_COIN, char(DB_COIN + 1)); } CBlockTreeDB::CBlockTreeDB(size_t nCacheSize, bool fMemory, bool fWipe) : CDBWrapper(GetDataDir() / "blocks" / "index", nCacheSize, fMemory, fWipe) {} bool CBlockTreeDB::ReadBlockFileInfo(int nFile, CBlockFileInfo &info) { return Read(std::make_pair(DB_BLOCK_FILES, nFile), info); } bool CBlockTreeDB::WriteReindexing(bool fReindexing) { if (fReindexing) { return Write(DB_REINDEX_FLAG, '1'); } else { return Erase(DB_REINDEX_FLAG); } } bool CBlockTreeDB::IsReindexing() const { return Exists(DB_REINDEX_FLAG); } bool CBlockTreeDB::ReadLastBlockFile(int &nFile) { return Read(DB_LAST_BLOCK, nFile); } CCoinsViewCursor *CCoinsViewDB::Cursor() const { CCoinsViewDBCursor *i = new CCoinsViewDBCursor( const_cast(db).NewIterator(), GetBestBlock()); /** * It seems that there are no "const iterators" for LevelDB. Since we only * need read operations on it, use a const-cast to get around that * restriction. */ i->pcursor->Seek(DB_COIN); // Cache key of first record if (i->pcursor->Valid()) { CoinEntry entry(&i->keyTmp.second); i->pcursor->GetKey(entry); i->keyTmp.first = entry.key; } else { // Make sure Valid() and GetKey() return false i->keyTmp.first = 0; } return i; } bool CCoinsViewDBCursor::GetKey(COutPoint &key) const { // Return cached key if (keyTmp.first == DB_COIN) { key = keyTmp.second; return true; } return false; } bool CCoinsViewDBCursor::GetValue(Coin &coin) const { return pcursor->GetValue(coin); } unsigned int CCoinsViewDBCursor::GetValueSize() const { return pcursor->GetValueSize(); } bool CCoinsViewDBCursor::Valid() const { return keyTmp.first == DB_COIN; } void CCoinsViewDBCursor::Next() { pcursor->Next(); CoinEntry entry(&keyTmp.second); if (!pcursor->Valid() || !pcursor->GetKey(entry)) { // Invalidate cached key after last record so that Valid() and GetKey() // return false keyTmp.first = 0; } else { keyTmp.first = entry.key; } } bool CBlockTreeDB::WriteBatchSync( const std::vector> &fileInfo, int nLastFile, const std::vector &blockinfo) { CDBBatch batch(*this); for (std::vector>::const_iterator it = fileInfo.begin(); it != fileInfo.end(); it++) { batch.Write(std::make_pair(DB_BLOCK_FILES, it->first), *it->second); } batch.Write(DB_LAST_BLOCK, nLastFile); for (std::vector::const_iterator it = blockinfo.begin(); it != blockinfo.end(); it++) { batch.Write(std::make_pair(DB_BLOCK_INDEX, (*it)->GetBlockHash()), CDiskBlockIndex(*it)); } return WriteBatch(batch, true); } bool CBlockTreeDB::WriteFlag(const std::string &name, bool fValue) { return Write(std::make_pair(DB_FLAG, name), fValue ? '1' : '0'); } bool CBlockTreeDB::ReadFlag(const std::string &name, bool &fValue) { char ch; if (!Read(std::make_pair(DB_FLAG, name), ch)) { return false; } fValue = ch == '1'; return true; } bool CBlockTreeDB::LoadBlockIndexGuts( const Consensus::Params ¶ms, std::function insertBlockIndex) { std::unique_ptr pcursor(NewIterator()); uint64_t version = 0; pcursor->Seek("version"); if (pcursor->Valid()) { pcursor->GetValue(version); } if (version != CLIENT_VERSION) { return error("%s: Invalid block index database version: %s", __func__, version); } pcursor->Seek(std::make_pair(DB_BLOCK_INDEX, uint256())); // Load m_block_index while (pcursor->Valid()) { boost::this_thread::interruption_point(); if (ShutdownRequested()) { return false; } std::pair key; if (!pcursor->GetKey(key) || key.first != DB_BLOCK_INDEX) { break; } CDiskBlockIndex diskindex; if (!pcursor->GetValue(diskindex)) { return error("%s : failed to read value", __func__); } // Construct block index object CBlockIndex *pindexNew = insertBlockIndex(diskindex.GetBlockHash()); pindexNew->pprev = insertBlockIndex(diskindex.hashPrev); pindexNew->nHeight = diskindex.nHeight; pindexNew->nFile = diskindex.nFile; pindexNew->nDataPos = diskindex.nDataPos; pindexNew->nUndoPos = diskindex.nUndoPos; pindexNew->nVersion = diskindex.nVersion; pindexNew->hashMerkleRoot = diskindex.hashMerkleRoot; pindexNew->nTime = diskindex.nTime; pindexNew->nBits = diskindex.nBits; pindexNew->nNonce = diskindex.nNonce; pindexNew->nStatus = diskindex.nStatus; pindexNew->nTx = diskindex.nTx; if (!CheckProofOfWork(pindexNew->GetBlockHash(), pindexNew->nBits, params)) { return error("%s: CheckProofOfWork failed: %s", __func__, pindexNew->ToString()); } pcursor->Next(); } return true; } namespace { //! Legacy class to deserialize pre-pertxout database entries without reindex. class CCoins { public: //! whether transaction is a coinbase bool fCoinBase; //! unspent transaction outputs; spent outputs are .IsNull(); spent outputs //! at the end of the array are dropped std::vector vout; //! at which height this transaction was included in the active block chain int nHeight; //! empty constructor CCoins() : fCoinBase(false), vout(0), nHeight(0) {} template void Unserialize(Stream &s) { uint32_t nCode = 0; // version unsigned int nVersionDummy = 0; ::Unserialize(s, VARINT(nVersionDummy)); // header code ::Unserialize(s, VARINT(nCode)); fCoinBase = nCode & 1; std::vector vAvail(2, false); vAvail[0] = (nCode & 2) != 0; vAvail[1] = (nCode & 4) != 0; uint32_t nMaskCode = (nCode / 8) + ((nCode & 6) != 0 ? 0 : 1); // spentness bitmask while (nMaskCode > 0) { uint8_t chAvail = 0; ::Unserialize(s, chAvail); for (unsigned int p = 0; p < 8; p++) { bool f = (chAvail & (1 << p)) != 0; vAvail.push_back(f); } if (chAvail != 0) { nMaskCode--; } } // txouts themself vout.assign(vAvail.size(), CTxOut()); for (size_t i = 0; i < vAvail.size(); i++) { if (vAvail[i]) { ::Unserialize(s, Using(vout[i])); } } // coinbase height - ::Unserialize(s, VARINT(nHeight, VarIntMode::NONNEGATIVE_SIGNED)); + ::Unserialize(s, VARINT_MODE(nHeight, VarIntMode::NONNEGATIVE_SIGNED)); } }; } // namespace /** * Upgrade the database from older formats. * * Currently implemented: from the per-tx utxo model (0.8..0.14.x) to per-txout. */ bool CCoinsViewDB::Upgrade() { std::unique_ptr pcursor(db.NewIterator()); pcursor->Seek(std::make_pair(DB_COINS, uint256())); if (!pcursor->Valid()) { return true; } int64_t count = 0; LogPrintf("Upgrading utxo-set database...\n"); size_t batch_size = 1 << 24; CDBBatch batch(db); int reportDone = -1; std::pair key; std::pair prev_key = {DB_COINS, uint256()}; while (pcursor->Valid()) { boost::this_thread::interruption_point(); if (ShutdownRequested()) { break; } if (!pcursor->GetKey(key) || key.first != DB_COINS) { break; } if (count++ % 256 == 0) { uint32_t high = 0x100 * *key.second.begin() + *(key.second.begin() + 1); int percentageDone = (int)(high * 100.0 / 65536.0 + 0.5); uiInterface.ShowProgress(_("Upgrading UTXO database").translated, percentageDone, true); if (reportDone < percentageDone / 10) { // report max. every 10% step LogPrintfToBeContinued("[%d%%]...", percentageDone); reportDone = percentageDone / 10; } } CCoins old_coins; if (!pcursor->GetValue(old_coins)) { return error("%s: cannot parse CCoins record", __func__); } const TxId id(key.second); for (size_t i = 0; i < old_coins.vout.size(); ++i) { if (!old_coins.vout[i].IsNull() && !old_coins.vout[i].scriptPubKey.IsUnspendable()) { Coin newcoin(std::move(old_coins.vout[i]), old_coins.nHeight, old_coins.fCoinBase); COutPoint outpoint(id, i); CoinEntry entry(&outpoint); batch.Write(entry, newcoin); } } batch.Erase(key); if (batch.SizeEstimate() > batch_size) { db.WriteBatch(batch); batch.Clear(); db.CompactRange(prev_key, key); prev_key = key; } pcursor->Next(); } db.WriteBatch(batch); db.CompactRange({DB_COINS, uint256()}, key); uiInterface.ShowProgress("", 100, false); LogPrintf("[%s].\n", ShutdownRequested() ? "CANCELLED" : "DONE"); return !ShutdownRequested(); } bool CBlockTreeDB::Upgrade(const Consensus::Params ¶ms) { std::unique_ptr pcursor(NewIterator()); uint64_t version = 0; pcursor->Seek("version"); if (pcursor->Valid()) { pcursor->GetValue(version); } if (version >= CLIENT_VERSION) { // The DB is already up to date. return true; } CDBBatch batch(*this); pcursor->Seek(std::make_pair(DB_BLOCK_INDEX, uint256())); if (!pcursor->Valid()) { // The DB is empty, so just write the version number and consider the // upgrade done. batch.Write("version", uint64_t(CLIENT_VERSION)); WriteBatch(batch); return true; } int64_t count = 0; LogPrintf("Upgrading block index database...\n"); int reportDone = -1; std::pair key = {DB_BLOCK_INDEX, uint256()}; while (pcursor->Valid()) { boost::this_thread::interruption_point(); if (ShutdownRequested()) { break; } if (!pcursor->GetKey(key) || key.first != DB_BLOCK_INDEX) { break; } if (count++ % 256 == 0) { uint32_t high = 0x100 * *key.second.begin() + *(key.second.begin() + 1); int percentageDone = (int)(high * 100.0 / 65536.0 + 0.5); uiInterface.ShowProgress( _("Upgrading block index database").translated, percentageDone, true); if (reportDone < percentageDone / 10) { // report max. every 10% step LogPrintfToBeContinued("[%d%%]...", percentageDone); reportDone = percentageDone / 10; } } // Read the block index entry and update it. CDiskBlockIndex diskindex; if (!pcursor->GetValue(diskindex)) { return error("%s: cannot parse CDiskBlockIndex record", __func__); } // The block hash needs to be usable. BlockHash blockhash = diskindex.GetBlockHash(); diskindex.phashBlock = &blockhash; bool mustUpdate = false; // We must update the block index to add the size. if (CLIENT_VERSION >= CDiskBlockIndex::TRACK_SIZE_VERSION && version < CDiskBlockIndex::TRACK_SIZE_VERSION && diskindex.nTx > 0 && diskindex.nSize == 0) { if (!diskindex.nStatus.hasData()) { // The block was pruned, we need a full reindex. LogPrintf("\nThe block %s is pruned. The block index cannot be " "upgraded and reindexing is required.\n", blockhash.GetHex()); return false; } CBlock block; if (!ReadBlockFromDisk(block, &diskindex, params)) { // Failed to read the block from disk, even though it is marked // that we have data for this block. return false; } mustUpdate = true; diskindex.nSize = ::GetSerializeSize(block, PROTOCOL_VERSION); } if (mustUpdate) { batch.Write(std::make_pair(DB_BLOCK_INDEX, blockhash), diskindex); } pcursor->Next(); } // Upgrade is done, now let's update the version number. batch.Write("version", uint64_t(CLIENT_VERSION)); WriteBatch(batch); CompactRange({DB_BLOCK_INDEX, uint256()}, key); uiInterface.ShowProgress("", 100, false); LogPrintf("[%s].\n", ShutdownRequested() ? "CANCELLED" : "DONE"); return !ShutdownRequested(); }