Changeset View
Changeset View
Standalone View
Standalone View
src/serialize.h
Show First 20 Lines • Show All 238 Lines • ▼ Show 20 Lines | |||||
/** | /** | ||||
* Compact Size | * Compact Size | ||||
* size < 253 -- 1 byte | * size < 253 -- 1 byte | ||||
* size <= USHRT_MAX -- 3 bytes (253 + 2 bytes) | * size <= USHRT_MAX -- 3 bytes (253 + 2 bytes) | ||||
* size <= UINT_MAX -- 5 bytes (254 + 4 bytes) | * size <= UINT_MAX -- 5 bytes (254 + 4 bytes) | ||||
* size > UINT_MAX -- 9 bytes (255 + 8 bytes) | * size > UINT_MAX -- 9 bytes (255 + 8 bytes) | ||||
*/ | */ | ||||
inline unsigned int GetSizeOfCompactSize(uint64_t nSize) { | inline uint32_t GetSizeOfCompactSize(uint64_t nSize) { | ||||
if (nSize < 253) | if (nSize < 253) { | ||||
return sizeof(uint8_t); | return sizeof(uint8_t); | ||||
else if (nSize <= std::numeric_limits<unsigned short>::max()) | } | ||||
return sizeof(uint8_t) + sizeof(unsigned short); | if (nSize <= std::numeric_limits<uint16_t>::max()) { | ||||
else if (nSize <= std::numeric_limits<unsigned int>::max()) | return sizeof(uint8_t) + sizeof(uint16_t); | ||||
return sizeof(uint8_t) + sizeof(unsigned int); | } | ||||
else | if (nSize <= std::numeric_limits<uint32_t>::max()) { | ||||
return sizeof(uint8_t) + sizeof(uint32_t); | |||||
} | |||||
return sizeof(uint8_t) + sizeof(uint64_t); | return sizeof(uint8_t) + sizeof(uint64_t); | ||||
} | } | ||||
inline void WriteCompactSize(CSizeComputer &os, uint64_t nSize); | inline void WriteCompactSize(CSizeComputer &os, uint64_t nSize); | ||||
template <typename Stream> void WriteCompactSize(Stream &os, uint64_t nSize) { | template <typename Stream> void WriteCompactSize(Stream &os, uint64_t nSize) { | ||||
if (nSize < 253) { | if (nSize < 253) { | ||||
ser_writedata8(os, nSize); | ser_writedata8(os, nSize); | ||||
} else if (nSize <= std::numeric_limits<unsigned short>::max()) { | } else if (nSize <= std::numeric_limits<uint16_t>::max()) { | ||||
ser_writedata8(os, 253); | ser_writedata8(os, 253); | ||||
ser_writedata16(os, nSize); | ser_writedata16(os, nSize); | ||||
} else if (nSize <= std::numeric_limits<unsigned int>::max()) { | } else if (nSize <= std::numeric_limits<uint32_t>::max()) { | ||||
ser_writedata8(os, 254); | ser_writedata8(os, 254); | ||||
ser_writedata32(os, nSize); | ser_writedata32(os, nSize); | ||||
} else { | } else { | ||||
ser_writedata8(os, 255); | ser_writedata8(os, 255); | ||||
ser_writedata64(os, nSize); | ser_writedata64(os, nSize); | ||||
} | } | ||||
return; | return; | ||||
} | } | ||||
template <typename Stream> uint64_t ReadCompactSize(Stream &is) { | template <typename Stream> uint64_t ReadCompactSize(Stream &is) { | ||||
uint8_t chSize = ser_readdata8(is); | uint8_t chSize = ser_readdata8(is); | ||||
uint64_t nSizeRet = 0; | uint64_t nSizeRet = 0; | ||||
if (chSize < 253) { | if (chSize < 253) { | ||||
nSizeRet = chSize; | nSizeRet = chSize; | ||||
} else if (chSize == 253) { | } else if (chSize == 253) { | ||||
nSizeRet = ser_readdata16(is); | nSizeRet = ser_readdata16(is); | ||||
if (nSizeRet < 253) | if (nSizeRet < 253) { | ||||
throw std::ios_base::failure("non-canonical ReadCompactSize()"); | throw std::ios_base::failure("non-canonical ReadCompactSize()"); | ||||
} | |||||
} else if (chSize == 254) { | } else if (chSize == 254) { | ||||
nSizeRet = ser_readdata32(is); | nSizeRet = ser_readdata32(is); | ||||
if (nSizeRet < 0x10000u) | if (nSizeRet < 0x10000u) { | ||||
throw std::ios_base::failure("non-canonical ReadCompactSize()"); | throw std::ios_base::failure("non-canonical ReadCompactSize()"); | ||||
} | |||||
} else { | } else { | ||||
nSizeRet = ser_readdata64(is); | nSizeRet = ser_readdata64(is); | ||||
if (nSizeRet < 0x100000000ULL) | if (nSizeRet < 0x100000000ULL) { | ||||
throw std::ios_base::failure("non-canonical ReadCompactSize()"); | throw std::ios_base::failure("non-canonical ReadCompactSize()"); | ||||
} | } | ||||
} | |||||
if (nSizeRet > MAX_SIZE) { | if (nSizeRet > MAX_SIZE) { | ||||
throw std::ios_base::failure("ReadCompactSize(): size too large"); | throw std::ios_base::failure("ReadCompactSize(): size too large"); | ||||
} | } | ||||
return nSizeRet; | return nSizeRet; | ||||
} | } | ||||
/** | /** | ||||
* Variable-length integers: bytes are a MSB base-128 encoding of the number. | * Variable-length integers: bytes are a MSB base-128 encoding of the number. | ||||
Show All 17 Lines | |||||
* 128: [0x80 0x00] 16511: [0xFF 0x7F] | * 128: [0x80 0x00] 16511: [0xFF 0x7F] | ||||
* 255: [0x80 0x7F] 65535: [0x82 0xFE 0x7F] | * 255: [0x80 0x7F] 65535: [0x82 0xFE 0x7F] | ||||
* 2^32: [0x8E 0xFE 0xFE 0xFF 0x00] | * 2^32: [0x8E 0xFE 0xFE 0xFF 0x00] | ||||
*/ | */ | ||||
template <typename I> inline unsigned int GetSizeOfVarInt(I n) { | template <typename I> inline unsigned int GetSizeOfVarInt(I n) { | ||||
int nRet = 0; | int nRet = 0; | ||||
while (true) { | while (true) { | ||||
nRet++; | nRet++; | ||||
if (n <= 0x7F) break; | if (n <= 0x7F) { | ||||
return nRet; | |||||
} | |||||
n = (n >> 7) - 1; | n = (n >> 7) - 1; | ||||
} | } | ||||
return nRet; | |||||
} | } | ||||
template <typename I> inline void WriteVarInt(CSizeComputer &os, I n); | template <typename I> inline void WriteVarInt(CSizeComputer &os, I n); | ||||
template <typename Stream, typename I> void WriteVarInt(Stream &os, I n) { | template <typename Stream, typename I> void WriteVarInt(Stream &os, I n) { | ||||
uint8_t tmp[(sizeof(n) * 8 + 6) / 7]; | uint8_t tmp[(sizeof(n) * 8 + 6) / 7]; | ||||
int len = 0; | int len = 0; | ||||
while (true) { | while (true) { | ||||
tmp[len] = (n & 0x7F) | (len ? 0x80 : 0x00); | tmp[len] = (n & 0x7F) | (len ? 0x80 : 0x00); | ||||
if (n <= 0x7F) break; | if (n <= 0x7F) { | ||||
break; | |||||
} | |||||
n = (n >> 7) - 1; | n = (n >> 7) - 1; | ||||
len++; | len++; | ||||
} | } | ||||
do { | do { | ||||
ser_writedata8(os, tmp[len]); | ser_writedata8(os, tmp[len]); | ||||
} while (len--); | } while (len--); | ||||
} | } | ||||
template <typename Stream, typename I> I ReadVarInt(Stream &is) { | template <typename Stream, typename I> I ReadVarInt(Stream &is) { | ||||
I n = 0; | I n = 0; | ||||
while (true) { | while (true) { | ||||
uint8_t chData = ser_readdata8(is); | uint8_t chData = ser_readdata8(is); | ||||
n = (n << 7) | (chData & 0x7F); | n = (n << 7) | (chData & 0x7F); | ||||
if (chData & 0x80) | if ((chData & 0x80) == 0) { | ||||
n++; | |||||
else | |||||
return n; | return n; | ||||
} | } | ||||
n++; | |||||
} | |||||
} | } | ||||
#define FLATDATA(obj) \ | #define FLATDATA(obj) \ | ||||
REF(CFlatData((char *)&(obj), (char *)&(obj) + sizeof(obj))) | REF(CFlatData((char *)&(obj), (char *)&(obj) + sizeof(obj))) | ||||
#define VARINT(obj) REF(WrapVarInt(REF(obj))) | #define VARINT(obj) REF(WrapVarInt(REF(obj))) | ||||
#define COMPACTSIZE(obj) REF(CCompactSize(REF(obj))) | #define COMPACTSIZE(obj) REF(CCompactSize(REF(obj))) | ||||
#define LIMITED_STRING(obj, n) REF(LimitedString<n>(REF(obj))) | #define LIMITED_STRING(obj, n) REF(LimitedString<n>(REF(obj))) | ||||
▲ Show 20 Lines • Show All 71 Lines • ▼ Show 20 Lines | public: | ||||
LimitedString(std::string &_string) : string(_string) {} | LimitedString(std::string &_string) : string(_string) {} | ||||
template <typename Stream> void Unserialize(Stream &s) { | template <typename Stream> void Unserialize(Stream &s) { | ||||
size_t size = ReadCompactSize(s); | size_t size = ReadCompactSize(s); | ||||
if (size > Limit) { | if (size > Limit) { | ||||
throw std::ios_base::failure("String length limit exceeded"); | throw std::ios_base::failure("String length limit exceeded"); | ||||
} | } | ||||
string.resize(size); | string.resize(size); | ||||
if (size != 0) s.read((char *)&string[0], size); | if (size != 0) { | ||||
s.read((char *)&string[0], size); | |||||
} | |||||
} | } | ||||
template <typename Stream> void Serialize(Stream &s) const { | template <typename Stream> void Serialize(Stream &s) const { | ||||
WriteCompactSize(s, string.size()); | WriteCompactSize(s, string.size()); | ||||
if (!string.empty()) s.write((char *)&string[0], string.size()); | if (!string.empty()) { | ||||
s.write((char *)&string[0], string.size()); | |||||
} | |||||
} | } | ||||
}; | }; | ||||
template <typename I> CVarInt<I> WrapVarInt(I &n) { | template <typename I> CVarInt<I> WrapVarInt(I &n) { | ||||
return CVarInt<I>(n); | return CVarInt<I>(n); | ||||
} | } | ||||
/** | /** | ||||
▲ Show 20 Lines • Show All 99 Lines • ▼ Show 20 Lines | |||||
} | } | ||||
/** | /** | ||||
* string | * string | ||||
*/ | */ | ||||
template <typename Stream, typename C> | template <typename Stream, typename C> | ||||
void Serialize(Stream &os, const std::basic_string<C> &str) { | void Serialize(Stream &os, const std::basic_string<C> &str) { | ||||
WriteCompactSize(os, str.size()); | WriteCompactSize(os, str.size()); | ||||
if (!str.empty()) os.write((char *)&str[0], str.size() * sizeof(str[0])); | if (!str.empty()) { | ||||
os.write((char *)&str[0], str.size() * sizeof(str[0])); | |||||
} | |||||
} | } | ||||
template <typename Stream, typename C> | template <typename Stream, typename C> | ||||
void Unserialize(Stream &is, std::basic_string<C> &str) { | void Unserialize(Stream &is, std::basic_string<C> &str) { | ||||
unsigned int nSize = ReadCompactSize(is); | size_t nSize = ReadCompactSize(is); | ||||
str.resize(nSize); | str.resize(nSize); | ||||
if (nSize != 0) is.read((char *)&str[0], nSize * sizeof(str[0])); | if (nSize != 0) { | ||||
is.read((char *)&str[0], nSize * sizeof(str[0])); | |||||
} | |||||
} | } | ||||
/** | /** | ||||
* prevector | * prevector | ||||
*/ | */ | ||||
template <typename Stream, unsigned int N, typename T> | template <typename Stream, unsigned int N, typename T> | ||||
void Serialize_impl(Stream &os, const prevector<N, T> &v, const uint8_t &) { | void Serialize_impl(Stream &os, const prevector<N, T> &v, const uint8_t &) { | ||||
WriteCompactSize(os, v.size()); | WriteCompactSize(os, v.size()); | ||||
if (!v.empty()) os.write((char *)&v[0], v.size() * sizeof(T)); | if (!v.empty()) { | ||||
os.write((char *)&v[0], v.size() * sizeof(T)); | |||||
} | |||||
} | } | ||||
template <typename Stream, unsigned int N, typename T, typename V> | template <typename Stream, unsigned int N, typename T, typename V> | ||||
void Serialize_impl(Stream &os, const prevector<N, T> &v, const V &) { | void Serialize_impl(Stream &os, const prevector<N, T> &v, const V &) { | ||||
WriteCompactSize(os, v.size()); | WriteCompactSize(os, v.size()); | ||||
for (typename prevector<N, T>::const_iterator vi = v.begin(); vi != v.end(); | for (const T &i : v) { | ||||
++vi) | ::Serialize(os, i); | ||||
::Serialize(os, (*vi)); | } | ||||
} | } | ||||
template <typename Stream, unsigned int N, typename T> | template <typename Stream, unsigned int N, typename T> | ||||
inline void Serialize(Stream &os, const prevector<N, T> &v) { | inline void Serialize(Stream &os, const prevector<N, T> &v) { | ||||
Serialize_impl(os, v, T()); | Serialize_impl(os, v, T()); | ||||
} | } | ||||
template <typename Stream, unsigned int N, typename T> | template <typename Stream, unsigned int N, typename T> | ||||
void Unserialize_impl(Stream &is, prevector<N, T> &v, const uint8_t &) { | void Unserialize_impl(Stream &is, prevector<N, T> &v, const uint8_t &) { | ||||
// Limit size per read so bogus size value won't cause out of memory | // Limit size per read so bogus size value won't cause out of memory | ||||
v.clear(); | v.clear(); | ||||
unsigned int nSize = ReadCompactSize(is); | size_t nSize = ReadCompactSize(is); | ||||
unsigned int i = 0; | size_t i = 0; | ||||
while (i < nSize) { | while (i < nSize) { | ||||
unsigned int blk = | size_t blk = std::min(nSize - i, size_t(1 + 4999999 / sizeof(T))); | ||||
std::min(nSize - i, (unsigned int)(1 + 4999999 / sizeof(T))); | |||||
v.resize(i + blk); | v.resize(i + blk); | ||||
is.read((char *)&v[i], blk * sizeof(T)); | is.read((char *)&v[i], blk * sizeof(T)); | ||||
i += blk; | i += blk; | ||||
} | } | ||||
} | } | ||||
template <typename Stream, unsigned int N, typename T, typename V> | template <typename Stream, unsigned int N, typename T, typename V> | ||||
void Unserialize_impl(Stream &is, prevector<N, T> &v, const V &) { | void Unserialize_impl(Stream &is, prevector<N, T> &v, const V &) { | ||||
v.clear(); | v.clear(); | ||||
unsigned int nSize = ReadCompactSize(is); | size_t nSize = ReadCompactSize(is); | ||||
unsigned int i = 0; | size_t i = 0; | ||||
unsigned int nMid = 0; | size_t nMid = 0; | ||||
while (nMid < nSize) { | while (nMid < nSize) { | ||||
nMid += 5000000 / sizeof(T); | nMid += 5000000 / sizeof(T); | ||||
if (nMid > nSize) nMid = nSize; | if (nMid > nSize) { | ||||
nMid = nSize; | |||||
} | |||||
v.resize(nMid); | v.resize(nMid); | ||||
for (; i < nMid; i++) | for (; i < nMid; i++) { | ||||
Unserialize(is, v[i]); | Unserialize(is, v[i]); | ||||
} | } | ||||
} | } | ||||
} | |||||
template <typename Stream, unsigned int N, typename T> | template <typename Stream, unsigned int N, typename T> | ||||
inline void Unserialize(Stream &is, prevector<N, T> &v) { | inline void Unserialize(Stream &is, prevector<N, T> &v) { | ||||
Unserialize_impl(is, v, T()); | Unserialize_impl(is, v, T()); | ||||
} | } | ||||
/** | /** | ||||
* vector | * vector | ||||
*/ | */ | ||||
template <typename Stream, typename T, typename A> | template <typename Stream, typename T, typename A> | ||||
void Serialize_impl(Stream &os, const std::vector<T, A> &v, const uint8_t &) { | void Serialize_impl(Stream &os, const std::vector<T, A> &v, const uint8_t &) { | ||||
WriteCompactSize(os, v.size()); | WriteCompactSize(os, v.size()); | ||||
if (!v.empty()) os.write((char *)&v[0], v.size() * sizeof(T)); | if (!v.empty()) { | ||||
os.write((char *)&v[0], v.size() * sizeof(T)); | |||||
} | |||||
} | } | ||||
template <typename Stream, typename T, typename A, typename V> | template <typename Stream, typename T, typename A, typename V> | ||||
void Serialize_impl(Stream &os, const std::vector<T, A> &v, const V &) { | void Serialize_impl(Stream &os, const std::vector<T, A> &v, const V &) { | ||||
WriteCompactSize(os, v.size()); | WriteCompactSize(os, v.size()); | ||||
for (typename std::vector<T, A>::const_iterator vi = v.begin(); | for (const T &i : v) { | ||||
vi != v.end(); ++vi) | ::Serialize(os, i); | ||||
::Serialize(os, (*vi)); | } | ||||
} | } | ||||
template <typename Stream, typename T, typename A> | template <typename Stream, typename T, typename A> | ||||
inline void Serialize(Stream &os, const std::vector<T, A> &v) { | inline void Serialize(Stream &os, const std::vector<T, A> &v) { | ||||
Serialize_impl(os, v, T()); | Serialize_impl(os, v, T()); | ||||
} | } | ||||
template <typename Stream, typename T, typename A> | template <typename Stream, typename T, typename A> | ||||
void Unserialize_impl(Stream &is, std::vector<T, A> &v, const uint8_t &) { | void Unserialize_impl(Stream &is, std::vector<T, A> &v, const uint8_t &) { | ||||
// Limit size per read so bogus size value won't cause out of memory | // Limit size per read so bogus size value won't cause out of memory | ||||
v.clear(); | v.clear(); | ||||
unsigned int nSize = ReadCompactSize(is); | size_t nSize = ReadCompactSize(is); | ||||
unsigned int i = 0; | size_t i = 0; | ||||
while (i < nSize) { | while (i < nSize) { | ||||
unsigned int blk = | size_t blk = std::min(nSize - i, size_t(1 + 4999999 / sizeof(T))); | ||||
std::min(nSize - i, (unsigned int)(1 + 4999999 / sizeof(T))); | |||||
v.resize(i + blk); | v.resize(i + blk); | ||||
is.read((char *)&v[i], blk * sizeof(T)); | is.read((char *)&v[i], blk * sizeof(T)); | ||||
i += blk; | i += blk; | ||||
} | } | ||||
} | } | ||||
template <typename Stream, typename T, typename A, typename V> | template <typename Stream, typename T, typename A, typename V> | ||||
void Unserialize_impl(Stream &is, std::vector<T, A> &v, const V &) { | void Unserialize_impl(Stream &is, std::vector<T, A> &v, const V &) { | ||||
v.clear(); | v.clear(); | ||||
unsigned int nSize = ReadCompactSize(is); | size_t nSize = ReadCompactSize(is); | ||||
unsigned int i = 0; | size_t i = 0; | ||||
unsigned int nMid = 0; | size_t nMid = 0; | ||||
while (nMid < nSize) { | while (nMid < nSize) { | ||||
nMid += 5000000 / sizeof(T); | nMid += 5000000 / sizeof(T); | ||||
if (nMid > nSize) nMid = nSize; | if (nMid > nSize) { | ||||
nMid = nSize; | |||||
} | |||||
v.resize(nMid); | v.resize(nMid); | ||||
for (; i < nMid; i++) | for (; i < nMid; i++) { | ||||
Unserialize(is, v[i]); | Unserialize(is, v[i]); | ||||
} | } | ||||
} | } | ||||
} | |||||
template <typename Stream, typename T, typename A> | template <typename Stream, typename T, typename A> | ||||
inline void Unserialize(Stream &is, std::vector<T, A> &v) { | inline void Unserialize(Stream &is, std::vector<T, A> &v) { | ||||
Unserialize_impl(is, v, T()); | Unserialize_impl(is, v, T()); | ||||
} | } | ||||
/** | /** | ||||
* pair | * pair | ||||
Show All 11 Lines | |||||
} | } | ||||
/** | /** | ||||
* map | * map | ||||
*/ | */ | ||||
template <typename Stream, typename K, typename T, typename Pred, typename A> | template <typename Stream, typename K, typename T, typename Pred, typename A> | ||||
void Serialize(Stream &os, const std::map<K, T, Pred, A> &m) { | void Serialize(Stream &os, const std::map<K, T, Pred, A> &m) { | ||||
WriteCompactSize(os, m.size()); | WriteCompactSize(os, m.size()); | ||||
for (typename std::map<K, T, Pred, A>::const_iterator mi = m.begin(); | for (const std::pair<K, T> &p : m) { | ||||
mi != m.end(); ++mi) | Serialize(os, p); | ||||
Serialize(os, (*mi)); | } | ||||
} | } | ||||
template <typename Stream, typename K, typename T, typename Pred, typename A> | template <typename Stream, typename K, typename T, typename Pred, typename A> | ||||
void Unserialize(Stream &is, std::map<K, T, Pred, A> &m) { | void Unserialize(Stream &is, std::map<K, T, Pred, A> &m) { | ||||
m.clear(); | m.clear(); | ||||
unsigned int nSize = ReadCompactSize(is); | size_t nSize = ReadCompactSize(is); | ||||
typename std::map<K, T, Pred, A>::iterator mi = m.begin(); | typename std::map<K, T, Pred, A>::iterator mi = m.begin(); | ||||
for (unsigned int i = 0; i < nSize; i++) { | for (size_t i = 0; i < nSize; i++) { | ||||
std::pair<K, T> item; | std::pair<K, T> item; | ||||
Unserialize(is, item); | Unserialize(is, item); | ||||
mi = m.insert(mi, item); | mi = m.insert(mi, item); | ||||
} | } | ||||
} | } | ||||
/** | /** | ||||
* set | * set | ||||
*/ | */ | ||||
template <typename Stream, typename K, typename Pred, typename A> | template <typename Stream, typename K, typename Pred, typename A> | ||||
void Serialize(Stream &os, const std::set<K, Pred, A> &m) { | void Serialize(Stream &os, const std::set<K, Pred, A> &m) { | ||||
WriteCompactSize(os, m.size()); | WriteCompactSize(os, m.size()); | ||||
for (typename std::set<K, Pred, A>::const_iterator it = m.begin(); | for (const K &i : m) { | ||||
it != m.end(); ++it) | Serialize(os, i); | ||||
Serialize(os, (*it)); | } | ||||
} | } | ||||
template <typename Stream, typename K, typename Pred, typename A> | template <typename Stream, typename K, typename Pred, typename A> | ||||
void Unserialize(Stream &is, std::set<K, Pred, A> &m) { | void Unserialize(Stream &is, std::set<K, Pred, A> &m) { | ||||
m.clear(); | m.clear(); | ||||
unsigned int nSize = ReadCompactSize(is); | size_t nSize = ReadCompactSize(is); | ||||
typename std::set<K, Pred, A>::iterator it = m.begin(); | typename std::set<K, Pred, A>::iterator it = m.begin(); | ||||
for (unsigned int i = 0; i < nSize; i++) { | for (size_t i = 0; i < nSize; i++) { | ||||
K key; | K key; | ||||
Unserialize(is, key); | Unserialize(is, key); | ||||
it = m.insert(it, key); | it = m.insert(it, key); | ||||
} | } | ||||
} | } | ||||
/** | /** | ||||
* unique_ptr | * unique_ptr | ||||
▲ Show 20 Lines • Show All 141 Lines • Show Last 20 Lines |