Page MenuHomePhabricator
Differential D560 Diff 1442 src/seeder/serialize.h

Changeset View

Standalone View

View Options

src/seeder/serialize.h

// Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2010 Satoshi Nakamoto
// Copyright (c) 2011 The Bitcoin developers // Copyright (c) 2011 The Bitcoin developers
// Distributed under the MIT/X11 software license, see the accompanying // Distributed under the MIT/X11 software license, see the accompanying
// file license.txt or http://www.opensource.org/licenses/mit-license.php. // file license.txt or http://www.opensource.org/licenses/mit-license.php.
#ifndef BITCOIN_SERIALIZE_H #ifndef BITCOIN_SEEDER_SERIALIZE_H
#define BITCOIN_SERIALIZE_H #define BITCOIN_SEEDER_SERIALIZE_H
#include <cassert> #include <cassert>
#include <climits> #include <climits>
#include <cstdio> #include <cstdio>
#include <cstring> #include <cstring>
#include <map> #include <map>
#include <set> #include <set>
#include <string> #include <string>
Show All 21 Lines
// Note that VirtualLock does not provide this as a guarantee on Windows, // Note that VirtualLock does not provide this as a guarantee on Windows,
// but, in practice, memory that has been VirtualLock'd almost never gets // but, in practice, memory that has been VirtualLock'd almost never gets
// written to // written to
// the pagefile except in rare circumstances where memory is extremely low. // the pagefile except in rare circumstances where memory is extremely low.
#include <windows.h> #include <windows.h>
#define mlock(p, n) VirtualLock((p), (n)); #define mlock(p, n) VirtualLock((p), (n));
#define munlock(p, n) VirtualUnlock((p), (n)); #define munlock(p, n) VirtualUnlock((p), (n));
#else #else
#include <limits.h> #include <climits>
#include <sys/mman.h> #include <sys/mman.h>
/* This comes from limits.h if it's not defined there set a sane default */ /* This comes from limits.h if it's not defined there set a sane default */
#ifndef PAGESIZE #ifndef PAGESIZE
#include <unistd.h> #include <unistd.h>
#define PAGESIZE sysconf(_SC_PAGESIZE) #define PAGESIZE sysconf(_SC_PAGESIZE)
#endif #endif
#define mlock(a, b) \ #define mlock(a, b) \
mlock(((void *)(((size_t)(a)) & (~((PAGESIZE)-1)))), \ mlock(((void *)(((size_t)(a)) & (~((PAGESIZE)-1)))), \
▲ Show 20 Lines • Show All 80 Lines • ▼ Show 20 Lines
#define READDATA(s, obj) s.read((char *)&(obj), sizeof(obj)) #define READDATA(s, obj) s.read((char *)&(obj), sizeof(obj))
inline unsigned int GetSerializeSize(char a, int, int = 0) { inline unsigned int GetSerializeSize(char a, int, int = 0) {
return sizeof(a); return sizeof(a);
} }
inline unsigned int GetSerializeSize(signed char a, int, int = 0) { inline unsigned int GetSerializeSize(signed char a, int, int = 0) {
return sizeof(a); return sizeof(a);
} }
inline unsigned int GetSerializeSize(unsigned char a, int, int = 0) { inline unsigned int GetSerializeSize(uint8_t a, int, int = 0) {
return sizeof(a); return sizeof(a);
} }
inline unsigned int GetSerializeSize(signed short a, int, int = 0) { inline unsigned int GetSerializeSize(signed short a, int, int = 0) {
return sizeof(a); return sizeof(a);
} }
inline unsigned int GetSerializeSize(unsigned short a, int, int = 0) { inline unsigned int GetSerializeSize(unsigned short a, int, int = 0) {
return sizeof(a); return sizeof(a);
} }
Show All 26 Lines
inline void Serialize(Stream &s, char a, int, int = 0) { inline void Serialize(Stream &s, char a, int, int = 0) {
WRITEDATA(s, a); WRITEDATA(s, a);
} }
template <typename Stream> template <typename Stream>
inline void Serialize(Stream &s, signed char a, int, int = 0) { inline void Serialize(Stream &s, signed char a, int, int = 0) {
WRITEDATA(s, a); WRITEDATA(s, a);
} }
template <typename Stream> template <typename Stream>
inline void Serialize(Stream &s, unsigned char a, int, int = 0) { inline void Serialize(Stream &s, uint8_t a, int, int = 0) {
WRITEDATA(s, a); WRITEDATA(s, a);
} }
template <typename Stream> template <typename Stream>
inline void Serialize(Stream &s, signed short a, int, int = 0) { inline void Serialize(Stream &s, signed short a, int, int = 0) {
WRITEDATA(s, a); WRITEDATA(s, a);
} }
template <typename Stream> template <typename Stream>
inline void Serialize(Stream &s, unsigned short a, int, int = 0) { inline void Serialize(Stream &s, unsigned short a, int, int = 0) {
Show All 36 Lines
inline void Unserialize(Stream &s, char &a, int, int = 0) { inline void Unserialize(Stream &s, char &a, int, int = 0) {
READDATA(s, a); READDATA(s, a);
} }
template <typename Stream> template <typename Stream>
inline void Unserialize(Stream &s, signed char &a, int, int = 0) { inline void Unserialize(Stream &s, signed char &a, int, int = 0) {
READDATA(s, a); READDATA(s, a);
} }
template <typename Stream> template <typename Stream>
inline void Unserialize(Stream &s, unsigned char &a, int, int = 0) { inline void Unserialize(Stream &s, uint8_t &a, int, int = 0) {
READDATA(s, a); READDATA(s, a);
} }
template <typename Stream> template <typename Stream>
inline void Unserialize(Stream &s, signed short &a, int, int = 0) { inline void Unserialize(Stream &s, signed short &a, int, int = 0) {
READDATA(s, a); READDATA(s, a);
} }
template <typename Stream> template <typename Stream>
inline void Unserialize(Stream &s, unsigned short &a, int, int = 0) { inline void Unserialize(Stream &s, unsigned short &a, int, int = 0) {
▲ Show 20 Lines • Show All 51 Lines • ▼ Show 20 Lines
// Compact size // Compact size
// size < 253 -- 1 byte // size < 253 -- 1 byte
// size <= USHRT_MAX -- 3 bytes (253 + 2 bytes) // size <= USHRT_MAX -- 3 bytes (253 + 2 bytes)
// size <= UINT_MAX -- 5 bytes (254 + 4 bytes) // size <= UINT_MAX -- 5 bytes (254 + 4 bytes)
// size > UINT_MAX -- 9 bytes (255 + 8 bytes) // size > UINT_MAX -- 9 bytes (255 + 8 bytes)
// //
inline unsigned int GetSizeOfCompactSize(uint64 nSize) { inline unsigned int GetSizeOfCompactSize(uint64 nSize) {
if (nSize < 253) if (nSize < 253)
return sizeof(unsigned char); return sizeof(uint8_t);
else if (nSize <= USHRT_MAX) else if (nSize <= USHRT_MAX)
return sizeof(unsigned char) + sizeof(unsigned short); return sizeof(uint8_t) + sizeof(unsigned short);
else if (nSize <= UINT_MAX) else if (nSize <= UINT_MAX)
return sizeof(unsigned char) + sizeof(unsigned int); return sizeof(uint8_t) + sizeof(unsigned int);
else else
return sizeof(unsigned char) + sizeof(uint64); return sizeof(uint8_t) + sizeof(uint64);
} }
template <typename Stream> void WriteCompactSize(Stream &os, uint64 nSize) { template <typename Stream> void WriteCompactSize(Stream &os, uint64 nSize) {
if (nSize < 253) { if (nSize < 253) {
unsigned char chSize = nSize; uint8_t chSize = nSize;
WRITEDATA(os, chSize); WRITEDATA(os, chSize);
} else if (nSize <= USHRT_MAX) { } else if (nSize <= USHRT_MAX) {
unsigned char chSize = 253; uint8_t chSize = 253;
unsigned short xSize = nSize; unsigned short xSize = nSize;
WRITEDATA(os, chSize); WRITEDATA(os, chSize);
WRITEDATA(os, xSize); WRITEDATA(os, xSize);
} else if (nSize <= UINT_MAX) { } else if (nSize <= UINT_MAX) {
unsigned char chSize = 254; uint8_t chSize = 254;
unsigned int xSize = nSize; unsigned int xSize = nSize;
WRITEDATA(os, chSize); WRITEDATA(os, chSize);
WRITEDATA(os, xSize); WRITEDATA(os, xSize);
} else { } else {
unsigned char chSize = 255; uint8_t chSize = 255;
uint64 xSize = nSize; uint64 xSize = nSize;
WRITEDATA(os, chSize); WRITEDATA(os, chSize);
WRITEDATA(os, xSize); WRITEDATA(os, xSize);
} }
return; return;
} }
template <typename Stream> uint64 ReadCompactSize(Stream &is) { template <typename Stream> uint64 ReadCompactSize(Stream &is) {
unsigned char chSize; uint8_t chSize;
READDATA(is, chSize); READDATA(is, chSize);
uint64 nSizeRet = 0; uint64 nSizeRet = 0;
if (chSize < 253) { if (chSize < 253) {
nSizeRet = chSize; nSizeRet = chSize;
} else if (chSize == 253) { } else if (chSize == 253) {
unsigned short xSize; unsigned short xSize;
READDATA(is, xSize); READDATA(is, xSize);
nSizeRet = xSize; nSizeRet = xSize;
▲ Show 20 Lines • Show All 47 Lines • ▼ Show 20 Lines
// //
template <std::size_t LEN> class CFixedFieldString { template <std::size_t LEN> class CFixedFieldString {
protected: protected:
const std::string *pcstr; const std::string *pcstr;
std::string *pstr; std::string *pstr;
public: public:
explicit CFixedFieldString(const std::string &str) explicit CFixedFieldString(const std::string &str)
: pcstr(&str), pstr(NULL) {} : pcstr(&str), pstr(nullptr) {}
explicit CFixedFieldString(std::string &str) : pcstr(&str), pstr(&str) {} explicit CFixedFieldString(std::string &str) : pcstr(&str), pstr(&str) {}
unsigned int GetSerializeSize(int, int = 0) const { return LEN; } unsigned int GetSerializeSize(int, int = 0) const { return LEN; }
template <typename Stream> void Serialize(Stream &s, int, int = 0) const { template <typename Stream> void Serialize(Stream &s, int, int = 0) const {
char pszBuf[LEN]; char pszBuf[LEN];
strncpy(pszBuf, pcstr->c_str(), LEN); strncpy(pszBuf, pcstr->c_str(), LEN);
s.write(pszBuf, LEN); s.write(pszBuf, LEN);
} }
template <typename Stream> void Unserialize(Stream &s, int, int = 0) { template <typename Stream> void Unserialize(Stream &s, int, int = 0) {
if (pstr == NULL) if (pstr == nullptr)
throw std::ios_base::failure("CFixedFieldString::Unserialize : " throw std::ios_base::failure("CFixedFieldString::Unserialize : "
"trying to unserialize to const " "trying to unserialize to const "
"string"); "string");
char pszBuf[LEN + 1]; char pszBuf[LEN + 1];
s.read(pszBuf, LEN); s.read(pszBuf, LEN);
pszBuf[LEN] = '\0'; pszBuf[LEN] = '\0';
*pstr = pszBuf; *pstr = pszBuf;
} }
▲ Show 20 Lines • Show All 248 Lines • ▼ Show 20 Linesinline void Unserialize(Stream &is, std::vector<T, A> &v, int nType,
Unserialize_impl(is, v, nType, nVersion, boost::is_fundamental<T>()); Unserialize_impl(is, v, nType, nVersion, boost::is_fundamental<T>());
} }
// //
// others derived from vector // others derived from vector
// //
inline unsigned int GetSerializeSize(const CScript &v, int nType, inline unsigned int GetSerializeSize(const CScript &v, int nType,
int nVersion) { int nVersion) {
return GetSerializeSize((const std::vector<unsigned char> &)v, nType, return GetSerializeSize((const std::vector<uint8_t> &)v, nType, nVersion);
nVersion);
} }
template <typename Stream> template <typename Stream>
void Serialize(Stream &os, const CScript &v, int nType, int nVersion) { void Serialize(Stream &os, const CScript &v, int nType, int nVersion) {
Serialize(os, (const std::vector<unsigned char> &)v, nType, nVersion); Serialize(os, (const std::vector<uint8_t> &)v, nType, nVersion);
} }
template <typename Stream> template <typename Stream>
void Unserialize(Stream &is, CScript &v, int nType, int nVersion) { void Unserialize(Stream &is, CScript &v, int nType, int nVersion) {
Unserialize(is, (std::vector<unsigned char> &)v, nType, nVersion); Unserialize(is, (std::vector<uint8_t> &)v, nType, nVersion);
} }
// //
// pair // pair
// //
template <typename K, typename T> template <typename K, typename T>
unsigned int GetSerializeSize(const std::pair<K, T> &item, int nType, unsigned int GetSerializeSize(const std::pair<K, T> &item, int nType,
int nVersion) { int nVersion) {
▲ Show 20 Lines • Show All 198 Lines • ▼ Show 20 Linestemplate <typename T> struct secure_allocator : public std::allocator<T> {
~secure_allocator() throw() {} ~secure_allocator() throw() {}
template <typename _Other> struct rebind { template <typename _Other> struct rebind {
typedef secure_allocator<_Other> other; typedef secure_allocator<_Other> other;
}; };
T *allocate(std::size_t n, const void *hint = 0) { T *allocate(std::size_t n, const void *hint = 0) {
T *p; T *p;
p = std::allocator<T>::allocate(n, hint); p = std::allocator<T>::allocate(n, hint);
if (p != NULL) mlock(p, sizeof(T) * n); if (p != nullptr) mlock(p, sizeof(T) * n);
return p; return p;
} }
void deallocate(T *p, std::size_t n) { void deallocate(T *p, std::size_t n) {
if (p != NULL) { if (p != nullptr) {
memset(p, 0, sizeof(T) * n); memset(p, 0, sizeof(T) * n);
munlock(p, sizeof(T) * n); munlock(p, sizeof(T) * n);
} }
std::allocator<T>::deallocate(p, n); std::allocator<T>::deallocate(p, n);
} }
}; };
// //
▲ Show 20 Lines • Show All 51 Lines • ▼ Show 20 Lines#endif
} }
CDataStream(const std::vector<char> &vchIn, int nTypeIn = SER_NETWORK, CDataStream(const std::vector<char> &vchIn, int nTypeIn = SER_NETWORK,
int nVersionIn = PROTOCOL_VERSION) int nVersionIn = PROTOCOL_VERSION)
: vch(vchIn.begin(), vchIn.end()) { : vch(vchIn.begin(), vchIn.end()) {
Init(nTypeIn, nVersionIn); Init(nTypeIn, nVersionIn);
} }
CDataStream(const std::vector<unsigned char> &vchIn, CDataStream(const std::vector<uint8_t> &vchIn, int nTypeIn = SER_NETWORK,
int nTypeIn = SER_NETWORK, int nVersionIn = PROTOCOL_VERSION) int nVersionIn = PROTOCOL_VERSION)
: vch((char *)&vchIn.begin()[0], (char *)&vchIn.end()[0]) { : vch((char *)&vchIn.begin()[0], (char *)&vchIn.end()[0]) {
Init(nTypeIn, nVersionIn); Init(nTypeIn, nVersionIn);
} }
void Init(int nTypeIn = SER_NETWORK, int nVersionIn = PROTOCOL_VERSION) { void Init(int nTypeIn = SER_NETWORK, int nVersionIn = PROTOCOL_VERSION) {
nReadPos = 0; nReadPos = 0;
nType = nTypeIn; nType = nTypeIn;
nVersion = nVersionIn; nVersion = nVersionIn;
▲ Show 20 Lines • Show All 224 Lines • ▼ Show 20 Lines
// n=8000 87 seconds // n=8000 87 seconds
// n=16000 400 seconds // n=16000 400 seconds
// n=32000 1660 seconds // n=32000 1660 seconds
// n=64000 6749 seconds // n=64000 6749 seconds
// n=128000 27241 seconds // n=128000 27241 seconds
// n=256000 109804 seconds // n=256000 109804 seconds
#include <iostream> #include <iostream>
int main(int argc, char *argv[]) { int main(int argc, char *argv[]) {
vector<unsigned char> vch(0xcc, 250); vector<uint8_t> vch(0xcc, 250);
printf("CDataStream:\n"); printf("CDataStream:\n");
for (int n = 1000; n <= 4500000; n *= 2) { for (int n = 1000; n <= 4500000; n *= 2) {
CDataStream ss; CDataStream ss;
time_t nStart = time(NULL); time_t nStart = time(nullptr);
for (int i = 0; i < n; i++) for (int i = 0; i < n; i++)
ss.write((char *)&vch[0], vch.size()); ss.write((char *)&vch[0], vch.size());
printf("n=%-10d %d seconds\n", n, time(NULL) - nStart); printf("n=%-10d %d seconds\n", n, time(nullptr) - nStart);
} }
printf("stringstream:\n"); printf("stringstream:\n");
for (int n = 1000; n <= 4500000; n *= 2) { for (int n = 1000; n <= 4500000; n *= 2) {
stringstream ss; stringstream ss;
time_t nStart = time(NULL); time_t nStart = time(nullptr);
for (int i = 0; i < n; i++) for (int i = 0; i < n; i++)
ss.write((char *)&vch[0], vch.size()); ss.write((char *)&vch[0], vch.size());
printf("n=%-10d %d seconds\n", n, time(NULL) - nStart); printf("n=%-10d %d seconds\n", n, time(nullptr) - nStart);
} }
} }
#endif #endif
// //
// Automatic closing wrapper for FILE* // Automatic closing wrapper for FILE*
// - Will automatically close the file when it goes out of scope if not null. // - Will automatically close the file when it goes out of scope if not null.
// - If you're returning the file pointer, return file.release(). // - If you're returning the file pointer, return file.release().
// - If you need to close the file early, use file.fclose() instead of // - If you need to close the file early, use file.fclose() instead of
// fclose(file). // fclose(file).
// //
class CAutoFile { class CAutoFile {
protected: protected:
FILE *file; FILE *file;
short state; short state;
short exceptmask; short exceptmask;
public: public:
int nType; int nType;
int nVersion; int nVersion;
typedef FILE element_type; typedef FILE element_type;
CAutoFile(FILE *filenew = NULL, int nTypeIn = SER_DISK, CAutoFile(FILE *filenew = nullptr, int nTypeIn = SER_DISK,
int nVersionIn = PROTOCOL_VERSION) { int nVersionIn = PROTOCOL_VERSION) {
file = filenew; file = filenew;
nType = nTypeIn; nType = nTypeIn;
nVersion = nVersionIn; nVersion = nVersionIn;
state = 0; state = 0;
exceptmask = std::ios::badbit | std::ios::failbit; exceptmask = std::ios::badbit | std::ios::failbit;
} }
~CAutoFile() { fclose(); } ~CAutoFile() { fclose(); }
void fclose() { void fclose() {
if (file != NULL && file != stdin && file != stdout && file != stderr) if (file != nullptr && file != stdin && file != stdout &&
file != stderr)
::fclose(file); ::fclose(file);
file = NULL; file = nullptr;
} }
FILE *release() { FILE *release() {
FILE *ret = file; FILE *ret = file;
file = NULL; file = nullptr;
return ret; return ret;
} }
operator FILE *() { return file; } operator FILE *() { return file; }
FILE *operator->() { return file; } FILE *operator->() { return file; }
FILE &operator*() { return *file; } FILE &operator*() { return *file; }
FILE **operator&() { return &file; } FILE **operator&() { return &file; }
FILE *operator=(FILE *pnew) { return file = pnew; } FILE *operator=(FILE *pnew) { return file = pnew; }
bool operator!() { return (file == NULL); } bool operator!() { return (file == nullptr); }
// //
// Stream subset // Stream subset
// //
void setstate(short bits, const char *psz) { void setstate(short bits, const char *psz) {
state |= bits; state |= bits;
if (state & exceptmask) throw std::ios_base::failure(psz); if (state & exceptmask) throw std::ios_base::failure(psz);
} }
Show All 14 Linespublic:
void SetVersion(int n) { nVersion = n; } void SetVersion(int n) { nVersion = n; }
int GetVersion() { return nVersion; } int GetVersion() { return nVersion; }
void ReadVersion() { *this >> nVersion; } void ReadVersion() { *this >> nVersion; }
void WriteVersion() { *this << nVersion; } void WriteVersion() { *this << nVersion; }
CAutoFile &read(char *pch, int nSize) { CAutoFile &read(char *pch, int nSize) {
if (!file) if (!file)
throw std::ios_base::failure( throw std::ios_base::failure(
"CAutoFile::read : file handle is NULL"); "CAutoFile::read : file handle is nullptr");
if (fread(pch, 1, nSize, file) != nSize) if (fread(pch, 1, nSize, file) != nSize)
setstate(std::ios::failbit, feof(file) setstate(std::ios::failbit, feof(file)
? "CAutoFile::read : end of file" ? "CAutoFile::read : end of file"
: "CAutoFile::read : fread failed"); : "CAutoFile::read : fread failed");
return (*this); return (*this);
} }
CAutoFile &write(const char *pch, int nSize) { CAutoFile &write(const char *pch, int nSize) {
if (!file) if (!file)
throw std::ios_base::failure( throw std::ios_base::failure(
"CAutoFile::write : file handle is NULL"); "CAutoFile::write : file handle is nullptr");
if (fwrite(pch, 1, nSize, file) != nSize) if (fwrite(pch, 1, nSize, file) != nSize)
setstate(std::ios::failbit, "CAutoFile::write : write failed"); setstate(std::ios::failbit, "CAutoFile::write : write failed");
return (*this); return (*this);
} }
template <typename T> unsigned int GetSerializeSize(const T &obj) { template <typename T> unsigned int GetSerializeSize(const T &obj) {
// Tells the size of the object if serialized to this stream // Tells the size of the object if serialized to this stream
return ::GetSerializeSize(obj, nType, nVersion); return ::GetSerializeSize(obj, nType, nVersion);
} }
template <typename T> CAutoFile &operator<<(const T &obj) { template <typename T> CAutoFile &operator<<(const T &obj) {
// Serialize to this stream // Serialize to this stream
if (!file) if (!file)
throw std::ios_base::failure( throw std::ios_base::failure(
"CAutoFile::operator<< : file handle is NULL"); "CAutoFile::operator<< : file handle is nullptr");
::Serialize(*this, obj, nType, nVersion); ::Serialize(*this, obj, nType, nVersion);
return (*this); return (*this);
} }
template <typename T> CAutoFile &operator>>(T &obj) { template <typename T> CAutoFile &operator>>(T &obj) {
// Unserialize from this stream // Unserialize from this stream
if (!file) if (!file)
throw std::ios_base::failure( throw std::ios_base::failure(
"CAutoFile::operator>> : file handle is NULL"); "CAutoFile::operator>> : file handle is nullptr");
::Unserialize(*this, obj, nType, nVersion); ::Unserialize(*this, obj, nType, nVersion);
return (*this); return (*this);
} }
}; };
#endif #endif