diff --git a/src/streams.h b/src/streams.h index ca9a0674f..dbf70decc 100644 --- a/src/streams.h +++ b/src/streams.h @@ -1,816 +1,836 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2016 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_STREAMS_H #define BITCOIN_STREAMS_H #include #include #include #include #include #include #include #include #include #include #include #include #include #include template class OverrideStream { Stream *stream; const int nType; const int nVersion; public: OverrideStream(Stream *stream_, int nType_, int nVersion_) : stream(stream_), nType(nType_), nVersion(nVersion_) {} template OverrideStream &operator<<(const T &obj) { // Serialize to this stream ::Serialize(*this, obj); return (*this); } template OverrideStream &operator>>(T &&obj) { // Unserialize from this stream ::Unserialize(*this, obj); return (*this); } void write(Span src) { stream->write(src); } void read(Span dst) { stream->read(dst); } int GetVersion() const { return nVersion; } int GetType() const { return nType; } void ignore(size_t size) { return stream->ignore(size); } }; template OverrideStream WithOrVersion(S *s, int nVersionFlag) { return OverrideStream(s, s->GetType(), s->GetVersion() | nVersionFlag); } /** * Minimal stream for overwriting and/or appending to an existing byte vector. * * The referenced vector will grow as necessary. */ class CVectorWriter { public: /** * @param[in] nTypeIn Serialization Type * @param[in] nVersionIn Serialization Version (including any flags) * @param[in] vchDataIn Referenced byte vector to overwrite/append * @param[in] nPosIn Starting position. Vector index where writes should * start. The vector will initially grow as necessary to max(nPosIn, * vec.size()). So to append, use vec.size(). */ CVectorWriter(int nTypeIn, int nVersionIn, std::vector &vchDataIn, size_t nPosIn) : nType(nTypeIn), nVersion(nVersionIn), vchData(vchDataIn), nPos(nPosIn) { if (nPos > vchData.size()) { vchData.resize(nPos); } } /** * (other params same as above) * @param[in] args A list of items to serialize starting at nPosIn. */ template CVectorWriter(int nTypeIn, int nVersionIn, std::vector &vchDataIn, size_t nPosIn, Args &&...args) : CVectorWriter(nTypeIn, nVersionIn, vchDataIn, nPosIn) { ::SerializeMany(*this, std::forward(args)...); } void write(Span src) { assert(nPos <= vchData.size()); size_t nOverwrite = std::min(src.size(), vchData.size() - nPos); if (nOverwrite) { memcpy(vchData.data() + nPos, src.data(), nOverwrite); } if (nOverwrite < src.size()) { vchData.insert(vchData.end(), UCharCast(src.data()) + nOverwrite, UCharCast(src.end())); } nPos += src.size(); } template CVectorWriter &operator<<(const T &obj) { // Serialize to this stream ::Serialize(*this, obj); return (*this); } int GetVersion() const { return nVersion; } int GetType() const { return nType; } void seek(size_t nSize) { nPos += nSize; if (nPos > vchData.size()) { vchData.resize(nPos); } } private: const int nType; const int nVersion; std::vector &vchData; size_t nPos; }; /** * Minimal stream for reading from an existing byte array by Span. */ class SpanReader { private: const int m_type; const int m_version; Span m_data; public: /** * @param[in] type Serialization Type * @param[in] version Serialization Version (including any flags) * @param[in] data Referenced byte vector to overwrite/append */ SpanReader(int type, int version, Span data) : m_type(type), m_version(version), m_data(data) {} template SpanReader &operator>>(T &obj) { // Unserialize from this stream ::Unserialize(*this, obj); return (*this); } int GetVersion() const { return m_version; } int GetType() const { return m_type; } size_t size() const { return m_data.size(); } bool empty() const { return m_data.empty(); } void read(Span dst) { if (dst.size() == 0) { return; } // Read from the beginning of the buffer if (dst.size() > m_data.size()) { throw std::ios_base::failure("SpanReader::read(): end of data"); } memcpy(dst.data(), m_data.data(), dst.size()); m_data = m_data.subspan(dst.size()); } }; /** * Double ended buffer combining vector and stream-like interfaces. * * >> and << read and write unformatted data using the above serialization * templates. Fills with data in linear time; some stringstream implementations * take N^2 time. */ class CDataStream { protected: using vector_type = SerializeData; vector_type vch; unsigned int nReadPos{0}; int nType; int nVersion; public: typedef vector_type::allocator_type allocator_type; typedef vector_type::size_type size_type; typedef vector_type::difference_type difference_type; typedef vector_type::reference reference; typedef vector_type::const_reference const_reference; typedef vector_type::value_type value_type; typedef vector_type::iterator iterator; typedef vector_type::const_iterator const_iterator; typedef vector_type::reverse_iterator reverse_iterator; explicit CDataStream(int nTypeIn, int nVersionIn) : nType{nTypeIn}, nVersion{nVersionIn} {} explicit CDataStream(Span sp, int type, int version) : CDataStream{AsBytes(sp), type, version} {} explicit CDataStream(Span sp, int nTypeIn, int nVersionIn) : vch(sp.data(), sp.data() + sp.size()), nType{nTypeIn}, nVersion{nVersionIn} {} template CDataStream(int nTypeIn, int nVersionIn, Args &&...args) : nType{nTypeIn}, nVersion{nVersionIn} { ::SerializeMany(*this, std::forward(args)...); } std::string str() const { return std::string{UCharCast(data()), UCharCast(data() + size())}; } // // Vector subset // const_iterator begin() const { return vch.begin() + nReadPos; } iterator begin() { return vch.begin() + nReadPos; } const_iterator end() const { return vch.end(); } iterator end() { return vch.end(); } size_type size() const { return vch.size() - nReadPos; } bool empty() const { return vch.size() == nReadPos; } void resize(size_type n, value_type c = value_type{}) { vch.resize(n + nReadPos, c); } void reserve(size_type n) { vch.reserve(n + nReadPos); } const_reference operator[](size_type pos) const { return vch[pos + nReadPos]; } reference operator[](size_type pos) { return vch[pos + nReadPos]; } void clear() { vch.clear(); nReadPos = 0; } iterator insert(iterator it, const value_type x) { return vch.insert(it, x); } void insert(iterator it, size_type n, const value_type x) { vch.insert(it, n, x); } value_type *data() { return vch.data() + nReadPos; } const value_type *data() const { return vch.data() + nReadPos; } void insert(iterator it, std::vector::const_iterator first, std::vector::const_iterator last) { if (last == first) { return; } assert(last - first > 0); if (it == vch.begin() + nReadPos && (unsigned int)(last - first) <= nReadPos) { // special case for inserting at the front when there's room nReadPos -= (last - first); memcpy(&vch[nReadPos], &first[0], last - first); } else { vch.insert(it, first, last); } } // This was added to have full compat with the std::vector interface but is // unused (except in a Bitcoin ABC specific test in stream_tests) void insert(iterator it, const value_type *first, const value_type *last) { if (last == first) { return; } assert(last - first > 0); if (it == vch.begin() + nReadPos && (unsigned int)(last - first) <= nReadPos) { // special case for inserting at the front when there's room nReadPos -= (last - first); memcpy(&vch[nReadPos], &first[0], last - first); } else { vch.insert(it, first, last); } } iterator erase(iterator it) { if (it == vch.begin() + nReadPos) { // special case for erasing from the front if (++nReadPos >= vch.size()) { // whenever we reach the end, we take the opportunity to clear // the buffer nReadPos = 0; return vch.erase(vch.begin(), vch.end()); } return vch.begin() + nReadPos; } else { return vch.erase(it); } } iterator erase(iterator first, iterator last) { if (first == vch.begin() + nReadPos) { // special case for erasing from the front if (last == vch.end()) { nReadPos = 0; return vch.erase(vch.begin(), vch.end()); } else { nReadPos = (last - vch.begin()); return last; } } else return vch.erase(first, last); } inline void Compact() { vch.erase(vch.begin(), vch.begin() + nReadPos); nReadPos = 0; } bool Rewind(std::optional n = std::nullopt) { // Total rewind if no size is passed if (!n) { nReadPos = 0; return true; } // Rewind by n characters if the buffer hasn't been compacted yet if (*n > nReadPos) { return false; } nReadPos -= *n; return true; } // // Stream subset // bool eof() const { return size() == 0; } CDataStream *rdbuf() { return this; } int in_avail() const { return size(); } void SetType(int n) { nType = n; } int GetType() const { return nType; } void SetVersion(int n) { nVersion = n; } int GetVersion() const { return nVersion; } void read(Span dst) { if (dst.size() == 0) { return; } // Read from the beginning of the buffer unsigned int nReadPosNext = nReadPos + dst.size(); if (nReadPosNext > vch.size()) { throw std::ios_base::failure("CDataStream::read(): end of data"); } memcpy(dst.data(), &vch[nReadPos], dst.size()); if (nReadPosNext == vch.size()) { nReadPos = 0; vch.clear(); return; } nReadPos = nReadPosNext; } void ignore(int nSize) { // Ignore from the beginning of the buffer if (nSize < 0) { throw std::ios_base::failure( "CDataStream::ignore(): nSize negative"); } unsigned int nReadPosNext = nReadPos + nSize; if (nReadPosNext >= vch.size()) { if (nReadPosNext > vch.size()) { throw std::ios_base::failure( "CDataStream::ignore(): end of data"); } nReadPos = 0; vch.clear(); return; } nReadPos = nReadPosNext; } void write(Span src) { // Write to the end of the buffer vch.insert(vch.end(), src.begin(), src.end()); } template void Serialize(Stream &s) const { // Special case: stream << stream concatenates like stream += stream if (!vch.empty()) { s.write(MakeByteSpan(vch)); } } template CDataStream &operator<<(const T &obj) { // Serialize to this stream ::Serialize(*this, obj); return (*this); } template CDataStream &operator>>(T &&obj) { // Unserialize from this stream ::Unserialize(*this, obj); return (*this); } /** * XOR the contents of this stream with a certain key. * * @param[in] key The key used to XOR the data in this stream. */ void Xor(const std::vector &key) { if (key.size() == 0) { return; } for (size_type i = 0, j = 0; i != size(); i++) { vch[i] ^= std::byte{key[j++]}; // This potentially acts on very many bytes of data, so it's // important that we calculate `j`, i.e. the `key` index in this way // instead of doing a %, which would effectively be a division for // each byte Xor'd -- much slower than need be. if (j == key.size()) j = 0; } } }; template class BitStreamReader { private: IStream &m_istream; /// Buffered byte read in from the input stream. A new byte is read into the /// buffer when m_offset reaches 8. uint8_t m_buffer{0}; /// Number of high order bits in m_buffer already returned by previous /// Read() calls. The next bit to be returned is at this offset from the /// most significant bit position. int m_offset{8}; public: explicit BitStreamReader(IStream &istream) : m_istream(istream) {} /** * Read the specified number of bits from the stream. The data is returned * in the nbits least significant bits of a 64-bit uint. */ uint64_t Read(int nbits) { if (nbits < 0 || nbits > 64) { throw std::out_of_range("nbits must be between 0 and 64"); } uint64_t data = 0; while (nbits > 0) { if (m_offset == 8) { m_istream >> m_buffer; m_offset = 0; } int bits = std::min(8 - m_offset, nbits); data <<= bits; data |= static_cast(m_buffer << m_offset) >> (8 - bits); m_offset += bits; nbits -= bits; } return data; } }; template class BitStreamWriter { private: OStream &m_ostream; /// Buffered byte waiting to be written to the output stream. The byte is /// written buffer when m_offset reaches 8 or Flush() is called. uint8_t m_buffer{0}; /// Number of high order bits in m_buffer already written by previous /// Write() calls and not yet flushed to the stream. The next bit to be /// written to is at this offset from the most significant bit position. int m_offset{0}; public: explicit BitStreamWriter(OStream &ostream) : m_ostream(ostream) {} ~BitStreamWriter() { Flush(); } /** * Write the nbits least significant bits of a 64-bit int to the output * stream. Data is buffered until it completes an octet. */ void Write(uint64_t data, int nbits) { if (nbits < 0 || nbits > 64) { throw std::out_of_range("nbits must be between 0 and 64"); } while (nbits > 0) { int bits = std::min(8 - m_offset, nbits); m_buffer |= (data << (64 - nbits)) >> (64 - 8 + m_offset); m_offset += bits; nbits -= bits; if (m_offset == 8) { Flush(); } } } /** * Flush any unwritten bits to the output stream, padding with 0's to the * next byte boundary. */ void Flush() { if (m_offset == 0) { return; } m_ostream << m_buffer; m_buffer = 0; m_offset = 0; } }; /** * Non-refcounted RAII wrapper for FILE* * * Will automatically close the file when it goes out of scope if not null. If * you're returning the file pointer, return file.release(). If you need to * close the file early, use file.fclose() instead of fclose(file). */ class AutoFile { protected: FILE *file; public: explicit AutoFile(FILE *filenew) : file{filenew} {} ~AutoFile() { fclose(); } // Disallow copies AutoFile(const AutoFile &) = delete; AutoFile &operator=(const AutoFile &) = delete; int fclose() { int retval{0}; if (file) { retval = ::fclose(file); file = nullptr; } return retval; } /** * Get wrapped FILE* with transfer of ownership. * @note This will invalidate the AutoFile object, and makes it the * responsibility of the caller of this function to clean up the returned * FILE*. */ FILE *release() { FILE *ret = file; file = nullptr; return ret; } /** * Get wrapped FILE* without transfer of ownership. * @note Ownership of the FILE* will remain with this class. Use this only * if the scope of the AutoFile outlives use of the passed pointer. */ FILE *Get() const { return file; } /** Return true if the wrapped FILE* is nullptr, false otherwise. */ bool IsNull() const { return (file == nullptr); } // // Stream subset // void read(Span dst) { if (!file) { throw std::ios_base::failure( "AutoFile::read: file handle is nullptr"); } if (fread(dst.data(), 1, dst.size(), file) != dst.size()) { throw std::ios_base::failure(feof(file) ? "AutoFile::read: end of file" : "AutoFile::read: fread failed"); } } void ignore(size_t nSize) { if (!file) { throw std::ios_base::failure( "AutoFile::ignore: file handle is nullptr"); } uint8_t data[4096]; while (nSize > 0) { size_t nNow = std::min(nSize, sizeof(data)); if (fread(data, 1, nNow, file) != nNow) { throw std::ios_base::failure( feof(file) ? "AutoFile::ignore: end of file" : "AutoFile::read: fread failed"); } nSize -= nNow; } } void write(Span src) { if (!file) { throw std::ios_base::failure( "AutoFile::write: file handle is nullptr"); } if (fwrite(src.data(), 1, src.size(), file) != src.size()) { throw std::ios_base::failure("AutoFile::write: write failed"); } } template AutoFile &operator<<(const T &obj) { if (!file) throw std::ios_base::failure( "AutoFile::operator<<: file handle is nullptr"); ::Serialize(*this, obj); return *this; } template AutoFile &operator>>(T &&obj) { if (!file) throw std::ios_base::failure( "AutoFile::operator>>: file handle is nullptr"); ::Unserialize(*this, obj); return *this; } }; class CAutoFile : public AutoFile { private: const int nType; const int nVersion; public: CAutoFile(FILE *filenew, int nTypeIn, int nVersionIn) : AutoFile{filenew}, nType(nTypeIn), nVersion(nVersionIn) {} int GetType() const { return nType; } int GetVersion() const { return nVersion; } template CAutoFile &operator<<(const T &obj) { // Serialize to this stream if (!file) { throw std::ios_base::failure( "CAutoFile::operator<<: file handle is nullptr"); } ::Serialize(*this, obj); return (*this); } template CAutoFile &operator>>(T &&obj) { // Unserialize from this stream if (!file) { throw std::ios_base::failure( "CAutoFile::operator>>: file handle is nullptr"); } ::Unserialize(*this, obj); return (*this); } }; /** * Non-refcounted RAII wrapper around a FILE* that implements a ring buffer to * deserialize from. It guarantees the ability to rewind a given number of * bytes. * * Will automatically close the file when it goes out of scope if not null. If * you need to close the file early, use file.fclose() instead of fclose(file). */ class CBufferedFile { private: const int nType; const int nVersion; //! source file FILE *src; //! how many bytes have been read from source uint64_t nSrcPos; //! how many bytes have been read from this uint64_t nReadPos; //! up to which position we're allowed to read uint64_t nReadLimit; //! how many bytes we guarantee to rewind uint64_t nRewind; //! the buffer std::vector vchBuf; -protected: //! read data from the source to fill the buffer bool Fill() { unsigned int pos = nSrcPos % vchBuf.size(); unsigned int readNow = vchBuf.size() - pos; unsigned int nAvail = vchBuf.size() - (nSrcPos - nReadPos) - nRewind; if (nAvail < readNow) { readNow = nAvail; } if (readNow == 0) { return false; } size_t nBytes = fread((void *)&vchBuf[pos], 1, readNow, src); if (nBytes == 0) { throw std::ios_base::failure( feof(src) ? "CBufferedFile::Fill: end of file" : "CBufferedFile::Fill: fread failed"); } nSrcPos += nBytes; return true; } + //! Advance the stream's read pointer (m_read_pos) by up to 'length' bytes, + //! filling the buffer from the file so that at least one byte is available. + //! Return a pointer to the available buffer data and the number of bytes + //! (which may be less than the requested length) that may be accessed + //! beginning at that pointer. + std::pair AdvanceStream(size_t length) { + assert(nReadPos <= nSrcPos); + if (nReadPos + length > nReadLimit) { + throw std::ios_base::failure( + "Attempt to position past buffer limit"); + } + // If there are no bytes available, read from the file. + if (nReadPos == nSrcPos && length > 0) { + Fill(); + } + + size_t buffer_offset{static_cast(nReadPos % vchBuf.size())}; + size_t buffer_available{ + static_cast(vchBuf.size() - buffer_offset)}; + size_t bytes_until_source_pos{static_cast(nSrcPos - nReadPos)}; + size_t advance{ + std::min({length, buffer_available, bytes_until_source_pos})}; + nReadPos += advance; + return std::make_pair(&vchBuf[buffer_offset], advance); + } + public: CBufferedFile(FILE *fileIn, uint64_t nBufSize, uint64_t nRewindIn, int nTypeIn, int nVersionIn) : nType(nTypeIn), nVersion(nVersionIn), nSrcPos(0), nReadPos(0), nReadLimit(std::numeric_limits::max()), nRewind(nRewindIn), vchBuf(nBufSize, std::byte{0}) { if (nRewindIn >= nBufSize) { throw std::ios_base::failure( "Rewind limit must be less than buffer size"); } src = fileIn; } ~CBufferedFile() { fclose(); } // Disallow copies CBufferedFile(const CBufferedFile &) = delete; CBufferedFile &operator=(const CBufferedFile &) = delete; int GetVersion() const { return nVersion; } int GetType() const { return nType; } void fclose() { if (src) { ::fclose(src); src = nullptr; } } //! check whether we're at the end of the source file bool eof() const { return nReadPos == nSrcPos && feof(src); } //! read a number of bytes void read(Span dst) { - if (dst.size() + nReadPos > nReadLimit) { - throw std::ios_base::failure("Read attempted past buffer limit"); - } while (dst.size() > 0) { - if (nReadPos == nSrcPos) { - Fill(); - } - unsigned int pos = nReadPos % vchBuf.size(); - size_t nNow = dst.size(); - if (nNow + pos > vchBuf.size()) { - nNow = vchBuf.size() - pos; - } - if (nNow + nReadPos > nSrcPos) { - nNow = nSrcPos - nReadPos; - } - memcpy(dst.data(), &vchBuf[pos], nNow); - nReadPos += nNow; - dst = dst.subspan(nNow); + auto [buffer_pointer, length]{AdvanceStream(dst.size())}; + memcpy(dst.data(), buffer_pointer, length); + dst = dst.subspan(length); + } + } + + //! Move the read position ahead in the stream to the given position. + //! Use SetPos() to back up in the stream, not SkipTo(). + void SkipTo(const uint64_t file_pos) { + assert(file_pos >= nReadPos); + while (nReadPos < file_pos) { + AdvanceStream(file_pos - nReadPos); } } //! return the current reading position uint64_t GetPos() const { return nReadPos; } //! rewind to a given reading position bool SetPos(uint64_t nPos) { size_t bufsize = vchBuf.size(); if (nPos + bufsize < nSrcPos) { // rewinding too far, rewind as far as possible nReadPos = nSrcPos - bufsize; return false; } if (nPos > nSrcPos) { // can't go this far forward, go as far as possible nReadPos = nSrcPos; return false; } nReadPos = nPos; return true; } //! Prevent reading beyond a certain position. No argument removes the //! limit. bool SetLimit(uint64_t nPos = std::numeric_limits::max()) { if (nPos < nReadPos) { return false; } nReadLimit = nPos; return true; } template CBufferedFile &operator>>(T &&obj) { // Unserialize from this stream ::Unserialize(*this, obj); return (*this); } //! search for a given byte in the stream, and remain positioned on it void FindByte(uint8_t ch) { while (true) { if (nReadPos == nSrcPos) { Fill(); } if (vchBuf[nReadPos % vchBuf.size()] == std::byte{ch}) { break; } nReadPos++; } } }; #endif // BITCOIN_STREAMS_H diff --git a/src/test/streams_tests.cpp b/src/test/streams_tests.cpp index b2bcdae56..67dd9d328 100644 --- a/src/test/streams_tests.cpp +++ b/src/test/streams_tests.cpp @@ -1,474 +1,538 @@ // Copyright (c) 2012-2019 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include #include #include #include #include BOOST_FIXTURE_TEST_SUITE(streams_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(streams_vector_writer) { uint8_t a(1); uint8_t b(2); uint8_t bytes[] = {3, 4, 5, 6}; std::vector vch; // Each test runs twice. Serializing a second time at the same starting // point should yield the same results, even if the first test grew the // vector. CVectorWriter(SER_NETWORK, INIT_PROTO_VERSION, vch, 0, a, b); BOOST_CHECK((vch == std::vector{{1, 2}})); CVectorWriter(SER_NETWORK, INIT_PROTO_VERSION, vch, 0, a, b); BOOST_CHECK((vch == std::vector{{1, 2}})); vch.clear(); CVectorWriter(SER_NETWORK, INIT_PROTO_VERSION, vch, 2, a, b); BOOST_CHECK((vch == std::vector{{0, 0, 1, 2}})); CVectorWriter(SER_NETWORK, INIT_PROTO_VERSION, vch, 2, a, b); BOOST_CHECK((vch == std::vector{{0, 0, 1, 2}})); vch.clear(); vch.resize(5, 0); CVectorWriter(SER_NETWORK, INIT_PROTO_VERSION, vch, 2, a, b); BOOST_CHECK((vch == std::vector{{0, 0, 1, 2, 0}})); CVectorWriter(SER_NETWORK, INIT_PROTO_VERSION, vch, 2, a, b); BOOST_CHECK((vch == std::vector{{0, 0, 1, 2, 0}})); vch.clear(); vch.resize(4, 0); CVectorWriter(SER_NETWORK, INIT_PROTO_VERSION, vch, 3, a, b); BOOST_CHECK((vch == std::vector{{0, 0, 0, 1, 2}})); CVectorWriter(SER_NETWORK, INIT_PROTO_VERSION, vch, 3, a, b); BOOST_CHECK((vch == std::vector{{0, 0, 0, 1, 2}})); vch.clear(); vch.resize(4, 0); CVectorWriter(SER_NETWORK, INIT_PROTO_VERSION, vch, 4, a, b); BOOST_CHECK((vch == std::vector{{0, 0, 0, 0, 1, 2}})); CVectorWriter(SER_NETWORK, INIT_PROTO_VERSION, vch, 4, a, b); BOOST_CHECK((vch == std::vector{{0, 0, 0, 0, 1, 2}})); vch.clear(); CVectorWriter(SER_NETWORK, INIT_PROTO_VERSION, vch, 0, bytes); BOOST_CHECK((vch == std::vector{{3, 4, 5, 6}})); CVectorWriter(SER_NETWORK, INIT_PROTO_VERSION, vch, 0, bytes); BOOST_CHECK((vch == std::vector{{3, 4, 5, 6}})); vch.clear(); vch.resize(4, 8); CVectorWriter(SER_NETWORK, INIT_PROTO_VERSION, vch, 2, a, bytes, b); BOOST_CHECK((vch == std::vector{{8, 8, 1, 3, 4, 5, 6, 2}})); CVectorWriter(SER_NETWORK, INIT_PROTO_VERSION, vch, 2, a, bytes, b); BOOST_CHECK((vch == std::vector{{8, 8, 1, 3, 4, 5, 6, 2}})); vch.clear(); } BOOST_AUTO_TEST_CASE(streams_vector_reader) { std::vector vch = {1, 255, 3, 4, 5, 6}; SpanReader reader{SER_NETWORK, INIT_PROTO_VERSION, vch}; BOOST_CHECK_EQUAL(reader.size(), 6U); BOOST_CHECK(!reader.empty()); // Read a single byte as an uint8_t. uint8_t a; reader >> a; BOOST_CHECK_EQUAL(a, 1); BOOST_CHECK_EQUAL(reader.size(), 5U); BOOST_CHECK(!reader.empty()); // Read a single byte as a (signed) int8_t. int8_t b; reader >> b; BOOST_CHECK_EQUAL(b, -1); BOOST_CHECK_EQUAL(reader.size(), 4U); BOOST_CHECK(!reader.empty()); // Read a 4 bytes as an unsigned uint32_t. uint32_t c; reader >> c; // 100992003 = 3,4,5,6 in little-endian base-256 BOOST_CHECK_EQUAL(c, 100992003); BOOST_CHECK_EQUAL(reader.size(), 0U); BOOST_CHECK(reader.empty()); // Reading after end of byte vector throws an error. int32_t d; BOOST_CHECK_THROW(reader >> d, std::ios_base::failure); // Read a 4 bytes as a (signed) int32_t from the beginning of the buffer. SpanReader new_reader{SER_NETWORK, INIT_PROTO_VERSION, vch}; new_reader >> d; // 67370753 = 1,255,3,4 in little-endian base-256 BOOST_CHECK_EQUAL(d, 67370753); BOOST_CHECK_EQUAL(new_reader.size(), 2U); BOOST_CHECK(!new_reader.empty()); // Reading after end of byte vector throws an error even if the reader is // not totally empty. BOOST_CHECK_THROW(new_reader >> d, std::ios_base::failure); } BOOST_AUTO_TEST_CASE(bitstream_reader_writer) { CDataStream data(SER_NETWORK, INIT_PROTO_VERSION); BitStreamWriter bit_writer(data); bit_writer.Write(0, 1); bit_writer.Write(2, 2); bit_writer.Write(6, 3); bit_writer.Write(11, 4); bit_writer.Write(1, 5); bit_writer.Write(32, 6); bit_writer.Write(7, 7); bit_writer.Write(30497, 16); bit_writer.Flush(); CDataStream data_copy(data); uint32_t serialized_int1; data >> serialized_int1; // NOTE: Serialized as LE BOOST_CHECK_EQUAL(serialized_int1, (uint32_t)0x7700C35A); uint16_t serialized_int2; data >> serialized_int2; // NOTE: Serialized as LE BOOST_CHECK_EQUAL(serialized_int2, (uint16_t)0x1072); BitStreamReader bit_reader(data_copy); BOOST_CHECK_EQUAL(bit_reader.Read(1), 0U); BOOST_CHECK_EQUAL(bit_reader.Read(2), 2U); BOOST_CHECK_EQUAL(bit_reader.Read(3), 6U); BOOST_CHECK_EQUAL(bit_reader.Read(4), 11U); BOOST_CHECK_EQUAL(bit_reader.Read(5), 1U); BOOST_CHECK_EQUAL(bit_reader.Read(6), 32U); BOOST_CHECK_EQUAL(bit_reader.Read(7), 7U); BOOST_CHECK_EQUAL(bit_reader.Read(16), 30497U); BOOST_CHECK_THROW(bit_reader.Read(8), std::ios_base::failure); } BOOST_AUTO_TEST_CASE(streams_serializedata_xor) { std::vector in; std::vector expected_xor; std::vector key; CDataStream ds(in, 0, 0); // Degenerate case key.push_back('\x00'); key.push_back('\x00'); ds.Xor(key); BOOST_CHECK_EQUAL(std::string(expected_xor.begin(), expected_xor.end()), ds.str()); in.push_back(std::byte{0x0f}); in.push_back(std::byte{0xf0}); expected_xor.push_back('\xf0'); expected_xor.push_back('\x0f'); // Single character key ds.clear(); ds.insert(ds.begin(), in.begin(), in.end()); key.clear(); key.push_back('\xff'); ds.Xor(key); BOOST_CHECK_EQUAL(std::string(expected_xor.begin(), expected_xor.end()), ds.str()); // Multi character key in.clear(); expected_xor.clear(); in.push_back(std::byte{0xf0}); in.push_back(std::byte{0x0f}); expected_xor.push_back('\x0f'); expected_xor.push_back('\x00'); ds.clear(); ds.insert(ds.begin(), in.begin(), in.end()); key.clear(); key.push_back('\xff'); key.push_back('\x0f'); ds.Xor(key); BOOST_CHECK_EQUAL(std::string(expected_xor.begin(), expected_xor.end()), ds.str()); } BOOST_AUTO_TEST_CASE(streams_empty_vector) { std::vector in; CDataStream ds(in, 0, 0); // read 0 bytes used to cause a segfault on some older systems. BOOST_CHECK_NO_THROW(ds.read({})); // Same goes for writing 0 bytes from a vector ... const std::vector vdata{std::byte{'f'}, std::byte{'o'}, std::byte{'o'}, std::byte{'b'}, std::byte{'a'}, std::byte{'r'}}; BOOST_CHECK_NO_THROW(ds.insert(ds.begin(), vdata.begin(), vdata.begin())); BOOST_CHECK_NO_THROW(ds.insert(ds.begin(), vdata.begin(), vdata.end())); // ... or an array. const std::byte adata[6] = {std::byte{'f'}, std::byte{'o'}, std::byte{'o'}, std::byte{'b'}, std::byte{'a'}, std::byte{'r'}}; BOOST_CHECK_NO_THROW(ds.insert(ds.begin(), &adata[0], &adata[0])); BOOST_CHECK_NO_THROW(ds.insert(ds.begin(), &adata[0], &adata[6])); } BOOST_AUTO_TEST_CASE(streams_buffered_file) { FILE *file = fsbridge::fopen("streams_test_tmp", "w+b"); // The value at each offset is the offset. for (uint8_t j = 0; j < 40; ++j) { fwrite(&j, 1, 1, file); } rewind(file); // The buffer size (second arg) must be greater than the rewind // amount (third arg). try { CBufferedFile bfbad(file, 25, 25, 222, 333); BOOST_CHECK(false); } catch (const std::exception &e) { BOOST_CHECK( strstr(e.what(), "Rewind limit must be less than buffer size") != nullptr); } // The buffer is 25 bytes, allow rewinding 10 bytes. CBufferedFile bf(file, 25, 10, 222, 333); BOOST_CHECK(!bf.eof()); // These two members have no functional effect. BOOST_CHECK_EQUAL(bf.GetType(), 222); BOOST_CHECK_EQUAL(bf.GetVersion(), 333); uint8_t i; bf >> i; BOOST_CHECK_EQUAL(i, 0); bf >> i; BOOST_CHECK_EQUAL(i, 1); // After reading bytes 0 and 1, we're positioned at 2. BOOST_CHECK_EQUAL(bf.GetPos(), 2U); // Rewind to offset 0, ok (within the 10 byte window). BOOST_CHECK(bf.SetPos(0)); bf >> i; BOOST_CHECK_EQUAL(i, 0); // We can go forward to where we've been, but beyond may fail. BOOST_CHECK(bf.SetPos(2)); bf >> i; BOOST_CHECK_EQUAL(i, 2); // If you know the maximum number of bytes that should be // read to deserialize the variable, you can limit the read // extent. The current file offset is 3, so the following // SetLimit() allows zero bytes to be read. BOOST_CHECK(bf.SetLimit(3)); try { bf >> i; BOOST_CHECK(false); } catch (const std::exception &e) { - BOOST_CHECK(strstr(e.what(), "Read attempted past buffer limit") != + BOOST_CHECK(strstr(e.what(), "Attempt to position past buffer limit") != nullptr); } // The default argument removes the limit completely. BOOST_CHECK(bf.SetLimit()); // The read position should still be at 3 (no change). BOOST_CHECK_EQUAL(bf.GetPos(), 3U); // Read from current offset, 3, forward until position 10. for (uint8_t j = 3; j < 10; ++j) { bf >> i; BOOST_CHECK_EQUAL(i, j); } BOOST_CHECK_EQUAL(bf.GetPos(), 10U); // We're guaranteed (just barely) to be able to rewind to zero. BOOST_CHECK(bf.SetPos(0)); BOOST_CHECK_EQUAL(bf.GetPos(), 0U); bf >> i; BOOST_CHECK_EQUAL(i, 0); // We can set the position forward again up to the farthest // into the stream we've been, but no farther. (Attempting // to go farther may succeed, but it's not guaranteed.) BOOST_CHECK(bf.SetPos(10)); bf >> i; BOOST_CHECK_EQUAL(i, 10); BOOST_CHECK_EQUAL(bf.GetPos(), 11U); // Now it's only guaranteed that we can rewind to offset 1 // (current read position, 11, minus rewind amount, 10). BOOST_CHECK(bf.SetPos(1)); BOOST_CHECK_EQUAL(bf.GetPos(), 1U); bf >> i; BOOST_CHECK_EQUAL(i, 1); // We can stream into large variables, even larger than // the buffer size. BOOST_CHECK(bf.SetPos(11)); { uint8_t a[40 - 11]; bf >> a; for (uint8_t j = 0; j < sizeof(a); ++j) { BOOST_CHECK_EQUAL(a[j], 11 + j); } } BOOST_CHECK_EQUAL(bf.GetPos(), 40U); // We've read the entire file, the next read should throw. try { bf >> i; BOOST_CHECK(false); } catch (const std::exception &e) { BOOST_CHECK(strstr(e.what(), "CBufferedFile::Fill: end of file") != nullptr); } // Attempting to read beyond the end sets the EOF indicator. BOOST_CHECK(bf.eof()); // Still at offset 40, we can go back 10, to 30. BOOST_CHECK_EQUAL(bf.GetPos(), 40U); BOOST_CHECK(bf.SetPos(30)); bf >> i; BOOST_CHECK_EQUAL(i, 30); BOOST_CHECK_EQUAL(bf.GetPos(), 31U); // We're too far to rewind to position zero. BOOST_CHECK(!bf.SetPos(0)); // But we should now be positioned at least as far back as allowed // by the rewind window (relative to our farthest read position, 40). - BOOST_CHECK(bf.GetPos() <= 30); + BOOST_CHECK(bf.GetPos() <= 30U); // We can explicitly close the file, or the destructor will do it. bf.fclose(); fs::remove("streams_test_tmp"); } +BOOST_AUTO_TEST_CASE(streams_buffered_file_skip) { + fs::path streams_test_filename = + m_args.GetDataDirBase() / "streams_test_tmp"; + FILE *file = fsbridge::fopen(streams_test_filename, "w+b"); + // The value at each offset is the byte offset (e.g. byte 1 in the file has + // the value 0x01). + for (uint8_t j = 0; j < 40; ++j) { + fwrite(&j, 1, 1, file); + } + rewind(file); + + // The buffer is 25 bytes, allow rewinding 10 bytes. + CBufferedFile bf(file, 25, 10, 222, 333); + + uint8_t i; + // This is like bf >> (7-byte-variable), in that it will cause data + // to be read from the file into memory, but it's not copied to us. + bf.SkipTo(7); + BOOST_CHECK_EQUAL(bf.GetPos(), 7U); + bf >> i; + BOOST_CHECK_EQUAL(i, 7); + + // The bytes in the buffer up to offset 7 are valid and can be read. + BOOST_CHECK(bf.SetPos(0)); + bf >> i; + BOOST_CHECK_EQUAL(i, 0); + bf >> i; + BOOST_CHECK_EQUAL(i, 1); + + bf.SkipTo(11); + bf >> i; + BOOST_CHECK_EQUAL(i, 11); + + // SkipTo() honors the transfer limit; we can't position beyond the limit. + bf.SetLimit(13); + try { + bf.SkipTo(14); + BOOST_CHECK(false); + } catch (const std::exception &e) { + BOOST_CHECK(strstr(e.what(), "Attempt to position past buffer limit") != + nullptr); + } + + // We can position exactly to the transfer limit. + bf.SkipTo(13); + BOOST_CHECK_EQUAL(bf.GetPos(), 13U); + + bf.fclose(); + fs::remove(streams_test_filename); +} + BOOST_AUTO_TEST_CASE(streams_buffered_file_rand) { // Make this test deterministic. SeedInsecureRand(SeedRand::ZEROS); for (int rep = 0; rep < 50; ++rep) { FILE *file = fsbridge::fopen("streams_test_tmp", "w+b"); size_t fileSize = InsecureRandRange(256); for (uint8_t i = 0; i < fileSize; ++i) { fwrite(&i, 1, 1, file); } rewind(file); size_t bufSize = InsecureRandRange(300) + 1; size_t rewindSize = InsecureRandRange(bufSize); CBufferedFile bf(file, bufSize, rewindSize, 222, 333); size_t currentPos = 0; size_t maxPos = 0; for (int step = 0; step < 100; ++step) { if (currentPos >= fileSize) { break; } // We haven't read to the end of the file yet. BOOST_CHECK(!bf.eof()); BOOST_CHECK_EQUAL(bf.GetPos(), currentPos); // Pretend the file consists of a series of objects of varying // sizes; the boundaries of the objects can interact arbitrarily // with the CBufferFile's internal buffer. These first three // cases simulate objects of various sizes (1, 2, 5 bytes). - switch (InsecureRandRange(5)) { + switch (InsecureRandRange(6)) { case 0: { uint8_t a[1]; if (currentPos + 1 > fileSize) { continue; } bf.SetLimit(currentPos + 1); bf >> a; for (uint8_t i = 0; i < 1; ++i) { BOOST_CHECK_EQUAL(a[i], currentPos); currentPos++; } break; } case 1: { uint8_t a[2]; if (currentPos + 2 > fileSize) { continue; } bf.SetLimit(currentPos + 2); bf >> a; for (uint8_t i = 0; i < 2; ++i) { BOOST_CHECK_EQUAL(a[i], currentPos); currentPos++; } break; } case 2: { uint8_t a[5]; if (currentPos + 5 > fileSize) { continue; } bf.SetLimit(currentPos + 5); bf >> a; for (uint8_t i = 0; i < 5; ++i) { BOOST_CHECK_EQUAL(a[i], currentPos); currentPos++; } break; } case 3: { + // SkipTo is similar to the "read" cases above, except + // we don't receive the data. + size_t skip_length{ + static_cast(InsecureRandRange(5))}; + if (currentPos + skip_length > fileSize) { + continue; + } + bf.SetLimit(currentPos + skip_length); + bf.SkipTo(currentPos + skip_length); + currentPos += skip_length; + break; + } + case 4: { // Find a byte value (that is at or ahead of the current // position). size_t find = currentPos + InsecureRandRange(8); if (find >= fileSize) { find = fileSize - 1; } bf.FindByte(static_cast(find)); // The value at each offset is the offset. BOOST_CHECK_EQUAL(bf.GetPos(), find); currentPos = find; bf.SetLimit(currentPos + 1); uint8_t i; bf >> i; BOOST_CHECK_EQUAL(i, currentPos); currentPos++; break; } - case 4: { + case 5: { size_t requestPos = InsecureRandRange(maxPos + 4); bool okay = bf.SetPos(requestPos); // The new position may differ from the requested position // because we may not be able to rewind beyond the rewind // window, and we may not be able to move forward beyond the // farthest position we've reached so far. currentPos = bf.GetPos(); BOOST_CHECK_EQUAL(okay, currentPos == requestPos); // Check that we can position within the rewind window. if (requestPos <= maxPos && maxPos > rewindSize && requestPos >= maxPos - rewindSize) { // We requested a position within the rewind window. BOOST_CHECK(okay); } break; } } if (maxPos < currentPos) { maxPos = currentPos; } } } fs::remove("streams_test_tmp"); } BOOST_AUTO_TEST_SUITE_END()