diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -331,6 +331,7 @@ config.cpp consensus/activation.cpp consensus/tx_verify.cpp + flatfile.cpp globals.cpp httprpc.cpp httpserver.cpp diff --git a/src/Makefile.am b/src/Makefile.am --- a/src/Makefile.am +++ b/src/Makefile.am @@ -136,6 +136,7 @@ core_memusage.h \ cuckoocache.h \ diskblockpos.h \ + flatfile.h \ fs.h \ globals.h \ httprpc.h \ @@ -251,6 +252,7 @@ config.cpp \ consensus/activation.cpp \ consensus/tx_verify.cpp \ + flatfile.cpp \ globals.cpp \ httprpc.cpp \ httpserver.cpp \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -67,6 +67,7 @@ test/excessiveblock_tests.cpp \ test/feerate_tests.cpp \ test/finalization_tests.cpp \ + test/flatfile_tests.cpp \ test/getarg_tests.cpp \ test/hash_tests.cpp \ test/inv_tests.cpp \ diff --git a/src/flatfile.h b/src/flatfile.h new file mode 100644 --- /dev/null +++ b/src/flatfile.h @@ -0,0 +1,63 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_FLATFILE_H +#define BITCOIN_FLATFILE_H + +#include +#include + +/** + * FlatFileSeq represents a sequence of numbered files storing raw data. This + * class facilitates access to and efficient management of these files. + */ +class FlatFileSeq { +private: + const fs::path m_dir; + const char *const m_prefix; + const size_t m_chunk_size; + +public: + /** + * Constructor + * + * @param dir The base directory that all files live in. + * @param prefix A short prefix given to all file names. + * @param chunk_size Disk space is pre-allocated in multiples of this + * amount. + */ + FlatFileSeq(fs::path dir, const char *prefix, size_t chunk_size); + + /** Get the name of the file at the given position. */ + fs::path FileName(const CDiskBlockPos &pos) const; + + /** Open a handle to the file at the given position. */ + FILE *Open(const CDiskBlockPos &pos, bool fReadOnly = false); + + /** + * Allocate additional space in a file after the given starting position. + * The amount allocated will be the minimum multiple of the sequence chunk + * size greater than add_size. + * + * @param[in] pos The starting position that bytes will be allocated after. + * @param[in] add_size The minimum number of bytes to be allocated. + * @param[out] out_of_space Whether the allocation failed due to + * insufficient disk space. + * @return The number of bytes successfully allocated. + */ + size_t Allocate(const CDiskBlockPos &pos, size_t add_size, + bool &out_of_space); + + /** + * Commit a file to disk, and optionally truncate off extra pre-allocated + * bytes if final. + * + * @param[in] pos The first unwritten position in the file to be flushed. + * @param[in] finalize True if no more data will be written to this file. + * @return true on success, false on failure. + */ + bool Flush(const CDiskBlockPos &pos, bool finalize = false); +}; + +#endif // BITCOIN_FLATFILE_H diff --git a/src/flatfile.cpp b/src/flatfile.cpp new file mode 100644 --- /dev/null +++ b/src/flatfile.cpp @@ -0,0 +1,93 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include +#include + +#include + +FlatFileSeq::FlatFileSeq(fs::path dir, const char *prefix, size_t chunk_size) + : m_dir(std::move(dir)), m_prefix(prefix), m_chunk_size(chunk_size) { + if (chunk_size == 0) { + throw std::invalid_argument("chunk_size must be positive"); + } +} + +fs::path FlatFileSeq::FileName(const CDiskBlockPos &pos) const { + return m_dir / strprintf("%s%05u.dat", m_prefix, pos.nFile); +} + +FILE *FlatFileSeq::Open(const CDiskBlockPos &pos, bool fReadOnly) { + if (pos.IsNull()) { + return nullptr; + } + fs::path path = FileName(pos); + fs::create_directories(path.parent_path()); + FILE *file = fsbridge::fopen(path, fReadOnly ? "rb" : "rb+"); + if (!file && !fReadOnly) { + file = fsbridge::fopen(path, "wb+"); + } + if (!file) { + LogPrintf("Unable to open file %s\n", path.string()); + return nullptr; + } + if (pos.nPos) { + if (fseek(file, pos.nPos, SEEK_SET)) { + LogPrintf("Unable to seek to position %u of %s\n", pos.nPos, + path.string()); + fclose(file); + return nullptr; + } + } + return file; +} + +size_t FlatFileSeq::Allocate(const CDiskBlockPos &pos, size_t add_size, + bool &out_of_space) { + out_of_space = false; + + unsigned int n_old_chunks = (pos.nPos + m_chunk_size - 1) / m_chunk_size; + unsigned int n_new_chunks = + (pos.nPos + add_size + m_chunk_size - 1) / m_chunk_size; + if (n_new_chunks > n_old_chunks) { + size_t old_size = pos.nPos; + size_t new_size = n_new_chunks * m_chunk_size; + size_t inc_size = new_size - old_size; + + if (CheckDiskSpace(m_dir, inc_size)) { + FILE *file = Open(pos); + if (file) { + LogPrintf("Pre-allocating up to position 0x%x in %s%05u.dat\n", + new_size, m_prefix, pos.nFile); + AllocateFileRange(file, pos.nPos, inc_size); + fclose(file); + return inc_size; + } + } else { + out_of_space = true; + } + } + return 0; +} + +bool FlatFileSeq::Flush(const CDiskBlockPos &pos, bool finalize) { + // Avoid fseek to nPos + FILE *file = Open(CDiskBlockPos(pos.nFile, 0)); + if (!file) { + return error("%s: failed to open file %d", __func__, pos.nFile); + } + if (finalize && !TruncateFile(file, pos.nPos)) { + fclose(file); + return error("%s: failed to truncate file %d", __func__, pos.nFile); + } + if (!FileCommit(file)) { + fclose(file); + return error("%s: failed to commit file %d", __func__, pos.nFile); + } + + fclose(file); + return true; +} diff --git a/src/init.cpp b/src/init.cpp --- a/src/init.cpp +++ b/src/init.cpp @@ -1101,7 +1101,7 @@ int nFile = 0; while (true) { CDiskBlockPos pos(nFile, 0); - if (!fs::exists(GetBlockPosFilename(pos, "blk"))) { + if (!fs::exists(GetBlockPosFilename(pos))) { // No block files left to reindex break; } diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -80,6 +80,7 @@ excessiveblock_tests.cpp feerate_tests.cpp finalization_tests.cpp + flatfile_tests.cpp getarg_tests.cpp hash_tests.cpp inv_tests.cpp diff --git a/src/test/flatfile_tests.cpp b/src/test/flatfile_tests.cpp new file mode 100644 --- /dev/null +++ b/src/test/flatfile_tests.cpp @@ -0,0 +1,133 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include + +#include +#include + +#include + +#include + +BOOST_FIXTURE_TEST_SUITE(flatfile_tests, BasicTestingSetup) + +BOOST_AUTO_TEST_CASE(flatfile_filename) { + auto data_dir = SetDataDir("flatfile_test"); + + CDiskBlockPos pos(456, 789); + + FlatFileSeq seq1(data_dir, "a", 16 * 1024); + BOOST_CHECK_EQUAL(seq1.FileName(pos), data_dir / "a00456.dat"); + + FlatFileSeq seq2(data_dir / "a", "b", 16 * 1024); + BOOST_CHECK_EQUAL(seq2.FileName(pos), data_dir / "a" / "b00456.dat"); +} + +BOOST_AUTO_TEST_CASE(flatfile_open) { + auto data_dir = SetDataDir("flatfile_test"); + FlatFileSeq seq(data_dir, "a", 16 * 1024); + + std::string line1( + "A purely peer-to-peer version of electronic cash would allow online " + "payments to be sent directly from one party to another without going " + "through a financial institution."); + std::string line2("Digital signatures provide part of the solution, but " + "the main benefits are lost if a trusted third party is " + "still required to prevent double-spending."); + + size_t pos1 = 0; + size_t pos2 = pos1 + GetSerializeSize(line1, 0, CLIENT_VERSION); + + // Write first line to file. + { + CAutoFile file(seq.Open(CDiskBlockPos(0, pos1)), SER_DISK, + CLIENT_VERSION); + file << LIMITED_STRING(line1, 256); + } + + // Attempt to append to file opened in read-only mode. + { + CAutoFile file(seq.Open(CDiskBlockPos(0, pos2), true), SER_DISK, + CLIENT_VERSION); + BOOST_CHECK_THROW(file << LIMITED_STRING(line2, 256), + std::ios_base::failure); + } + + // Append second line to file. + { + CAutoFile file(seq.Open(CDiskBlockPos(0, pos2)), SER_DISK, + CLIENT_VERSION); + file << LIMITED_STRING(line2, 256); + } + + // Read text from file in read-only mode. + { + std::string text; + CAutoFile file(seq.Open(CDiskBlockPos(0, pos1), true), SER_DISK, + CLIENT_VERSION); + + file >> LIMITED_STRING(text, 256); + BOOST_CHECK_EQUAL(text, line1); + + file >> LIMITED_STRING(text, 256); + BOOST_CHECK_EQUAL(text, line2); + } + + // Read text from file with position offset. + { + std::string text; + CAutoFile file(seq.Open(CDiskBlockPos(0, pos2)), SER_DISK, + CLIENT_VERSION); + + file >> LIMITED_STRING(text, 256); + BOOST_CHECK_EQUAL(text, line2); + } + + // Ensure another file in the sequence has no data. + { + std::string text; + CAutoFile file(seq.Open(CDiskBlockPos(1, pos2)), SER_DISK, + CLIENT_VERSION); + BOOST_CHECK_THROW(file >> LIMITED_STRING(text, 256), + std::ios_base::failure); + } +} + +BOOST_AUTO_TEST_CASE(flatfile_allocate) { + auto data_dir = SetDataDir("flatfile_test"); + FlatFileSeq seq(data_dir, "a", 100); + + bool out_of_space; + + BOOST_CHECK_EQUAL(seq.Allocate(CDiskBlockPos(0, 0), 1, out_of_space), 100); + BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(CDiskBlockPos(0, 0))), 100); + BOOST_CHECK(!out_of_space); + + BOOST_CHECK_EQUAL(seq.Allocate(CDiskBlockPos(0, 99), 1, out_of_space), 0); + BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(CDiskBlockPos(0, 99))), 100); + BOOST_CHECK(!out_of_space); + + BOOST_CHECK_EQUAL(seq.Allocate(CDiskBlockPos(0, 99), 2, out_of_space), 101); + BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(CDiskBlockPos(0, 99))), 200); + BOOST_CHECK(!out_of_space); +} + +BOOST_AUTO_TEST_CASE(flatfile_flush) { + auto data_dir = SetDataDir("flatfile_test"); + FlatFileSeq seq(data_dir, "a", 100); + + bool out_of_space; + seq.Allocate(CDiskBlockPos(0, 0), 1, out_of_space); + + // Flush without finalize should not truncate file. + seq.Flush(CDiskBlockPos(0, 1)); + BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(CDiskBlockPos(0, 1))), 100); + + // Flush with finalize should truncate file. + seq.Flush(CDiskBlockPos(0, 1), true); + BOOST_CHECK_EQUAL(fs::file_size(seq.FileName(CDiskBlockPos(0, 1))), 1); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/validation.h b/src/validation.h --- a/src/validation.h +++ b/src/validation.h @@ -355,7 +355,7 @@ /** * Translation to a filesystem path. */ -fs::path GetBlockPosFilename(const CDiskBlockPos &pos, const char *prefix); +fs::path GetBlockPosFilename(const CDiskBlockPos &pos); /** * Import blocks from an external file. diff --git a/src/validation.cpp b/src/validation.cpp --- a/src/validation.cpp +++ b/src/validation.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -311,6 +312,8 @@ static void FindFilesToPrune(std::set &setFilesToPrune, uint64_t nPruneAfterHeight); static FILE *OpenUndoFile(const CDiskBlockPos &pos, bool fReadOnly = false); +static FlatFileSeq BlockFileSeq(); +static FlatFileSeq UndoFileSeq(); static uint32_t GetNextBlockScriptFlags(const Config &config, const CBlockIndex *pindex); @@ -1486,29 +1489,14 @@ static void FlushBlockFile(bool fFinalize = false) { LOCK(cs_LastBlockFile); - CDiskBlockPos posOld(nLastBlockFile, 0); - bool status = true; - - FILE *fileOld = OpenBlockFile(posOld); - if (fileOld) { - if (fFinalize) { - status &= - TruncateFile(fileOld, vinfoBlockFile[nLastBlockFile].nSize); - } - status &= FileCommit(fileOld); - fclose(fileOld); - } - - fileOld = OpenUndoFile(posOld); - if (fileOld) { - if (fFinalize) { - status &= - TruncateFile(fileOld, vinfoBlockFile[nLastBlockFile].nUndoSize); - } - status &= FileCommit(fileOld); - fclose(fileOld); - } + CDiskBlockPos block_pos_old(nLastBlockFile, + vinfoBlockFile[nLastBlockFile].nSize); + CDiskBlockPos undo_pos_old(nLastBlockFile, + vinfoBlockFile[nLastBlockFile].nUndoSize); + bool status = true; + status &= BlockFileSeq().Flush(block_pos_old, fFinalize); + status &= UndoFileSeq().Flush(undo_pos_old, fFinalize); if (!status) { AbortNode("Flushing block file to disk failed. This is likely the " "result of an I/O error."); @@ -3347,32 +3335,15 @@ } if (!fKnown) { - unsigned int nOldChunks = - (pos.nPos + BLOCKFILE_CHUNK_SIZE - 1) / BLOCKFILE_CHUNK_SIZE; - unsigned int nNewChunks = - (vinfoBlockFile[nFile].nSize + BLOCKFILE_CHUNK_SIZE - 1) / - BLOCKFILE_CHUNK_SIZE; - if (nNewChunks > nOldChunks) { - if (fPruneMode) { - fCheckForPruning = true; - } - - if (CheckDiskSpace(GetBlocksDir(), - nNewChunks * BLOCKFILE_CHUNK_SIZE - pos.nPos)) { - FILE *file = OpenBlockFile(pos); - if (file) { - LogPrintf( - "Pre-allocating up to position 0x%x in blk%05u.dat\n", - nNewChunks * BLOCKFILE_CHUNK_SIZE, pos.nFile); - AllocateFileRange(file, pos.nPos, - nNewChunks * BLOCKFILE_CHUNK_SIZE - - pos.nPos); - fclose(file); - } - } else { - return AbortNode("Disk space is low!", - _("Error: Disk space is low!")); - } + bool out_of_space; + size_t bytes_allocated = + BlockFileSeq().Allocate(pos, nAddSize, out_of_space); + if (out_of_space) { + return AbortNode("Disk space is low!", + _("Error: Disk space is low!")); + } + if (bytes_allocated != 0 && fPruneMode) { + fCheckForPruning = true; } } @@ -3386,34 +3357,19 @@ LOCK(cs_LastBlockFile); - unsigned int nNewSize; pos.nPos = vinfoBlockFile[nFile].nUndoSize; - nNewSize = vinfoBlockFile[nFile].nUndoSize += nAddSize; + vinfoBlockFile[nFile].nUndoSize += nAddSize; setDirtyFileInfo.insert(nFile); - unsigned int nOldChunks = - (pos.nPos + UNDOFILE_CHUNK_SIZE - 1) / UNDOFILE_CHUNK_SIZE; - unsigned int nNewChunks = - (nNewSize + UNDOFILE_CHUNK_SIZE - 1) / UNDOFILE_CHUNK_SIZE; - if (nNewChunks > nOldChunks) { - if (fPruneMode) { - fCheckForPruning = true; - } - - if (CheckDiskSpace(GetBlocksDir(), - nNewChunks * UNDOFILE_CHUNK_SIZE - pos.nPos)) { - FILE *file = OpenUndoFile(pos); - if (file) { - LogPrintf("Pre-allocating up to position 0x%x in rev%05u.dat\n", - nNewChunks * UNDOFILE_CHUNK_SIZE, pos.nFile); - AllocateFileRange(file, pos.nPos, - nNewChunks * UNDOFILE_CHUNK_SIZE - pos.nPos); - fclose(file); - } - } else { - return AbortNode(state, "Disk space is low!", - _("Error: Disk space is low!")); - } + bool out_of_space; + size_t bytes_allocated = + UndoFileSeq().Allocate(pos, nAddSize, out_of_space); + if (out_of_space) { + return AbortNode(state, "Disk space is low!", + _("Error: Disk space is low!")); + } + if (bytes_allocated != 0 && fPruneMode) { + fCheckForPruning = true; } return true; @@ -4204,8 +4160,8 @@ void UnlinkPrunedFiles(const std::set &setFilesToPrune) { for (const int i : setFilesToPrune) { CDiskBlockPos pos(i, 0); - fs::remove(GetBlockPosFilename(pos, "blk")); - fs::remove(GetBlockPosFilename(pos, "rev")); + fs::remove(BlockFileSeq().FileName(pos)); + fs::remove(UndoFileSeq().FileName(pos)); LogPrintf("Prune: %s deleted blk/rev (%05u)\n", __func__, i); } } @@ -4327,47 +4283,25 @@ nLastBlockWeCanPrune, count); } -static FILE *OpenDiskFile(const CDiskBlockPos &pos, const char *prefix, - bool fReadOnly) { - if (pos.IsNull()) { - return nullptr; - } - - fs::path path = GetBlockPosFilename(pos, prefix); - fs::create_directories(path.parent_path()); - FILE *file = fsbridge::fopen(path, fReadOnly ? "rb" : "rb+"); - if (!file && !fReadOnly) { - file = fsbridge::fopen(path, "wb+"); - } - - if (!file) { - LogPrintf("Unable to open file %s\n", path.string()); - return nullptr; - } - - if (pos.nPos) { - if (fseek(file, pos.nPos, SEEK_SET)) { - LogPrintf("Unable to seek to position %u of %s\n", pos.nPos, - path.string()); - fclose(file); - return nullptr; - } - } +static FlatFileSeq BlockFileSeq() { + return FlatFileSeq(GetBlocksDir(), "blk", BLOCKFILE_CHUNK_SIZE); +} - return file; +static FlatFileSeq UndoFileSeq() { + return FlatFileSeq(GetBlocksDir(), "rev", UNDOFILE_CHUNK_SIZE); } FILE *OpenBlockFile(const CDiskBlockPos &pos, bool fReadOnly) { - return OpenDiskFile(pos, "blk", fReadOnly); + return BlockFileSeq().Open(pos, fReadOnly); } /** Open an undo file (rev?????.dat) */ static FILE *OpenUndoFile(const CDiskBlockPos &pos, bool fReadOnly) { - return OpenDiskFile(pos, "rev", fReadOnly); + return UndoFileSeq().Open(pos, fReadOnly); } -fs::path GetBlockPosFilename(const CDiskBlockPos &pos, const char *prefix) { - return GetBlocksDir() / strprintf("%s%05u.dat", prefix, pos.nFile); +fs::path GetBlockPosFilename(const CDiskBlockPos &pos) { + return BlockFileSeq().FileName(pos); } CBlockIndex *CChainState::InsertBlockIndex(const uint256 &hash) {