diff --git a/src/radix.h b/src/radix.h index 7d5c2f3a2..5325473cb 100644 --- a/src/radix.h +++ b/src/radix.h @@ -1,254 +1,260 @@ // Copyright (c) 2019 The Bitcoin developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_RADIX_H #define BITCOIN_RADIX_H #include #include #include #include #include #include #include +#include /** * This is a radix tree storing values identified by a unique key. * * The tree is composed of nodes (RadixNode) containing an array of * RadixElement. The key is split into chunks of a few bits that serve as an * index into that array. RadixElement is a discriminated union of either a * RadixNode* representing the next level in the tree, or a T* representing a * leaf. New RadixNode are added lazily when two leaves would go in the same * slot. * * Reads walk the tree using sequential atomic loads, and insertions are done * using CAS, which ensures both can be executed lock free. Removing any * elements from the tree can also be done using CAS, but requires waiting for * other readers before being destroyed. The tree uses RCU to track which thread * is reading the tree, which allows deletion to wait for other readers to be up * to speed before destroying anything. It is therefore crucial that the lock be * taken before reading anything in the tree. * * It is not possible to delete anything from the tree at this time. The tree * itself cannot be destroyed and will leak memory instead of cleaning up after * itself. This obviously needs to be fixed in subsequent revisions. */ template struct RadixTree : public boost::noncopyable { private: static const int BITS = 4; static const int MASK = (1 << BITS) - 1; static const size_t CHILD_PER_LEVEL = 1 << BITS; - typedef decltype(std::declval().getId()) K; + typedef typename std::remove_reference().getId())>::type K; static const size_t KEY_BITS = 8 * sizeof(K); static const uint32_t TOP_LEVEL = (KEY_BITS - 1) / BITS; struct RadixElement; struct RadixNode; std::atomic root; public: RadixTree() : root(RadixElement()) {} /** * Insert a value into the tree. * Returns true if the value was inserted, false if it was already present. */ - bool insert(T *value) { return insert(value->getId(), value); } + bool insert(const RCUPtr &value) { + return insert(value->getId(), value); + } /** * Get the value corresponding to a key. - * Returns the value if found, null if not. + * Returns the value if found, nullptr if not. */ - T *get(const K &key) { + RCUPtr get(const K &key) { uint32_t level = TOP_LEVEL; RCULock lock; RadixElement e = root.load(); // Find a leaf. while (e.isNode()) { e = e.getNode()->get(level--, key)->load(); } T *leaf = e.getLeaf(); if (leaf == nullptr || leaf->getId() != key) { // We failed to find the proper element. - return nullptr; + return RCUPtr(); } // The leaf is non-null and the keys match. We have our guy. - return leaf; + return RCUPtr::copy(leaf); } - const T *get(const K &key) const { - return const_cast(this)->get(key); + RCUPtr get(const K &key) const { + T const *ptr = const_cast(this)->get(key).release(); + return RCUPtr::acquire(ptr); } /** * Remove an element from the tree. - * Returns true if the element was removed from the tree, false if the - * element wasn't found in the tree. + * Returns the removed element, or nullptr if there isn't one. */ - bool remove(const K &key) { + RCUPtr remove(const K &key) { uint32_t level = TOP_LEVEL; RCULock lock; std::atomic *eptr = &root; RadixElement e = eptr->load(); // Walk down the tree until we find a leaf for our node. while (e.isNode()) { Node: eptr = e.getNode()->get(level--, key); e = eptr->load(); } T *leaf = e.getLeaf(); if (leaf == nullptr || leaf->getId() != key) { // We failed to find the proper element. - return false; + return RCUPtr(); } // We have the proper element, try to delete it. if (eptr->compare_exchange_strong(e, RadixElement())) { - return true; + return RCUPtr::acquire(leaf); } // The element was replaced, either by a subtree or another element. if (e.isNode()) { goto Node; } // The element in the slot is not the one we are looking for. - return false; + return RCUPtr(); } private: - bool insert(const K &key, T *value) { + bool insert(const K &key, RCUPtr value) { uint32_t level = TOP_LEVEL; RCULock lock; std::atomic *eptr = &root; while (true) { RadixElement e = eptr->load(); // Walk down the tree until we find a leaf for our node. while (e.isNode()) { Node: eptr = e.getNode()->get(level--, key); e = eptr->load(); } // If the slot is empty, try to insert right there. if (e.getLeaf() == nullptr) { - if (eptr->compare_exchange_strong(e, RadixElement(value))) { + if (eptr->compare_exchange_strong(e, + RadixElement(value.get()))) { + value.release(); return true; } // CAS failed, we may have a node in there now. if (e.isNode()) { goto Node; } } // The element was already in the tree. const K &leafKey = e.getLeaf()->getId(); if (key == leafKey) { return false; } // There is an element there, but it isn't a subtree. We need to // convert it into a subtree and resume insertion into that subtree. std::unique_ptr newChild = MakeUnique(level, leafKey, e); if (eptr->compare_exchange_strong(e, RadixElement(newChild.get()))) { // We have a subtree, resume normal operations from there. newChild.release(); } } } struct RadixElement { private: union { RadixNode *node; T *leaf; uintptr_t raw; }; static const uintptr_t DISCRIMINANT = 0x01; bool getDiscriminant() const { return (raw & DISCRIMINANT) != 0; } public: explicit RadixElement() noexcept : raw(DISCRIMINANT) {} explicit RadixElement(RadixNode *nodeIn) noexcept : node(nodeIn) {} explicit RadixElement(T *leafIn) noexcept : leaf(leafIn) { raw |= DISCRIMINANT; } /** * Node features. */ bool isNode() const { return !getDiscriminant(); } RadixNode *getNode() { assert(isNode()); return node; } const RadixNode *getNode() const { assert(isNode()); return node; } /** * Leaf features. */ bool isLeaf() const { return getDiscriminant(); } T *getLeaf() { assert(isLeaf()); return reinterpret_cast(raw & ~DISCRIMINANT); } const T *getLeaf() const { assert(isLeaf()); return const_cast(this)->getLeaf(); } }; struct RadixNode { private: union { std::array, CHILD_PER_LEVEL> childs; std::array non_atomic_childs_DO_NOT_USE; }; public: RadixNode(uint32_t level, const K &key, RadixElement e) : non_atomic_childs_DO_NOT_USE() { get(level, key)->store(e); } std::atomic *get(uint32_t level, const K &key) { return &childs[(key >> (level * BITS)) & MASK]; } }; // Make sure the alignment works for T and RadixElement. static_assert(alignof(T) > 1, "T's alignment must be 2 or more."); static_assert(alignof(RadixNode) > 1, "RadixNode alignment must be 2 or more."); }; #endif // BITCOIN_RADIX_H diff --git a/src/test/radix_tests.cpp b/src/test/radix_tests.cpp index efbfff455..3e9dbaa54 100644 --- a/src/test/radix_tests.cpp +++ b/src/test/radix_tests.cpp @@ -1,318 +1,386 @@ // Copyright (c) 2019 The Bitcoin developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include #include #include #include #include BOOST_FIXTURE_TEST_SUITE(radix_tests, BasicTestingSetup) /** * Version of Boost::test prior to 1.64 have issues when dealing with nullptr_t. * In order to work around this, we ensure that the null pointers are typed in a * way that Boost will like better. * * TODO: Use nullptr directly once the minimum version of boost is 1.64 or more. */ #define NULLPTR(T) static_cast(nullptr) template struct TestElement { K key; TestElement(K keyIn) : key(keyIn) {} const K &getId() const { return key; } + + IMPLEMENT_RCU_REFCOUNT(uint32_t); }; template void testInsert() { typedef TestElement E; RadixTree mytree; - E zero(0); - E one(1); - E two(2); - E three(3); + auto zero = RCUPtr::make(0); + auto one = RCUPtr::make(1); + auto two = RCUPtr::make(2); + auto three = RCUPtr::make(3); // Inserting a new element into the tree returns true. - BOOST_CHECK(mytree.insert(&one)); + BOOST_CHECK(mytree.insert(one)); // Inserting an element already in the tree returns false. - BOOST_CHECK(!mytree.insert(&one)); + BOOST_CHECK(!mytree.insert(one)); // Let's insert more elements and check behavior stays consistent. - BOOST_CHECK(mytree.insert(&zero)); - BOOST_CHECK(!mytree.insert(&one)); - BOOST_CHECK(mytree.insert(&two)); - BOOST_CHECK(mytree.insert(&three)); - BOOST_CHECK(!mytree.insert(&zero)); - BOOST_CHECK(!mytree.insert(&one)); - BOOST_CHECK(!mytree.insert(&two)); - BOOST_CHECK(!mytree.insert(&three)); + BOOST_CHECK(mytree.insert(zero)); + BOOST_CHECK(!mytree.insert(one)); + BOOST_CHECK(mytree.insert(two)); + BOOST_CHECK(mytree.insert(three)); + BOOST_CHECK(!mytree.insert(zero)); + BOOST_CHECK(!mytree.insert(one)); + BOOST_CHECK(!mytree.insert(two)); + BOOST_CHECK(!mytree.insert(three)); // Check extreme values. typedef typename std::make_signed::type SK; - E maxsigned(std::numeric_limits::max()); - E minsigned(std::numeric_limits::min()); + auto maxsigned = RCUPtr::make(std::numeric_limits::max()); + auto minsigned = RCUPtr::make(std::numeric_limits::min()); typedef typename std::make_unsigned::type UK; - E minusone(std::numeric_limits::max()); - E minustwo(std::numeric_limits::max() - 1); + auto minusone = RCUPtr::make(std::numeric_limits::max()); + auto minustwo = RCUPtr::make(std::numeric_limits::max() - 1); // Insert them into the tree. - BOOST_CHECK(mytree.insert(&maxsigned)); - BOOST_CHECK(mytree.insert(&minsigned)); - BOOST_CHECK(mytree.insert(&minusone)); - BOOST_CHECK(mytree.insert(&minustwo)); + BOOST_CHECK(mytree.insert(maxsigned)); + BOOST_CHECK(mytree.insert(minsigned)); + BOOST_CHECK(mytree.insert(minusone)); + BOOST_CHECK(mytree.insert(minustwo)); // All elements are now in the tree. - BOOST_CHECK(!mytree.insert(&zero)); - BOOST_CHECK(!mytree.insert(&one)); - BOOST_CHECK(!mytree.insert(&two)); - BOOST_CHECK(!mytree.insert(&three)); - BOOST_CHECK(!mytree.insert(&maxsigned)); - BOOST_CHECK(!mytree.insert(&minsigned)); - BOOST_CHECK(!mytree.insert(&minusone)); - BOOST_CHECK(!mytree.insert(&minustwo)); + BOOST_CHECK(!mytree.insert(zero)); + BOOST_CHECK(!mytree.insert(one)); + BOOST_CHECK(!mytree.insert(two)); + BOOST_CHECK(!mytree.insert(three)); + BOOST_CHECK(!mytree.insert(maxsigned)); + BOOST_CHECK(!mytree.insert(minsigned)); + BOOST_CHECK(!mytree.insert(minusone)); + BOOST_CHECK(!mytree.insert(minustwo)); } BOOST_AUTO_TEST_CASE(insert_test) { testInsert(); testInsert(); testInsert(); testInsert(); } template void testGet() { typedef TestElement E; RadixTree mytree; - E zero(0); - E one(1); - E two(2); - E three(3); + auto zero = RCUPtr::make(0); + auto one = RCUPtr::make(1); + auto two = RCUPtr::make(2); + auto three = RCUPtr::make(3); // There are no elements in the tree so far. BOOST_CHECK_EQUAL(mytree.get(1), NULLPTR(E)); // Insert an element into the tree and check it is there. - BOOST_CHECK(mytree.insert(&one)); - BOOST_CHECK_EQUAL(mytree.get(1), &one); + BOOST_CHECK(mytree.insert(one)); + BOOST_CHECK_EQUAL(mytree.get(1), one); // Let's insert more elements and check they are recovered properly. BOOST_CHECK_EQUAL(mytree.get(0), NULLPTR(E)); - BOOST_CHECK(mytree.insert(&zero)); - BOOST_CHECK_EQUAL(mytree.get(0), &zero); - BOOST_CHECK_EQUAL(mytree.get(1), &one); + BOOST_CHECK(mytree.insert(zero)); + BOOST_CHECK_EQUAL(mytree.get(0), zero); + BOOST_CHECK_EQUAL(mytree.get(1), one); // More elements. BOOST_CHECK_EQUAL(mytree.get(2), NULLPTR(E)); BOOST_CHECK_EQUAL(mytree.get(3), NULLPTR(E)); - BOOST_CHECK(mytree.insert(&two)); - BOOST_CHECK(mytree.insert(&three)); + BOOST_CHECK(mytree.insert(two)); + BOOST_CHECK(mytree.insert(three)); - BOOST_CHECK_EQUAL(mytree.get(0), &zero); - BOOST_CHECK_EQUAL(mytree.get(1), &one); - BOOST_CHECK_EQUAL(mytree.get(2), &two); - BOOST_CHECK_EQUAL(mytree.get(3), &three); + BOOST_CHECK_EQUAL(mytree.get(0), zero); + BOOST_CHECK_EQUAL(mytree.get(1), one); + BOOST_CHECK_EQUAL(mytree.get(2), two); + BOOST_CHECK_EQUAL(mytree.get(3), three); // Check extreme values. typedef typename std::make_signed::type SK; K maxsk = std::numeric_limits::max(); K minsk = std::numeric_limits::min(); typedef typename std::make_unsigned::type UK; K maxuk = std::numeric_limits::max(); - E maxsigned(maxsk); - E minsigned(minsk); - E minusone(maxuk); - E minustwo(maxuk - 1); + auto maxsigned = RCUPtr::make(maxsk); + auto minsigned = RCUPtr::make(minsk); + auto minusone = RCUPtr::make(maxuk); + auto minustwo = RCUPtr::make(maxuk - 1); // Check that they are not in the tree. BOOST_CHECK_EQUAL(mytree.get(maxsk), NULLPTR(E)); BOOST_CHECK_EQUAL(mytree.get(minsk), NULLPTR(E)); BOOST_CHECK_EQUAL(mytree.get(maxuk), NULLPTR(E)); BOOST_CHECK_EQUAL(mytree.get(maxuk - 1), NULLPTR(E)); // Insert into the tree. - BOOST_CHECK(mytree.insert(&maxsigned)); - BOOST_CHECK(mytree.insert(&minsigned)); - BOOST_CHECK(mytree.insert(&minusone)); - BOOST_CHECK(mytree.insert(&minustwo)); + BOOST_CHECK(mytree.insert(maxsigned)); + BOOST_CHECK(mytree.insert(minsigned)); + BOOST_CHECK(mytree.insert(minusone)); + BOOST_CHECK(mytree.insert(minustwo)); // And now they all are in the tree. - BOOST_CHECK_EQUAL(mytree.get(0), &zero); - BOOST_CHECK_EQUAL(mytree.get(1), &one); - BOOST_CHECK_EQUAL(mytree.get(2), &two); - BOOST_CHECK_EQUAL(mytree.get(3), &three); - BOOST_CHECK_EQUAL(mytree.get(maxsk), &maxsigned); - BOOST_CHECK_EQUAL(mytree.get(minsk), &minsigned); - BOOST_CHECK_EQUAL(mytree.get(maxuk), &minusone); - BOOST_CHECK_EQUAL(mytree.get(maxuk - 1), &minustwo); + BOOST_CHECK_EQUAL(mytree.get(0), zero); + BOOST_CHECK_EQUAL(mytree.get(1), one); + BOOST_CHECK_EQUAL(mytree.get(2), two); + BOOST_CHECK_EQUAL(mytree.get(3), three); + BOOST_CHECK_EQUAL(mytree.get(maxsk), maxsigned); + BOOST_CHECK_EQUAL(mytree.get(minsk), minsigned); + BOOST_CHECK_EQUAL(mytree.get(maxuk), minusone); + BOOST_CHECK_EQUAL(mytree.get(maxuk - 1), minustwo); } BOOST_AUTO_TEST_CASE(get_test) { testGet(); testGet(); testGet(); testGet(); } template void testRemove() { typedef TestElement E; RadixTree mytree; - E zero(0); - E one(1); - E two(2); - E three(3); + auto zero = RCUPtr::make(0); + auto one = RCUPtr::make(1); + auto two = RCUPtr::make(2); + auto three = RCUPtr::make(3); // Removing an element that isn't in the tree returns false. BOOST_CHECK(!mytree.remove(1)); // Insert an element into the tree and check you can remove it. - BOOST_CHECK(mytree.insert(&one)); + BOOST_CHECK(mytree.insert(one)); BOOST_CHECK(mytree.remove(1)); BOOST_CHECK_EQUAL(mytree.get(1), NULLPTR(E)); // Insert several elements and remove them. - BOOST_CHECK(mytree.insert(&zero)); - BOOST_CHECK(mytree.insert(&one)); - BOOST_CHECK(mytree.insert(&two)); - BOOST_CHECK(mytree.insert(&three)); + BOOST_CHECK(mytree.insert(zero)); + BOOST_CHECK(mytree.insert(one)); + BOOST_CHECK(mytree.insert(two)); + BOOST_CHECK(mytree.insert(three)); BOOST_CHECK(mytree.remove(0)); BOOST_CHECK(mytree.remove(1)); BOOST_CHECK(mytree.remove(2)); BOOST_CHECK(mytree.remove(3)); BOOST_CHECK_EQUAL(mytree.get(0), NULLPTR(E)); BOOST_CHECK_EQUAL(mytree.get(1), NULLPTR(E)); BOOST_CHECK_EQUAL(mytree.get(2), NULLPTR(E)); BOOST_CHECK_EQUAL(mytree.get(3), NULLPTR(E)); // Once the elements are removed, removing them again returns false. BOOST_CHECK(!mytree.remove(0)); BOOST_CHECK(!mytree.remove(1)); BOOST_CHECK(!mytree.remove(2)); BOOST_CHECK(!mytree.remove(3)); // Check extreme values. typedef typename std::make_signed::type SK; K maxsk = std::numeric_limits::max(); K minsk = std::numeric_limits::min(); typedef typename std::make_unsigned::type UK; K maxuk = std::numeric_limits::max(); - E maxsigned(maxsk); - E minsigned(minsk); - E minusone(maxuk); - E minustwo(maxuk - 1); + auto maxsigned = RCUPtr::make(maxsk); + auto minsigned = RCUPtr::make(minsk); + auto minusone = RCUPtr::make(maxuk); + auto minustwo = RCUPtr::make(maxuk - 1); // Insert them all. - BOOST_CHECK(mytree.insert(&zero)); - BOOST_CHECK(mytree.insert(&one)); - BOOST_CHECK(mytree.insert(&two)); - BOOST_CHECK(mytree.insert(&three)); - BOOST_CHECK(mytree.insert(&maxsigned)); - BOOST_CHECK(mytree.insert(&minsigned)); - BOOST_CHECK(mytree.insert(&minusone)); - BOOST_CHECK(mytree.insert(&minustwo)); + BOOST_CHECK(mytree.insert(zero)); + BOOST_CHECK(mytree.insert(one)); + BOOST_CHECK(mytree.insert(two)); + BOOST_CHECK(mytree.insert(three)); + BOOST_CHECK(mytree.insert(maxsigned)); + BOOST_CHECK(mytree.insert(minsigned)); + BOOST_CHECK(mytree.insert(minusone)); + BOOST_CHECK(mytree.insert(minustwo)); // Delete them all BOOST_CHECK(mytree.remove(0)); BOOST_CHECK(mytree.remove(1)); BOOST_CHECK(mytree.remove(2)); BOOST_CHECK(mytree.remove(3)); BOOST_CHECK(mytree.remove(maxsk)); BOOST_CHECK(mytree.remove(minsk)); BOOST_CHECK(mytree.remove(maxuk)); BOOST_CHECK(mytree.remove(maxuk - 1)); BOOST_CHECK_EQUAL(mytree.get(0), NULLPTR(E)); BOOST_CHECK_EQUAL(mytree.get(1), NULLPTR(E)); BOOST_CHECK_EQUAL(mytree.get(2), NULLPTR(E)); BOOST_CHECK_EQUAL(mytree.get(3), NULLPTR(E)); BOOST_CHECK_EQUAL(mytree.get(maxsk), NULLPTR(E)); BOOST_CHECK_EQUAL(mytree.get(minsk), NULLPTR(E)); BOOST_CHECK_EQUAL(mytree.get(maxuk), NULLPTR(E)); BOOST_CHECK_EQUAL(mytree.get(maxuk - 1), NULLPTR(E)); } BOOST_AUTO_TEST_CASE(remove_test) { testRemove(); testRemove(); testRemove(); testRemove(); } +BOOST_AUTO_TEST_CASE(const_element_test) { + typedef const TestElement C; + RadixTree mytree; + + BOOST_CHECK(mytree.insert(RCUPtr::make(0))); + BOOST_CHECK(mytree.insert(RCUPtr::make(1))); + BOOST_CHECK(mytree.insert(RCUPtr::make(2))); + BOOST_CHECK(mytree.insert(RCUPtr::make(3))); + + BOOST_CHECK(!mytree.insert(RCUPtr::make(0))); + BOOST_CHECK(!mytree.insert(RCUPtr::make(1))); + BOOST_CHECK(!mytree.insert(RCUPtr::make(2))); + BOOST_CHECK(!mytree.insert(RCUPtr::make(3))); + + BOOST_CHECK(mytree.get(0)); + BOOST_CHECK(mytree.get(1)); + BOOST_CHECK(mytree.get(2)); + BOOST_CHECK(mytree.get(3)); + + BOOST_CHECK(mytree.remove(0)); + BOOST_CHECK(mytree.remove(1)); + BOOST_CHECK(mytree.remove(2)); + BOOST_CHECK(mytree.remove(3)); + + BOOST_CHECK(!mytree.get(0)); + BOOST_CHECK(!mytree.get(1)); + BOOST_CHECK(!mytree.get(2)); + BOOST_CHECK(!mytree.get(3)); + + BOOST_CHECK(!mytree.remove(0)); + BOOST_CHECK(!mytree.remove(1)); + BOOST_CHECK(!mytree.remove(2)); + BOOST_CHECK(!mytree.remove(3)); +} + +void CheckConstTree(const RadixTree> &mytree, + bool expected) { + BOOST_CHECK_EQUAL(!!mytree.get(0), expected); + BOOST_CHECK_EQUAL(!!mytree.get(1), expected); + BOOST_CHECK_EQUAL(!!mytree.get(2), expected); + BOOST_CHECK_EQUAL(!!mytree.get(3), expected); +} + +BOOST_AUTO_TEST_CASE(const_tree_test) { + typedef TestElement E; + RadixTree mytree; + + CheckConstTree(mytree, false); + + BOOST_CHECK(mytree.insert(RCUPtr::make(0))); + BOOST_CHECK(mytree.insert(RCUPtr::make(1))); + BOOST_CHECK(mytree.insert(RCUPtr::make(2))); + BOOST_CHECK(mytree.insert(RCUPtr::make(3))); + + CheckConstTree(mytree, true); + + BOOST_CHECK(mytree.remove(0)); + BOOST_CHECK(mytree.remove(1)); + BOOST_CHECK(mytree.remove(2)); + BOOST_CHECK(mytree.remove(3)); + + CheckConstTree(mytree, false); +} + #define THREADS 128 #define ELEMENTS 65536 static uint64_t next(uint64_t x) { // Simple linear congruential generator by Donald Knuth. return x * 6364136223846793005 + 1442695040888963407; } BOOST_AUTO_TEST_CASE(insert_stress_test) { typedef TestElement E; RadixTree mytree; std::atomic success{0}; std::vector threads; for (int i = 0; i < THREADS; i++) { threads.push_back(std::thread([&] { uint64_t rand = 0; for (int j = 0; j < ELEMENTS; j++) { rand = next(rand); uint32_t v(rand >> 32); if (mytree.remove(v)) { success--; std::this_thread::yield(); } - std::unique_ptr ptr = MakeUnique(v); - if (mytree.insert(ptr.get())) { + auto ptr = RCUPtr::make(v); + if (mytree.insert(ptr)) { success++; std::this_thread::yield(); } if (mytree.remove(v)) { success--; - RCULock::synchronize(); + std::this_thread::yield(); } - if (mytree.insert(ptr.get())) { + if (mytree.insert(ptr)) { success++; - ptr.release(); std::this_thread::yield(); } } })); } // Wait for all the threads to finish. for (std::thread &t : threads) { t.join(); } BOOST_CHECK_EQUAL(success.load(), ELEMENTS); // All the elements have been inserted into the tree. uint64_t rand = 0; for (int i = 0; i < ELEMENTS; i++) { rand = next(rand); uint32_t v(rand >> 32); BOOST_CHECK_EQUAL(mytree.get(v)->getId(), v); - E e(v); - BOOST_CHECK(!mytree.insert(&e)); - BOOST_CHECK(mytree.get(v) != &e); + auto ptr = RCUPtr::make(v); + BOOST_CHECK(!mytree.insert(ptr)); + BOOST_CHECK(mytree.get(v) != ptr); } + + // Cleanup after ourselves. + RCULock::synchronize(); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/rcu_tests.cpp b/src/test/rcu_tests.cpp index 91fab7d5d..cf4296e5f 100644 --- a/src/test/rcu_tests.cpp +++ b/src/test/rcu_tests.cpp @@ -1,399 +1,400 @@ // Copyright (c) 2018 The Bitcoin developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include "rcu.h" #include "test/test_bitcoin.h" #include #include struct RCUTest { static uint64_t getRevision() { return RCUInfos::revision.load(); } static uint64_t hasSyncedTo(uint64_t syncRev) { return RCUInfos::infos.hasSyncedTo(syncRev); } static std::map> &getCleanups() { return RCUInfos::infos.cleanups; } }; BOOST_FIXTURE_TEST_SUITE(rcu_tests, BasicTestingSetup) enum RCUTestStep { Init, Locked, LockAck, RCULocked, Synchronizing, Synchronized, }; #define WAIT_FOR_STEP(step) \ do { \ cond.notify_all(); \ } while (!cond.wait_for(lock, std::chrono::milliseconds(1), \ [&] { return otherstep == step; })) void synchronize(std::atomic &step, const std::atomic &otherstep, CWaitableCriticalSection &cs, std::condition_variable &cond, std::atomic &syncRev) { assert(step == RCUTestStep::Init); { WAIT_LOCK(cs, lock); step = RCUTestStep::Locked; // Wait for our lock to be acknowledged. WAIT_FOR_STEP(RCUTestStep::LockAck); RCULock rculock; // Update step. step = RCUTestStep::RCULocked; // Wait for master. WAIT_FOR_STEP(RCUTestStep::RCULocked); } // Update step. syncRev = RCUTest::getRevision() + 1; step = RCUTestStep::Synchronizing; assert(!RCUTest::hasSyncedTo(syncRev)); // We wait for readers. RCULock::synchronize(); // Update step. step = RCUTestStep::Synchronized; } void lockAndWaitForSynchronize(std::atomic &step, const std::atomic &otherstep, CWaitableCriticalSection &cs, std::condition_variable &cond, std::atomic &syncRev) { assert(step == RCUTestStep::Init); WAIT_LOCK(cs, lock); // Wait for th eother thread to be locked. WAIT_FOR_STEP(RCUTestStep::Locked); step = RCUTestStep::LockAck; // Wait for the synchronizing tread to take its RCU lock. WAIT_FOR_STEP(RCUTestStep::RCULocked); assert(!RCUTest::hasSyncedTo(syncRev)); { RCULock rculock; // Update master step. step = RCUTestStep::RCULocked; while (RCUTest::getRevision() < syncRev) { WAIT_FOR_STEP(RCUTestStep::Synchronizing); } assert(RCUTest::getRevision() >= syncRev); assert(otherstep.load() == RCUTestStep::Synchronizing); } assert(RCUTest::hasSyncedTo(syncRev) >= syncRev); WAIT_FOR_STEP(RCUTestStep::Synchronized); } static const int COUNT = 128; BOOST_AUTO_TEST_CASE(synchronize_test) { CWaitableCriticalSection cs; std::condition_variable cond; std::atomic parentstep; std::atomic childstep; std::atomic syncRev; for (int i = 0; i < COUNT; i++) { parentstep = RCUTestStep::Init; childstep = RCUTestStep::Init; syncRev = RCUTest::getRevision() + 1; std::thread tlock([&] { lockAndWaitForSynchronize(parentstep, childstep, cs, cond, syncRev); }); std::thread tsync( [&] { synchronize(childstep, parentstep, cs, cond, syncRev); }); tlock.join(); tsync.join(); } } /** * Version of Boost::test prior to 1.64 have issues when dealing with nullptr_t. * In order to work around this, we ensure that the null pointers are typed in a * way that Boost will like better. * * TODO: Use nullptr directly once the minimum version of boost is 1.64 or more. */ #define NULLPTR(T) static_cast(nullptr) BOOST_AUTO_TEST_CASE(cleanup_test) { + RCULock::synchronize(); BOOST_CHECK(RCUTest::getCleanups().empty()); bool isClean1 = false; RCULock::registerCleanup([&] { isClean1 = true; }); BOOST_CHECK(!isClean1); BOOST_CHECK_EQUAL(RCUTest::getCleanups().size(), 1); BOOST_CHECK_EQUAL(RCUTest::getRevision(), RCUTest::getCleanups().begin()->first); // Synchronize runs the cleanups. RCULock::synchronize(); BOOST_CHECK(RCUTest::getCleanups().empty()); BOOST_CHECK(isClean1); // Check multiple callbacks. isClean1 = false; bool isClean2 = false; bool isClean3 = false; RCULock::registerCleanup([&] { isClean1 = true; }); RCULock::registerCleanup([&] { isClean2 = true; }); RCULock::registerCleanup([&] { isClean3 = true; }); BOOST_CHECK_EQUAL(RCUTest::getCleanups().size(), 3); RCULock::synchronize(); BOOST_CHECK(RCUTest::getCleanups().empty()); BOOST_CHECK(isClean1); BOOST_CHECK(isClean2); BOOST_CHECK(isClean3); // Check callbacks adding each others. isClean1 = false; isClean2 = false; isClean3 = false; RCULock::registerCleanup([&] { isClean1 = true; RCULock::registerCleanup([&] { isClean2 = true; RCULock::registerCleanup([&] { isClean3 = true; }); }); }); BOOST_CHECK_EQUAL(RCUTest::getCleanups().size(), 1); RCULock::synchronize(); BOOST_CHECK(RCUTest::getCleanups().empty()); BOOST_CHECK(isClean1); BOOST_CHECK(isClean2); BOOST_CHECK(isClean3); } class RCURefTestItem { IMPLEMENT_RCU_REFCOUNT(uint32_t); const std::function cleanupfun; public: explicit RCURefTestItem(const std::function &fun) : cleanupfun(fun) {} ~RCURefTestItem() { cleanupfun(); } uint32_t getRefCount() const { return refcount.load(); } }; BOOST_AUTO_TEST_CASE(rcuptr_test) { // Make sure it works for null. { RCURefTestItem *ptr = nullptr; RCUPtr::copy(ptr); RCUPtr::acquire(ptr); } // Check the destruction mechanism. bool isDestroyed = false; { auto rcuptr = RCUPtr::make([&] { isDestroyed = true; }); BOOST_CHECK_EQUAL(rcuptr->getRefCount(), 0); } // rcuptr waits for synchronization to destroy. BOOST_CHECK(!isDestroyed); RCULock::synchronize(); BOOST_CHECK(isDestroyed); // Check that copy behaves properly. isDestroyed = false; RCUPtr gptr; { auto rcuptr = RCUPtr::make([&] { isDestroyed = true; }); BOOST_CHECK_EQUAL(rcuptr->getRefCount(), 0); gptr = rcuptr; BOOST_CHECK_EQUAL(rcuptr->getRefCount(), 1); BOOST_CHECK_EQUAL(gptr->getRefCount(), 1); auto rcuptrcopy = rcuptr; BOOST_CHECK_EQUAL(rcuptrcopy->getRefCount(), 2); BOOST_CHECK_EQUAL(rcuptr->getRefCount(), 2); BOOST_CHECK_EQUAL(gptr->getRefCount(), 2); } BOOST_CHECK_EQUAL(gptr->getRefCount(), 0); RCULock::synchronize(); BOOST_CHECK(!isDestroyed); gptr = RCUPtr(); BOOST_CHECK(!isDestroyed); RCULock::synchronize(); BOOST_CHECK(isDestroyed); } BOOST_AUTO_TEST_CASE(rcuptr_operator_test) { auto gptr = RCUPtr(); auto ptr = new RCURefTestItem([] {}); auto oldPtr = ptr; auto altptr = RCUPtr::make([] {}); // Check various operators. BOOST_CHECK_EQUAL(gptr.get(), NULLPTR(RCURefTestItem)); BOOST_CHECK_EQUAL(&*gptr, NULLPTR(RCURefTestItem)); BOOST_CHECK_EQUAL(gptr, NULLPTR(RCURefTestItem)); BOOST_CHECK(!gptr); auto copyptr = gptr; BOOST_CHECK(gptr == nullptr); BOOST_CHECK(gptr != oldPtr); BOOST_CHECK(gptr == copyptr); BOOST_CHECK(gptr != altptr); gptr = RCUPtr::acquire(ptr); BOOST_CHECK_EQUAL(ptr, NULLPTR(RCURefTestItem)); BOOST_CHECK_EQUAL(gptr.get(), oldPtr); BOOST_CHECK_EQUAL(&*gptr, oldPtr); BOOST_CHECK_EQUAL(gptr, oldPtr); BOOST_CHECK(gptr); copyptr = gptr; BOOST_CHECK(gptr != nullptr); BOOST_CHECK(gptr == oldPtr); BOOST_CHECK(gptr == copyptr); BOOST_CHECK(gptr != altptr); } BOOST_AUTO_TEST_CASE(const_rcuptr_test) { bool isDestroyed = false; auto ptr = RCUPtr::make([&] { isDestroyed = true; }); // Now let's destroy it. ptr = RCUPtr(); BOOST_CHECK(!isDestroyed); RCULock::synchronize(); BOOST_CHECK(isDestroyed); } class RCURefMoveTestItem { const std::function cleanupfun; public: explicit RCURefMoveTestItem(const std::function &fun) : cleanupfun(fun) {} ~RCURefMoveTestItem() { cleanupfun(); } void acquire() { throw std::runtime_error("RCUPtr incremented the refcount"); } void release() { RCULock::registerCleanup([this] { delete this; }); } }; BOOST_AUTO_TEST_CASE(move_rcuptr_test) { bool isDestroyed = false; // Check tat copy is failing. auto rcuptr1 = RCUPtr::make([&] { isDestroyed = true; }); BOOST_CHECK_THROW(rcuptr1->acquire(), std::runtime_error); BOOST_CHECK_THROW(auto rcuptrcopy = rcuptr1;, std::runtime_error); // Try to move. auto rcuptr2 = std::move(rcuptr1); RCULock::synchronize(); BOOST_CHECK(!isDestroyed); // Move to a local and check proper destruction. { auto rcuptr3 = std::move(rcuptr2); } BOOST_CHECK(!isDestroyed); RCULock::synchronize(); BOOST_CHECK(isDestroyed); // Let's try to swap. isDestroyed = false; rcuptr1 = RCUPtr::make([&] { isDestroyed = true; }); std::swap(rcuptr1, rcuptr2); RCULock::synchronize(); BOOST_CHECK(!isDestroyed); // Chain moves to make sure there are no double free. { auto rcuptr3 = std::move(rcuptr2); auto rcuptr4 = std::move(rcuptr3); std::swap(rcuptr1, rcuptr4); } RCULock::synchronize(); BOOST_CHECK(!isDestroyed); // Check we can return from a function. { auto r = ([&] { auto moved = std::move(rcuptr1); return moved; })(); RCULock::synchronize(); BOOST_CHECK(!isDestroyed); } BOOST_CHECK(!isDestroyed); RCULock::synchronize(); BOOST_CHECK(isDestroyed); // Acquire/release workflow. isDestroyed = false; auto ptr = new RCURefMoveTestItem([&] { isDestroyed = true; }); auto ptrCopy = ptr; BOOST_CHECK_THROW(RCUPtr::copy(ptr), std::runtime_error); rcuptr1 = RCUPtr::acquire(ptr); BOOST_CHECK_EQUAL(rcuptr1, ptrCopy); BOOST_CHECK_EQUAL(ptr, NULLPTR(RCURefMoveTestItem)); ptr = rcuptr1.release(); BOOST_CHECK_EQUAL(rcuptr1, NULLPTR(RCURefMoveTestItem)); BOOST_CHECK_EQUAL(ptr, ptrCopy); RCULock::synchronize(); BOOST_CHECK(!isDestroyed); RCUPtr::acquire(ptr); BOOST_CHECK_EQUAL(ptr, NULLPTR(RCURefMoveTestItem)); BOOST_CHECK(!isDestroyed); RCULock::synchronize(); BOOST_CHECK(isDestroyed); } BOOST_AUTO_TEST_SUITE_END()