diff --git a/src/radix.h b/src/radix.h index e6dcb6962..deb268e77 100644 --- a/src/radix.h +++ b/src/radix.h @@ -1,389 +1,410 @@ // Copyright (c) 2019 The Bitcoin developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #ifndef BITCOIN_RADIX_H #define BITCOIN_RADIX_H +#include #include #include #include #include #include #include #include /** * This is a radix tree storing values identified by a unique key. * * The tree is composed of nodes (RadixNode) containing an array of * RadixElement. The key is split into chunks of a few bits that serve as an * index into that array. RadixElement is a discriminated union of either a * RadixNode* representing the next level in the tree, or a T* representing a * leaf. New RadixNode are added lazily when two leaves would go in the same * slot. * * Reads walk the tree using sequential atomic loads, and insertions are done * using CAS, which ensures both can be executed lock free. Removing any * elements from the tree can also be done using CAS, but requires waiting for * other readers before being destroyed. The tree uses RCU to track which thread * is reading the tree, which allows deletion to wait for other readers to be up * to speed before destroying anything. It is therefore crucial that the lock be * taken before reading anything in the tree. */ template struct RadixTree { private: static const int BITS = 4; static const int MASK = (1 << BITS) - 1; static const size_t CHILD_PER_LEVEL = 1 << BITS; using KeyType = typename std::remove_reference().getId())>::type; static const size_t KEY_BITS = 8 * sizeof(KeyType); static const uint32_t TOP_LEVEL = (KEY_BITS - 1) / BITS; struct RadixElement; struct RadixNode; std::atomic root; public: RadixTree() : root(RadixElement()) {} ~RadixTree() { root.load().decrementRefCount(); } /** * Copy semantic. */ RadixTree(const RadixTree &src) : RadixTree() { { RCULock lock; RadixElement e = src.root.load(); e.incrementRefCount(); root = e; } // Make sure we the writes in the tree are behind us so // this copy won't mutate behind our back. RCULock::synchronize(); } RadixTree &operator=(const RadixTree &rhs) { { RCULock lock; RadixElement e = rhs.root.load(); e.incrementRefCount(); root.load().decrementRefCount(); root = e; } // Make sure we the writes in the tree are behind us so // this copy won't mutate behind our back. RCULock::synchronize(); return *this; } /** * Move semantic. */ RadixTree(RadixTree &&src) : RadixTree() { *this = std::move(src); } RadixTree &operator=(RadixTree &&rhs) { { RCULock lock; RadixElement e = rhs.root.load(); rhs.root = root.load(); root = e; } return *this; } /** * Insert a value into the tree. * Returns true if the value was inserted, false if it was already present. */ bool insert(const RCUPtr &value) { return insert(value->getId(), value); } /** * Get the value corresponding to a key. * Returns the value if found, nullptr if not. */ RCUPtr get(const KeyType &key) { uint32_t level = TOP_LEVEL; RCULock lock; RadixElement e = root.load(); // Find a leaf. while (e.isNode()) { e = e.getNode()->get(level--, key)->load(); } T *leaf = e.getLeaf(); if (leaf == nullptr || leaf->getId() != key) { // We failed to find the proper element. return RCUPtr(); } // The leaf is non-null and the keys match. We have our guy. return RCUPtr::copy(leaf); } RCUPtr get(const KeyType &key) const { T const *ptr = const_cast(this)->get(key).release(); return RCUPtr::acquire(ptr); } template void forEachLeaf(Callable &&func) const { RCULock lock; forEachLeaf(root.load(), std::move(func)); } #define SEEK_LEAF_LOOP() \ RadixElement e = eptr->load(); \ \ /* Walk down the tree until we find a leaf for our node. */ \ do { \ while (e.isNode()) { \ Node: \ auto nptr = e.getNode(); \ if (!nptr->isShared()) { \ eptr = nptr->get(level--, key); \ e = eptr->load(); \ continue; \ } \ \ auto copy = std::make_unique(*nptr); \ if (!eptr->compare_exchange_strong(e, RadixElement(copy.get()))) { \ /* We failed to insert our subtree, just try again. */ \ continue; \ } \ \ /* We have a subtree, resume normal operations from there. */ \ e.decrementRefCount(); \ eptr = copy->get(level--, key); \ e = eptr->load(); \ copy.release(); \ } \ } while (0) /** * Remove an element from the tree. * Returns the removed element, or nullptr if there isn't one. */ RCUPtr remove(const KeyType &key) { uint32_t level = TOP_LEVEL; RCULock lock; std::atomic *eptr = &root; SEEK_LEAF_LOOP(); T *leaf = e.getLeaf(); if (leaf == nullptr || leaf->getId() != key) { // We failed to find the proper element. return RCUPtr(); } // We have the proper element, try to delete it. if (eptr->compare_exchange_strong(e, RadixElement())) { return RCUPtr::acquire(leaf); } // The element was replaced, either by a subtree or another element. if (e.isNode()) { goto Node; } // The element in the slot is not the one we are looking for. return RCUPtr(); } private: bool insert(const KeyType &key, RCUPtr value) { uint32_t level = TOP_LEVEL; RCULock lock; std::atomic *eptr = &root; while (true) { SEEK_LEAF_LOOP(); // If the slot is empty, try to insert right there. if (e.getLeaf() == nullptr) { if (eptr->compare_exchange_strong(e, RadixElement(value.get()))) { value.release(); return true; } // CAS failed, we may have a node in there now. if (e.isNode()) { goto Node; } } // The element was already in the tree. const KeyType &leafKey = e.getLeaf()->getId(); if (key == leafKey) { return false; } // There is an element there, but it isn't a subtree. We need to // convert it into a subtree and resume insertion into that subtree. auto newChild = std::make_unique(level, leafKey, e); if (eptr->compare_exchange_strong(e, RadixElement(newChild.get()))) { // We have a subtree, resume normal operations from there. newChild.release(); } else { // We failed to insert our subtree, clean it before it is freed. newChild->get(level, leafKey)->store(RadixElement()); } } } #undef SEEK_LEAF_LOOP template void forEachLeaf(RadixElement e, Callable &&func) const { if (e.isNode()) { e.getNode()->forEachChild( [&](const std::atomic *pElement) { forEachLeaf(pElement->load(), func); }); return; } T *leaf = e.getLeaf(); if (leaf != nullptr) { func(RCUPtr::copy(leaf)); } } struct RadixElement { private: union { RadixNode *node; T *leaf; uintptr_t raw; }; static const uintptr_t DISCRIMINANT = 0x01; bool getDiscriminant() const { return (raw & DISCRIMINANT) != 0; } public: explicit RadixElement() noexcept : raw(DISCRIMINANT) {} explicit RadixElement(RadixNode *nodeIn) noexcept : node(nodeIn) {} explicit RadixElement(T *leafIn) noexcept : leaf(leafIn) { raw |= DISCRIMINANT; } /** * RadixElement is designed to be a dumb wrapper. This allows any * container to release what is held by the RadixElement. */ void incrementRefCount() { if (isNode()) { RCUPtr::copy(getNode()).release(); } else { RCUPtr::copy(getLeaf()).release(); } } void decrementRefCount() { if (isNode()) { RadixNode *ptr = getNode(); RCUPtr::acquire(ptr); } else { T *ptr = getLeaf(); RCUPtr::acquire(ptr); } } /** * Node features. */ bool isNode() const { return !getDiscriminant(); } RadixNode *getNode() { assert(isNode()); return node; } const RadixNode *getNode() const { assert(isNode()); return node; } /** * Leaf features. */ bool isLeaf() const { return getDiscriminant(); } T *getLeaf() { assert(isLeaf()); return reinterpret_cast(raw & ~DISCRIMINANT); } const T *getLeaf() const { assert(isLeaf()); return const_cast(this)->getLeaf(); } }; struct RadixNode { IMPLEMENT_RCU_REFCOUNT(uint64_t); private: union { std::array, CHILD_PER_LEVEL> children; std::array non_atomic_children_DO_NOT_USE; }; public: RadixNode(uint32_t level, const KeyType &key, RadixElement e) : non_atomic_children_DO_NOT_USE() { get(level, key)->store(e); } ~RadixNode() { for (RadixElement e : non_atomic_children_DO_NOT_USE) { e.decrementRefCount(); } } RadixNode(const RadixNode &rhs) : non_atomic_children_DO_NOT_USE() { for (size_t i = 0; i < CHILD_PER_LEVEL; i++) { auto e = rhs.children[i].load(); e.incrementRefCount(); non_atomic_children_DO_NOT_USE[i] = e; } } RadixNode &operator=(const RadixNode &) = delete; std::atomic *get(uint32_t level, const KeyType &key) { - return &children[(key >> (level * BITS)) & MASK]; + return &children[(key >> uint32_t(level * BITS)) & MASK]; } bool isShared() const { return refcount > 0; } template void forEachChild(Callable &&func) const { for (size_t i = 0; i < CHILD_PER_LEVEL; i++) { func(&children[i]); } } }; // Make sure the alignment works for T and RadixElement. static_assert(alignof(T) > 1, "T's alignment must be 2 or more."); static_assert(alignof(RadixNode) > 1, "RadixNode alignment must be 2 or more."); }; +/** + * Facility for using an uint256 as a radix tree key. + */ +struct Uint256KeyWrapper { + arith_uint256 base; + + Uint256KeyWrapper(const uint256 &keyIn) : base(UintToArith256(keyIn)) {} + Uint256KeyWrapper(const base_uint<256> &keyIn) : base(keyIn) {} + + Uint256KeyWrapper operator>>(uint32_t shift) const { return base >> shift; } + Uint256KeyWrapper operator&(const Uint256KeyWrapper &mask) const { + return base & mask.base; + } + operator size_t() const { return size_t(base.GetLow64()); } +}; + +// The radix tree relies on sizeof to gather the bit length of the key +static_assert(sizeof(Uint256KeyWrapper) == 32, + "Uint256KeyWrapper key size should be 256 bits"); + #endif // BITCOIN_RADIX_H diff --git a/src/test/radix_tests.cpp b/src/test/radix_tests.cpp index 4bd83c069..3a3d1db6a 100644 --- a/src/test/radix_tests.cpp +++ b/src/test/radix_tests.cpp @@ -1,478 +1,583 @@ // Copyright (c) 2019 The Bitcoin developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include +#include +#include + #include #include #include #include #include BOOST_FIXTURE_TEST_SUITE(radix_tests, BasicTestingSetup) template struct TestElement { K key; TestElement(K keyIn) : key(keyIn) {} const K &getId() const { return key; } IMPLEMENT_RCU_REFCOUNT(uint32_t); }; -template void testInsert() { - using E = TestElement; +template struct TestElementInt : TestElement { + TestElementInt(K keyIn) : TestElement(std::move(keyIn)) {} + + static auto SignedMin() { + return std::numeric_limits::type>::min(); + } + static auto SignedMax() { + return std::numeric_limits::type>::max(); + } + static auto MinusOne() { + return std::numeric_limits::type>::max(); + } + static auto MinusTwo() { + return std::numeric_limits< + typename std::make_unsigned::type>::max() - + 1; + } +}; + +struct TestElementUint256 : TestElement { + TestElementUint256(const uint256 &keyIn) + : TestElement(Uint256KeyWrapper(keyIn)) {} + TestElementUint256(uint64_t keyIn) + : TestElement(Uint256KeyWrapper(keyIn)) {} + + static inline arith_uint256 signedMin = arith_uint256(1) << 255; + static inline arith_uint256 signedMax = ~signedMin; + static inline arith_uint256 minusOne = arith_uint256(0) - 1; + static inline arith_uint256 minusTwo = minusOne - 1; + + static uint256 SignedMin() { return ArithToUint256(signedMin); } + static uint256 SignedMax() { return ArithToUint256(signedMax); } + static uint256 MinusOne() { return ArithToUint256(minusOne); } + static uint256 MinusTwo() { return ArithToUint256(minusTwo); } +}; + +template void testInsert() { RadixTree mytree; auto zero = RCUPtr::make(0); auto one = RCUPtr::make(1); auto two = RCUPtr::make(2); auto three = RCUPtr::make(3); // Inserting a new element into the tree returns true. BOOST_CHECK(mytree.insert(one)); // Inserting an element already in the tree returns false. BOOST_CHECK(!mytree.insert(one)); // Let's insert more elements and check behavior stays consistent. BOOST_CHECK(mytree.insert(zero)); BOOST_CHECK(!mytree.insert(one)); BOOST_CHECK(mytree.insert(two)); BOOST_CHECK(mytree.insert(three)); BOOST_CHECK(!mytree.insert(zero)); BOOST_CHECK(!mytree.insert(one)); BOOST_CHECK(!mytree.insert(two)); BOOST_CHECK(!mytree.insert(three)); // Check extreme values. - using SK = typename std::make_signed::type; - auto maxsigned = RCUPtr::make(std::numeric_limits::max()); - auto minsigned = RCUPtr::make(std::numeric_limits::min()); - using UK = typename std::make_unsigned::type; - auto minusone = RCUPtr::make(std::numeric_limits::max()); - auto minustwo = RCUPtr::make(std::numeric_limits::max() - 1); + auto maxsigned = RCUPtr::make(E::SignedMax()); + auto minsigned = RCUPtr::make(E::SignedMin()); + auto minusone = RCUPtr::make(E::MinusOne()); + auto minustwo = RCUPtr::make(E::MinusTwo()); // Insert them into the tree. BOOST_CHECK(mytree.insert(maxsigned)); BOOST_CHECK(mytree.insert(minsigned)); BOOST_CHECK(mytree.insert(minusone)); BOOST_CHECK(mytree.insert(minustwo)); // All elements are now in the tree. BOOST_CHECK(!mytree.insert(zero)); BOOST_CHECK(!mytree.insert(one)); BOOST_CHECK(!mytree.insert(two)); BOOST_CHECK(!mytree.insert(three)); BOOST_CHECK(!mytree.insert(maxsigned)); BOOST_CHECK(!mytree.insert(minsigned)); BOOST_CHECK(!mytree.insert(minusone)); BOOST_CHECK(!mytree.insert(minustwo)); } BOOST_AUTO_TEST_CASE(insert_test) { - testInsert(); - testInsert(); - testInsert(); - testInsert(); + testInsert>(); + testInsert>(); + testInsert>(); + testInsert>(); + + testInsert(); } -template void testGet() { - using E = TestElement; +template void testGet() { RadixTree mytree; auto zero = RCUPtr::make(0); auto one = RCUPtr::make(1); auto two = RCUPtr::make(2); auto three = RCUPtr::make(3); + auto keyZero = zero->getId(); + auto keyOne = one->getId(); + auto keyTwo = two->getId(); + auto keyThree = three->getId(); + // There are no elements in the tree so far. - BOOST_CHECK_EQUAL(mytree.get(1), NULLPTR(E)); + BOOST_CHECK_EQUAL(mytree.get(keyOne), NULLPTR(E)); // Insert an element into the tree and check it is there. BOOST_CHECK(mytree.insert(one)); - BOOST_CHECK_EQUAL(mytree.get(1), one); + BOOST_CHECK_EQUAL(mytree.get(keyOne), one); // Let's insert more elements and check they are recovered properly. - BOOST_CHECK_EQUAL(mytree.get(0), NULLPTR(E)); + BOOST_CHECK_EQUAL(mytree.get(keyZero), NULLPTR(E)); BOOST_CHECK(mytree.insert(zero)); - BOOST_CHECK_EQUAL(mytree.get(0), zero); - BOOST_CHECK_EQUAL(mytree.get(1), one); + BOOST_CHECK_EQUAL(mytree.get(keyZero), zero); + BOOST_CHECK_EQUAL(mytree.get(keyOne), one); // More elements. - BOOST_CHECK_EQUAL(mytree.get(2), NULLPTR(E)); - BOOST_CHECK_EQUAL(mytree.get(3), NULLPTR(E)); + BOOST_CHECK_EQUAL(mytree.get(keyTwo), NULLPTR(E)); + BOOST_CHECK_EQUAL(mytree.get(keyThree), NULLPTR(E)); BOOST_CHECK(mytree.insert(two)); BOOST_CHECK(mytree.insert(three)); - BOOST_CHECK_EQUAL(mytree.get(0), zero); - BOOST_CHECK_EQUAL(mytree.get(1), one); - BOOST_CHECK_EQUAL(mytree.get(2), two); - BOOST_CHECK_EQUAL(mytree.get(3), three); + BOOST_CHECK_EQUAL(mytree.get(keyZero), zero); + BOOST_CHECK_EQUAL(mytree.get(keyOne), one); + BOOST_CHECK_EQUAL(mytree.get(keyTwo), two); + BOOST_CHECK_EQUAL(mytree.get(keyThree), three); - // Check extreme values. - using SK = typename std::make_signed::type; - K maxsk = std::numeric_limits::max(); - K minsk = std::numeric_limits::min(); - using UK = typename std::make_unsigned::type; - K maxuk = std::numeric_limits::max(); - - auto maxsigned = RCUPtr::make(maxsk); - auto minsigned = RCUPtr::make(minsk); - auto minusone = RCUPtr::make(maxuk); - auto minustwo = RCUPtr::make(maxuk - 1); + auto maxsigned = RCUPtr::make(E::SignedMax()); + auto minsigned = RCUPtr::make(E::SignedMin()); + auto minusone = RCUPtr::make(E::MinusOne()); + auto minustwo = RCUPtr::make(E::MinusTwo()); // Check that they are not in the tree. - BOOST_CHECK_EQUAL(mytree.get(maxsk), NULLPTR(E)); - BOOST_CHECK_EQUAL(mytree.get(minsk), NULLPTR(E)); - BOOST_CHECK_EQUAL(mytree.get(maxuk), NULLPTR(E)); - BOOST_CHECK_EQUAL(mytree.get(maxuk - 1), NULLPTR(E)); + BOOST_CHECK_EQUAL(mytree.get(E::SignedMax()), NULLPTR(E)); + BOOST_CHECK_EQUAL(mytree.get(E::SignedMin()), NULLPTR(E)); + BOOST_CHECK_EQUAL(mytree.get(E::MinusOne()), NULLPTR(E)); + BOOST_CHECK_EQUAL(mytree.get(E::MinusTwo()), NULLPTR(E)); // Insert into the tree. BOOST_CHECK(mytree.insert(maxsigned)); BOOST_CHECK(mytree.insert(minsigned)); BOOST_CHECK(mytree.insert(minusone)); BOOST_CHECK(mytree.insert(minustwo)); // And now they all are in the tree. - BOOST_CHECK_EQUAL(mytree.get(0), zero); - BOOST_CHECK_EQUAL(mytree.get(1), one); - BOOST_CHECK_EQUAL(mytree.get(2), two); - BOOST_CHECK_EQUAL(mytree.get(3), three); - BOOST_CHECK_EQUAL(mytree.get(maxsk), maxsigned); - BOOST_CHECK_EQUAL(mytree.get(minsk), minsigned); - BOOST_CHECK_EQUAL(mytree.get(maxuk), minusone); - BOOST_CHECK_EQUAL(mytree.get(maxuk - 1), minustwo); + BOOST_CHECK_EQUAL(mytree.get(keyZero), zero); + BOOST_CHECK_EQUAL(mytree.get(keyOne), one); + BOOST_CHECK_EQUAL(mytree.get(keyTwo), two); + BOOST_CHECK_EQUAL(mytree.get(keyThree), three); + BOOST_CHECK_EQUAL(mytree.get(E::SignedMax()), maxsigned); + BOOST_CHECK_EQUAL(mytree.get(E::SignedMin()), minsigned); + BOOST_CHECK_EQUAL(mytree.get(E::MinusOne()), minusone); + BOOST_CHECK_EQUAL(mytree.get(E::MinusTwo()), minustwo); } BOOST_AUTO_TEST_CASE(get_test) { - testGet(); - testGet(); - testGet(); - testGet(); + testGet>(); + testGet>(); + testGet>(); + testGet>(); + + testGet(); } -template void testRemove() { - using E = TestElement; +template void testRemove() { RadixTree mytree; auto zero = RCUPtr::make(0); auto one = RCUPtr::make(1); auto two = RCUPtr::make(2); auto three = RCUPtr::make(3); + auto keyZero = zero->getId(); + auto keyOne = one->getId(); + auto keyTwo = two->getId(); + auto keyThree = three->getId(); + // Removing an element that isn't in the tree returns false. - BOOST_CHECK(!mytree.remove(1)); + BOOST_CHECK(!mytree.remove(keyOne)); // Insert an element into the tree and check you can remove it. BOOST_CHECK(mytree.insert(one)); - BOOST_CHECK(mytree.remove(1)); - BOOST_CHECK_EQUAL(mytree.get(1), NULLPTR(E)); + BOOST_CHECK(mytree.remove(keyOne)); + BOOST_CHECK_EQUAL(mytree.get(keyOne), NULLPTR(E)); // Insert several elements and remove them. BOOST_CHECK(mytree.insert(zero)); BOOST_CHECK(mytree.insert(one)); BOOST_CHECK(mytree.insert(two)); BOOST_CHECK(mytree.insert(three)); - BOOST_CHECK(mytree.remove(0)); - BOOST_CHECK(mytree.remove(1)); - BOOST_CHECK(mytree.remove(2)); - BOOST_CHECK(mytree.remove(3)); + BOOST_CHECK(mytree.remove(keyZero)); + BOOST_CHECK(mytree.remove(keyOne)); + BOOST_CHECK(mytree.remove(keyTwo)); + BOOST_CHECK(mytree.remove(keyThree)); - BOOST_CHECK_EQUAL(mytree.get(0), NULLPTR(E)); - BOOST_CHECK_EQUAL(mytree.get(1), NULLPTR(E)); - BOOST_CHECK_EQUAL(mytree.get(2), NULLPTR(E)); - BOOST_CHECK_EQUAL(mytree.get(3), NULLPTR(E)); + BOOST_CHECK_EQUAL(mytree.get(keyZero), NULLPTR(E)); + BOOST_CHECK_EQUAL(mytree.get(keyOne), NULLPTR(E)); + BOOST_CHECK_EQUAL(mytree.get(keyTwo), NULLPTR(E)); + BOOST_CHECK_EQUAL(mytree.get(keyThree), NULLPTR(E)); // Once the elements are removed, removing them again returns false. - BOOST_CHECK(!mytree.remove(0)); - BOOST_CHECK(!mytree.remove(1)); - BOOST_CHECK(!mytree.remove(2)); - BOOST_CHECK(!mytree.remove(3)); + BOOST_CHECK(!mytree.remove(keyZero)); + BOOST_CHECK(!mytree.remove(keyOne)); + BOOST_CHECK(!mytree.remove(keyTwo)); + BOOST_CHECK(!mytree.remove(keyThree)); // Check extreme values. - using SK = typename std::make_signed::type; - K maxsk = std::numeric_limits::max(); - K minsk = std::numeric_limits::min(); - using UK = typename std::make_unsigned::type; - K maxuk = std::numeric_limits::max(); - - auto maxsigned = RCUPtr::make(maxsk); - auto minsigned = RCUPtr::make(minsk); - auto minusone = RCUPtr::make(maxuk); - auto minustwo = RCUPtr::make(maxuk - 1); + auto maxsigned = RCUPtr::make(E::SignedMax()); + auto minsigned = RCUPtr::make(E::SignedMin()); + auto minusone = RCUPtr::make(E::MinusOne()); + auto minustwo = RCUPtr::make(E::MinusTwo()); // Insert them all. BOOST_CHECK(mytree.insert(zero)); BOOST_CHECK(mytree.insert(one)); BOOST_CHECK(mytree.insert(two)); BOOST_CHECK(mytree.insert(three)); BOOST_CHECK(mytree.insert(maxsigned)); BOOST_CHECK(mytree.insert(minsigned)); BOOST_CHECK(mytree.insert(minusone)); BOOST_CHECK(mytree.insert(minustwo)); // Delete them all - BOOST_CHECK(mytree.remove(0)); - BOOST_CHECK(mytree.remove(1)); - BOOST_CHECK(mytree.remove(2)); - BOOST_CHECK(mytree.remove(3)); - BOOST_CHECK(mytree.remove(maxsk)); - BOOST_CHECK(mytree.remove(minsk)); - BOOST_CHECK(mytree.remove(maxuk)); - BOOST_CHECK(mytree.remove(maxuk - 1)); - - BOOST_CHECK_EQUAL(mytree.get(0), NULLPTR(E)); - BOOST_CHECK_EQUAL(mytree.get(1), NULLPTR(E)); - BOOST_CHECK_EQUAL(mytree.get(2), NULLPTR(E)); - BOOST_CHECK_EQUAL(mytree.get(3), NULLPTR(E)); - BOOST_CHECK_EQUAL(mytree.get(maxsk), NULLPTR(E)); - BOOST_CHECK_EQUAL(mytree.get(minsk), NULLPTR(E)); - BOOST_CHECK_EQUAL(mytree.get(maxuk), NULLPTR(E)); - BOOST_CHECK_EQUAL(mytree.get(maxuk - 1), NULLPTR(E)); + BOOST_CHECK(mytree.remove(keyZero)); + BOOST_CHECK(mytree.remove(keyOne)); + BOOST_CHECK(mytree.remove(keyTwo)); + BOOST_CHECK(mytree.remove(keyThree)); + BOOST_CHECK(mytree.remove(E::SignedMax())); + BOOST_CHECK(mytree.remove(E::SignedMin())); + BOOST_CHECK(mytree.remove(E::MinusOne())); + BOOST_CHECK(mytree.remove(E::MinusTwo())); + + BOOST_CHECK_EQUAL(mytree.get(keyZero), NULLPTR(E)); + BOOST_CHECK_EQUAL(mytree.get(keyOne), NULLPTR(E)); + BOOST_CHECK_EQUAL(mytree.get(keyTwo), NULLPTR(E)); + BOOST_CHECK_EQUAL(mytree.get(keyThree), NULLPTR(E)); + BOOST_CHECK_EQUAL(mytree.get(E::SignedMax()), NULLPTR(E)); + BOOST_CHECK_EQUAL(mytree.get(E::SignedMin()), NULLPTR(E)); + BOOST_CHECK_EQUAL(mytree.get(E::MinusOne()), NULLPTR(E)); + BOOST_CHECK_EQUAL(mytree.get(E::MinusTwo()), NULLPTR(E)); } BOOST_AUTO_TEST_CASE(remove_test) { - testRemove(); - testRemove(); - testRemove(); - testRemove(); + testRemove>(); + testRemove>(); + testRemove>(); + testRemove>(); + + testRemove(); } BOOST_AUTO_TEST_CASE(const_element_test) { - using C = const TestElement; + using C = const TestElementInt; RadixTree mytree; BOOST_CHECK(mytree.insert(RCUPtr::make(0))); BOOST_CHECK(mytree.insert(RCUPtr::make(1))); BOOST_CHECK(mytree.insert(RCUPtr::make(2))); BOOST_CHECK(mytree.insert(RCUPtr::make(3))); BOOST_CHECK(!mytree.insert(RCUPtr::make(0))); BOOST_CHECK(!mytree.insert(RCUPtr::make(1))); BOOST_CHECK(!mytree.insert(RCUPtr::make(2))); BOOST_CHECK(!mytree.insert(RCUPtr::make(3))); BOOST_CHECK(mytree.get(0)); BOOST_CHECK(mytree.get(1)); BOOST_CHECK(mytree.get(2)); BOOST_CHECK(mytree.get(3)); BOOST_CHECK(mytree.remove(0)); BOOST_CHECK(mytree.remove(1)); BOOST_CHECK(mytree.remove(2)); BOOST_CHECK(mytree.remove(3)); BOOST_CHECK(!mytree.get(0)); BOOST_CHECK(!mytree.get(1)); BOOST_CHECK(!mytree.get(2)); BOOST_CHECK(!mytree.get(3)); BOOST_CHECK(!mytree.remove(0)); BOOST_CHECK(!mytree.remove(1)); BOOST_CHECK(!mytree.remove(2)); BOOST_CHECK(!mytree.remove(3)); } -void CheckConstTree(const RadixTree> &mytree, +void CheckConstTree(const RadixTree> &mytree, bool expected) { BOOST_CHECK_EQUAL(!!mytree.get(0), expected); BOOST_CHECK_EQUAL(!!mytree.get(1), expected); BOOST_CHECK_EQUAL(!!mytree.get(2), expected); BOOST_CHECK_EQUAL(!!mytree.get(3), expected); } BOOST_AUTO_TEST_CASE(const_tree_test) { - using E = TestElement; + using E = TestElementInt; RadixTree mytree; CheckConstTree(mytree, false); BOOST_CHECK(mytree.insert(RCUPtr::make(0))); BOOST_CHECK(mytree.insert(RCUPtr::make(1))); BOOST_CHECK(mytree.insert(RCUPtr::make(2))); BOOST_CHECK(mytree.insert(RCUPtr::make(3))); CheckConstTree(mytree, true); BOOST_CHECK(mytree.remove(0)); BOOST_CHECK(mytree.remove(1)); BOOST_CHECK(mytree.remove(2)); BOOST_CHECK(mytree.remove(3)); CheckConstTree(mytree, false); } BOOST_AUTO_TEST_CASE(test_cow) { - using E = TestElement; + using E = TestElementInt; RadixTree mytree; BOOST_CHECK(mytree.insert(RCUPtr::make(0))); BOOST_CHECK(mytree.insert(RCUPtr::make(1))); BOOST_CHECK(mytree.insert(RCUPtr::make(2))); BOOST_CHECK(mytree.insert(RCUPtr::make(3))); RadixTree copyTree = mytree; // The copy tree is identical. BOOST_CHECK(copyTree.get(0)); BOOST_CHECK(copyTree.get(1)); BOOST_CHECK(copyTree.get(2)); BOOST_CHECK(copyTree.get(3)); BOOST_CHECK(mytree.insert(RCUPtr::make(90))); BOOST_CHECK(mytree.insert(RCUPtr::make(91))); BOOST_CHECK(mytree.insert(RCUPtr::make(92))); BOOST_CHECK(mytree.insert(RCUPtr::make(93))); // The copy was unaffected. BOOST_CHECK(copyTree.get(0)); BOOST_CHECK(copyTree.get(1)); BOOST_CHECK(copyTree.get(2)); BOOST_CHECK(copyTree.get(3)); BOOST_CHECK(!copyTree.get(90)); BOOST_CHECK(!copyTree.get(91)); BOOST_CHECK(!copyTree.get(92)); BOOST_CHECK(!copyTree.get(93)); copyTree = mytree; BOOST_CHECK(mytree.remove(0)); BOOST_CHECK(mytree.remove(1)); BOOST_CHECK(mytree.remove(2)); BOOST_CHECK(mytree.remove(3)); // The copy was unaffected by the removals. BOOST_CHECK(copyTree.get(0)); BOOST_CHECK(copyTree.get(1)); BOOST_CHECK(copyTree.get(2)); BOOST_CHECK(copyTree.get(3)); BOOST_CHECK(copyTree.get(90)); BOOST_CHECK(copyTree.get(91)); BOOST_CHECK(copyTree.get(92)); BOOST_CHECK(copyTree.get(93)); } BOOST_AUTO_TEST_CASE(test_move) { - using E = TestElement; + using E = TestElementInt; RadixTree mytree; BOOST_CHECK(mytree.insert(RCUPtr::make(0))); BOOST_CHECK(mytree.insert(RCUPtr::make(1))); BOOST_CHECK(mytree.insert(RCUPtr::make(2))); BOOST_CHECK(mytree.insert(RCUPtr::make(3))); RadixTree movedTree = std::move(mytree); BOOST_CHECK(!mytree.remove(0)); BOOST_CHECK(!mytree.remove(1)); BOOST_CHECK(!mytree.remove(2)); BOOST_CHECK(!mytree.remove(3)); BOOST_CHECK(movedTree.remove(0)); BOOST_CHECK(movedTree.remove(1)); BOOST_CHECK(movedTree.remove(2)); BOOST_CHECK(movedTree.remove(3)); } #define THREADS 128 #define ELEMENTS 65536 BOOST_AUTO_TEST_CASE(insert_stress_test) { - using E = TestElement; + using E = TestElementInt; RadixTree mytree; std::atomic success{0}; std::vector threads; for (int i = 0; i < THREADS; i++) { threads.push_back(std::thread([&] { MMIXLinearCongruentialGenerator lcg; for (int j = 0; j < ELEMENTS; j++) { uint32_t v = lcg.next(); if (mytree.remove(v)) { success--; std::this_thread::yield(); } auto ptr = RCUPtr::make(v); if (mytree.insert(ptr)) { success++; std::this_thread::yield(); } if (mytree.remove(v)) { success--; std::this_thread::yield(); } if (mytree.insert(ptr)) { success++; std::this_thread::yield(); } } })); } // Wait for all the threads to finish. for (std::thread &t : threads) { t.join(); } BOOST_CHECK_EQUAL(success.load(), ELEMENTS); // All the elements have been inserted into the tree. MMIXLinearCongruentialGenerator lcg; for (int i = 0; i < ELEMENTS; i++) { uint32_t v = lcg.next(); BOOST_CHECK_EQUAL(mytree.get(v)->getId(), v); auto ptr = RCUPtr::make(v); BOOST_CHECK(!mytree.insert(ptr)); BOOST_CHECK(mytree.get(v) != ptr); } // Cleanup after ourselves. RCULock::synchronize(); } BOOST_AUTO_TEST_CASE(tree_traversal) { using E = TestElement; RadixTree mytree; // Build a vector of elements in ascending key order std::vector> elements; elements.reserve(ELEMENTS); for (uint32_t i = 0; i < ELEMENTS; i++) { auto ptr = RCUPtr::make(i); elements.push_back(std::move(ptr)); } // Insert in random order auto randomizedElements = elements; Shuffle(randomizedElements.begin(), randomizedElements.end(), FastRandomContext()); for (auto &element : randomizedElements) { BOOST_CHECK(mytree.insert(element)); } // Check the tree is traversed in ascending key order size_t count = 0; mytree.forEachLeaf([&](RCUPtr ptr) { // This test assumes the key is equal to the value BOOST_CHECK_EQUAL(ptr->getId(), count); BOOST_CHECK_EQUAL(ptr, elements[count++]); }); // All the elements are parsed BOOST_CHECK_EQUAL(count, ELEMENTS); } +BOOST_AUTO_TEST_CASE(uint256_key_wrapper) { + Uint256KeyWrapper key = uint256S( + "AA00000000000000000000000000000000000000000000000000000000000000"); + + auto checkEqual = [&](const Uint256KeyWrapper &val, + const uint256 &expected) { + BOOST_CHECK_EQUAL(ArithToUint256(val.base), expected); + }; + + auto checkOperands = [&](const Uint256KeyWrapper &val, + const uint256 &expected_uint256, + const size_t expected_size_t) { + checkEqual(val, expected_uint256); + + checkEqual(val & Uint256KeyWrapper(uint256::ZERO), uint256::ZERO); + checkEqual(val & val, expected_uint256); + checkEqual(val & TestElementUint256::MinusOne(), expected_uint256); + + BOOST_CHECK_EQUAL(size_t(val), expected_size_t); + }; + + // clang-format off + checkOperands(key >> 0u, uint256S("AA00000000000000000000000000000000000000000000000000000000000000"), 0x0000000000000000); + checkOperands(key >> 1u, uint256S("5500000000000000000000000000000000000000000000000000000000000000"), 0x0000000000000000); + checkOperands(key >> 2u, uint256S("2A80000000000000000000000000000000000000000000000000000000000000"), 0x0000000000000000); + checkOperands(key >> 3u, uint256S("1540000000000000000000000000000000000000000000000000000000000000"), 0x0000000000000000); + checkOperands(key >> 4u, uint256S("0AA0000000000000000000000000000000000000000000000000000000000000"), 0x0000000000000000); + checkOperands(key >> 8u, uint256S("00AA000000000000000000000000000000000000000000000000000000000000"), 0x0000000000000000); + checkOperands(key >> 16u, uint256S("0000AA0000000000000000000000000000000000000000000000000000000000"), 0x0000000000000000); + checkOperands(key >> 24u, uint256S("000000AA00000000000000000000000000000000000000000000000000000000"), 0x0000000000000000); + checkOperands(key >> 32u, uint256S("00000000AA000000000000000000000000000000000000000000000000000000"), 0x0000000000000000); + checkOperands(key >> 40u, uint256S("0000000000AA0000000000000000000000000000000000000000000000000000"), 0x0000000000000000); + checkOperands(key >> 48u, uint256S("000000000000AA00000000000000000000000000000000000000000000000000"), 0x0000000000000000); + checkOperands(key >> 56u, uint256S("00000000000000AA000000000000000000000000000000000000000000000000"), 0x0000000000000000); + checkOperands(key >> 64u, uint256S("0000000000000000AA0000000000000000000000000000000000000000000000"), 0x0000000000000000); + checkOperands(key >> 72u, uint256S("000000000000000000AA00000000000000000000000000000000000000000000"), 0x0000000000000000); + checkOperands(key >> 80u, uint256S("00000000000000000000AA000000000000000000000000000000000000000000"), 0x0000000000000000); + checkOperands(key >> 88u, uint256S("0000000000000000000000AA0000000000000000000000000000000000000000"), 0x0000000000000000); + checkOperands(key >> 96u, uint256S("000000000000000000000000AA00000000000000000000000000000000000000"), 0x0000000000000000); + checkOperands(key >> 104u, uint256S("00000000000000000000000000AA000000000000000000000000000000000000"), 0x0000000000000000); + checkOperands(key >> 112u, uint256S("0000000000000000000000000000AA0000000000000000000000000000000000"), 0x0000000000000000); + checkOperands(key >> 120u, uint256S("000000000000000000000000000000AA00000000000000000000000000000000"), 0x0000000000000000); + checkOperands(key >> 128u, uint256S("00000000000000000000000000000000AA000000000000000000000000000000"), 0x0000000000000000); + checkOperands(key >> 136u, uint256S("0000000000000000000000000000000000AA0000000000000000000000000000"), 0x0000000000000000); + checkOperands(key >> 144u, uint256S("000000000000000000000000000000000000AA00000000000000000000000000"), 0x0000000000000000); + checkOperands(key >> 152u, uint256S("00000000000000000000000000000000000000AA000000000000000000000000"), 0x0000000000000000); + checkOperands(key >> 160u, uint256S("0000000000000000000000000000000000000000AA0000000000000000000000"), 0x0000000000000000); + checkOperands(key >> 168u, uint256S("000000000000000000000000000000000000000000AA00000000000000000000"), 0x0000000000000000); + checkOperands(key >> 176u, uint256S("00000000000000000000000000000000000000000000AA000000000000000000"), 0x0000000000000000); + checkOperands(key >> 184u, uint256S("0000000000000000000000000000000000000000000000AA0000000000000000"), 0x0000000000000000); + checkOperands(key >> 185u, uint256S("0000000000000000000000000000000000000000000000550000000000000000"), 0x0000000000000000); + checkOperands(key >> 186u, uint256S("00000000000000000000000000000000000000000000002A8000000000000000"), 0x8000000000000000); + checkOperands(key >> 192u, uint256S("000000000000000000000000000000000000000000000000AA00000000000000"), 0xAA00000000000000); + checkOperands(key >> 200u, uint256S("00000000000000000000000000000000000000000000000000AA000000000000"), 0x00AA000000000000); + checkOperands(key >> 208u, uint256S("0000000000000000000000000000000000000000000000000000AA0000000000"), 0x0000AA0000000000); + checkOperands(key >> 216u, uint256S("000000000000000000000000000000000000000000000000000000AA00000000"), 0x000000AA00000000); + checkOperands(key >> 224u, uint256S("00000000000000000000000000000000000000000000000000000000AA000000"), 0x00000000AA000000); + checkOperands(key >> 232u, uint256S("0000000000000000000000000000000000000000000000000000000000AA0000"), 0x0000000000AA0000); + checkOperands(key >> 240u, uint256S("000000000000000000000000000000000000000000000000000000000000AA00"), 0x000000000000AA00); + checkOperands(key >> 248u, uint256S("00000000000000000000000000000000000000000000000000000000000000AA"), 0x00000000000000AA); + checkOperands(key >> 252u, uint256S("000000000000000000000000000000000000000000000000000000000000000A"), 0x000000000000000A); + checkOperands(key >> 253u, uint256S("0000000000000000000000000000000000000000000000000000000000000005"), 0x0000000000000005); + checkOperands(key >> 254u, uint256S("0000000000000000000000000000000000000000000000000000000000000002"), 0x0000000000000002); + checkOperands(key >> 255u, uint256S("0000000000000000000000000000000000000000000000000000000000000001"), 0x0000000000000001); + checkOperands(key >> 256u, uint256S("0000000000000000000000000000000000000000000000000000000000000000"), 0x0000000000000000); + // clang-format on +} + BOOST_AUTO_TEST_SUITE_END()