diff --git a/src/Makefile.am b/src/Makefile.am --- a/src/Makefile.am +++ b/src/Makefile.am @@ -158,6 +158,7 @@ policy/policy.h \ pow.h \ protocol.h \ + radix.h \ random.h \ rcu.h \ reverse_iterator.h \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -77,6 +77,7 @@ test/policyestimator_tests.cpp \ test/pow_tests.cpp \ test/prevector_tests.cpp \ + test/radix_tests.cpp \ test/raii_event_tests.cpp \ test/random_tests.cpp \ test/rcu_tests.cpp \ diff --git a/src/radix.h b/src/radix.h new file mode 100644 --- /dev/null +++ b/src/radix.h @@ -0,0 +1,206 @@ +// Copyright (c) 2019 The Bitcoin developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_RADIX_H +#define BITCOIN_RADIX_H + +#include +#include + +#include + +#include +#include +#include +#include + +/** + * This is a radix tree storing values identified by a unique key. + * + * The tree is composed of nodes (RadixNode) containing an array of + * RadixElement. The key is split in chunk of a few bits that serve as index + * into that array. RadixElement is a discriminated union of either a RadixNode* + * representing the next level in the tree, or a T* representing a leaf. New + * RadixNode are added lazyly when two leaf would go in the same slot. + * + * Reads walk the tree using sequential atomic loads, and insertions are done + * using CAS, which ensures both can be executed lock free. Removing any element + * from the tree can also be done using CAS, but require to wait for other + * readers before being destroyed. The tree uses RCU to track which thread is + * reading the tree, which allows deletion to wait for other readers to be up to + * speed before destroying anything. It is therefore crucial that the lock be + * taken beofre reading anything in the tree. + * + * It is not possible to delete anything from the tree at this time. The tree + * itself cannot be destroyed and will leak memory instead of cleaning up after + * itself. This obviously needs to be fixed in subsequent revisions. + */ +template struct RadixTree : public boost::noncopyable { +private: + static const int BITS = 4; + static const int MASK = (1 << BITS) - 1; + static const size_t CHILD_PER_LEVEL = 1 << BITS; + + typedef decltype(std::declval().getId()) K; + static const size_t KEY_BITS = 8 * sizeof(K); + static const uint32_t TOP_LEVEL = (KEY_BITS - 1) / BITS; + + struct RadixElement; + struct RadixNode; + + std::atomic root; + +public: + RadixTree() : root(RadixElement()) {} + + /** + * Insert a value into the tree. + * Returns true is the value was inserted, false if it was already present. + */ + bool insert(T *value) { return insert(value->getId(), value); } + + /** + * Get the value corresponding to a key. + * Returns the value if found, null if not. + */ + T *get(const K &key) const { + uint32_t level = TOP_LEVEL; + + RCULock lock; + RadixElement e = root.load(); + + // Find a leaf. + while (e.isNode()) { + e = e.getNode()->get(level--, key)->load(); + } + + T *leaf = e.getLeaf(); + if (leaf == nullptr || leaf->getId() != key) { + // We failed to find the proper element. + return nullptr; + } + + // The leaf is non-null and the key match. We have our guy. + return leaf; + } + +private: + bool insert(const K &key, T *value) { + uint32_t level = TOP_LEVEL; + + RCULock lock; + std::atomic *eptr = &root; + + while (true) { + RadixElement e = eptr->load(); + + // Walk down the tree untill we find a leaf for our node. + while (e.isNode()) { + Node: + eptr = e.getNode()->get(level--, key); + e = eptr->load(); + } + + // If the slot is empty, try to insert right there. + while (e.getLeaf() == nullptr) { + if (eptr->compare_exchange_weak(e, RadixElement(value))) { + return true; + } + + // CAS failed, we may have a node in there now. + if (e.isNode()) { + goto Node; + } + } + + // The element already was in the tree. + const K &leafKey = e.getLeaf()->getId(); + if (key == leafKey) { + return false; + } + + // There is an element there, but it isn't a subtree. We need to + // convert it into a subtree and resume inertion in that subtree. + std::unique_ptr newChild = + MakeUnique(level, leafKey, e); + if (eptr->compare_exchange_weak(e, RadixElement(newChild.get()))) { + // We have a subtree, resume normal operations from there. + newChild.release(); + } + } + } + + struct RadixElement { + private: + union { + RadixNode *node; + T *leaf; + uintptr_t raw; + }; + + static const uintptr_t DISCRIMINANT = 0x01; + bool getDiscriminent() const { return (raw & DISCRIMINANT) != 0; } + + public: + explicit RadixElement() : raw(DISCRIMINANT) {} + explicit RadixElement(RadixNode *nodeIn) : node(nodeIn) {} + explicit RadixElement(T *leafIn) : leaf(leafIn) { raw |= DISCRIMINANT; } + + /** + * Node features. + */ + bool isNode() const { return !getDiscriminent(); } + + RadixNode *getNode() { + assert(isNode()); + return node; + } + + const RadixNode *getNode() const { + assert(isNode()); + return node; + } + + /** + * Leaf features. + */ + bool isLeaf() const { return getDiscriminent(); } + + T *getLeaf() { + assert(isLeaf()); + return reinterpret_cast(raw & ~DISCRIMINANT); + } + + const T *getLeaf() const { + assert(isLeaf()); + return const_cast(this)->getLeaf(); + } + }; + + struct RadixNode { + private: + union { + std::array, CHILD_PER_LEVEL> childs; + std::array + non_atomic_childs_DO_NOT_USE; + }; + + public: + RadixNode(uint32_t level, const K &key, RadixElement e) + : non_atomic_childs_DO_NOT_USE() { + get(level, key)->store(e); + } + + std::atomic *get(uint32_t level, const K &key) { + return &childs[(key >> (level * BITS)) & MASK]; + } + }; + + // Make sure the alignment work for RadixElement. + static_assert(alignof(T) > 1, "T's alignment must be 2 or more."); + static_assert(alignof(RadixNode) > 1, + "RadixNode alignment must be 2 or more."); +}; + +#endif // BITCOIN_RADIX_H diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -94,6 +94,7 @@ policyestimator_tests.cpp pow_tests.cpp prevector_tests.cpp + radix_tests.cpp raii_event_tests.cpp random_tests.cpp rcu_tests.cpp diff --git a/src/test/radix_tests.cpp b/src/test/radix_tests.cpp new file mode 100644 --- /dev/null +++ b/src/test/radix_tests.cpp @@ -0,0 +1,211 @@ +// Copyright (c) 2019 The Bitcoin developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include + +#include + +#include + +#include +#include + +BOOST_FIXTURE_TEST_SUITE(radix_tests, BasicTestingSetup) + +template struct TestElement { + K key; + + TestElement(K keyIn) : key(keyIn) {} + const K &getId() const { return key; } +}; + +template void testInsert() { + typedef TestElement E; + RadixTree mytree; + + E zero(0); + E one(1); + E two(2); + E three(3); + + // Inserting a new element in the tree returns true. + BOOST_CHECK(mytree.insert(&one)); + + // Inserting an element already in the tree returns false. + BOOST_CHECK(!mytree.insert(&one)); + + // Let's insert more elements and check behavior stays consistent. + BOOST_CHECK(mytree.insert(&zero)); + BOOST_CHECK(!mytree.insert(&one)); + BOOST_CHECK(mytree.insert(&two)); + BOOST_CHECK(mytree.insert(&three)); + BOOST_CHECK(!mytree.insert(&zero)); + BOOST_CHECK(!mytree.insert(&one)); + BOOST_CHECK(!mytree.insert(&two)); + BOOST_CHECK(!mytree.insert(&three)); + + // Check extreme values. + typedef typename std::make_signed::type SK; + E maxsigned(std::numeric_limits::max()); + E minsigned(std::numeric_limits::min()); + typedef typename std::make_unsigned::type UK; + E minusone(std::numeric_limits::max()); + E minustwo(std::numeric_limits::max() - 1); + + // They can a be inserted. + BOOST_CHECK(mytree.insert(&maxsigned)); + BOOST_CHECK(mytree.insert(&minsigned)); + BOOST_CHECK(mytree.insert(&minusone)); + BOOST_CHECK(mytree.insert(&minustwo)); + + // All elements are already in the tree. + BOOST_CHECK(!mytree.insert(&zero)); + BOOST_CHECK(!mytree.insert(&one)); + BOOST_CHECK(!mytree.insert(&two)); + BOOST_CHECK(!mytree.insert(&three)); + BOOST_CHECK(!mytree.insert(&maxsigned)); + BOOST_CHECK(!mytree.insert(&minsigned)); + BOOST_CHECK(!mytree.insert(&minusone)); + BOOST_CHECK(!mytree.insert(&minustwo)); +} + +BOOST_AUTO_TEST_CASE(insert_test) { + testInsert(); + testInsert(); + testInsert(); + testInsert(); +} + +template void testGet() { + typedef TestElement E; + RadixTree mytree; + + E zero(0); + E one(1); + E two(2); + E three(3); + + // There are no elements in the tree so far. + BOOST_CHECK_EQUAL(mytree.get(1), nullptr); + + // Insert an element in the tree and check it is there. + BOOST_CHECK(mytree.insert(&one)); + BOOST_CHECK_EQUAL(mytree.get(1), &one); + + // Let's insert more element and check they are recovered properly. + BOOST_CHECK_EQUAL(mytree.get(0), nullptr); + BOOST_CHECK(mytree.insert(&zero)); + BOOST_CHECK_EQUAL(mytree.get(0), &zero); + BOOST_CHECK_EQUAL(mytree.get(1), &one); + + // More elements. + BOOST_CHECK_EQUAL(mytree.get(2), nullptr); + BOOST_CHECK_EQUAL(mytree.get(3), nullptr); + BOOST_CHECK(mytree.insert(&two)); + BOOST_CHECK(mytree.insert(&three)); + + BOOST_CHECK_EQUAL(mytree.get(0), &zero); + BOOST_CHECK_EQUAL(mytree.get(1), &one); + BOOST_CHECK_EQUAL(mytree.get(2), &two); + BOOST_CHECK_EQUAL(mytree.get(3), &three); + + // Check extreme values. + typedef typename std::make_signed::type SK; + K maxsk = std::numeric_limits::max(); + K minsk = std::numeric_limits::min(); + typedef typename std::make_unsigned::type UK; + K maxuk = std::numeric_limits::max(); + + E maxsigned(maxsk); + E minsigned(minsk); + E minusone(maxuk); + E minustwo(maxuk - 1); + + // Check that they are not in the tree. + BOOST_CHECK_EQUAL(mytree.get(maxsk), nullptr); + BOOST_CHECK_EQUAL(mytree.get(minsk), nullptr); + BOOST_CHECK_EQUAL(mytree.get(maxuk), nullptr); + BOOST_CHECK_EQUAL(mytree.get(maxuk - 1), nullptr); + + // Insert in the tree. + BOOST_CHECK(mytree.insert(&maxsigned)); + BOOST_CHECK(mytree.insert(&minsigned)); + BOOST_CHECK(mytree.insert(&minusone)); + BOOST_CHECK(mytree.insert(&minustwo)); + + // And now they all are in the tree. + BOOST_CHECK_EQUAL(mytree.get(0), &zero); + BOOST_CHECK_EQUAL(mytree.get(1), &one); + BOOST_CHECK_EQUAL(mytree.get(2), &two); + BOOST_CHECK_EQUAL(mytree.get(3), &three); + BOOST_CHECK_EQUAL(mytree.get(maxsk), &maxsigned); + BOOST_CHECK_EQUAL(mytree.get(minsk), &minsigned); + BOOST_CHECK_EQUAL(mytree.get(maxuk), &minusone); + BOOST_CHECK_EQUAL(mytree.get(maxuk - 1), &minustwo); +} + +BOOST_AUTO_TEST_CASE(get_test) { + testGet(); + testGet(); + testGet(); + testGet(); +} + +#define THREADS 128 +#define ELEMENTS 65536 + +static uint64_t next(uint64_t x) { + // Simple linear congruential generator by Donald Knuth. + return x * 6364136223846793005 + 1442695040888963407; +} + +BOOST_AUTO_TEST_CASE(insert_stress_test) { + typedef TestElement E; + + RadixTree mytree; + std::atomic success{0}; + std::vector threads; + + for (int i = 0; i < THREADS; i++) { + threads.push_back(std::thread([&] { + uint64_t rand = 0; + for (int j = 0; j < ELEMENTS; j++) { + rand = next(rand); + uint32_t v(rand >> 32); + std::unique_ptr ptr = MakeUnique(v); + if (mytree.insert(ptr.get())) { + success++; + ptr.release(); + std::this_thread::yield(); + } + + // Regardless if we inserted or someone else did, the element + // should now be in the tree. For some reason, boost is spewing + // an inordinate amount of log when performing checks in others + // threads than the main one, so we'll use asserts. + assert(mytree.get(v)->getId() == v); + } + })); + } + + // Wait for all the threads to finish. + for (std::thread &t : threads) { + t.join(); + } + + BOOST_CHECK_EQUAL(success.load(), ELEMENTS); + + // All the element have been inserted in the tree. + uint64_t rand = 0; + for (int i = 0; i < ELEMENTS; i++) { + rand = next(rand); + uint32_t v(rand >> 32); + BOOST_CHECK_EQUAL(mytree.get(v)->getId(), v); + E e(v); + BOOST_CHECK(!mytree.insert(&e)); + BOOST_CHECK(mytree.get(v) != &e); + } +} + +BOOST_AUTO_TEST_SUITE_END()