diff --git a/Cargo.lock b/Cargo.lock --- a/Cargo.lock +++ b/Cargo.lock @@ -164,9 +164,9 @@ [[package]] name = "bindgen" -version = "0.60.1" +version = "0.64.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "062dddbc1ba4aca46de6338e2bf87771414c335f7b2f2036e8f3e9befebf88e6" +checksum = "c4243e6031260db77ede97ad86c27e501d646a27ab57b59a574f725d98ab1fb4" dependencies = [ "bitflags", "cexpr", @@ -179,6 +179,7 @@ "regex", "rustc-hash", "shlex", + "syn", ] [[package]] @@ -289,6 +290,7 @@ "abc-rust-lint", "async-trait", "axum", + "chronik-indexer", "chronik-util", "futures", "hyper", @@ -318,8 +320,11 @@ dependencies = [ "abc-rust-error", "abc-rust-lint", + "bitcoinsuite-core", "chronik-bridge", + "chronik-db", "chronik-http", + "chronik-indexer", "chronik-util", "cxx", "cxx-build", @@ -838,9 +843,8 @@ [[package]] name = "librocksdb-sys" -version = "0.8.0+7.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "611804e4666a25136fcc5f8cf425ab4d26c7f74ea245ffe92ea23b85b6420b5d" +version = "0.10.0+7.9.2" +source = "git+https://github.com/rust-rocksdb/rust-rocksdb.git?rev=a6103ef#a6103ef311c96e14ee19b36832a373a0954f946b" dependencies = [ "bindgen", "bzip2-sys", @@ -1100,9 +1104,9 @@ [[package]] name = "proc-macro2" -version = "1.0.40" +version = "1.0.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd96a1e8ed2596c337f8eae5f24924ec83f5ad5ab21ea8e455d3566c69fbcaf7" +checksum = "5d727cae5b39d21da60fa540906919ad737832fe0b1c165da3a34d6548c849d6" dependencies = [ "unicode-ident", ] @@ -1265,9 +1269,8 @@ [[package]] name = "rocksdb" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e9562ea1d70c0cc63a34a22d977753b50cca91cc6b6527750463bd5dd8697bc" +version = "0.20.1" +source = "git+https://github.com/rust-rocksdb/rust-rocksdb.git?rev=a6103ef#a6103ef311c96e14ee19b36832a373a0954f946b" dependencies = [ "libc", "librocksdb-sys", @@ -1467,9 +1470,9 @@ [[package]] name = "syn" -version = "1.0.98" +version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c50aef8a904de4c23c788f104b7dddc7d6f79c647c7c8ce4cc8f73eb0ca773dd" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ "proc-macro2", "quote", diff --git a/chronik/CMakeLists.txt b/chronik/CMakeLists.txt --- a/chronik/CMakeLists.txt +++ b/chronik/CMakeLists.txt @@ -125,10 +125,21 @@ chronik-lib-static ) -# mio crate (dependency of tokio) requires stuff from winternl.h, found in ntdll if(${CMAKE_SYSTEM_NAME} MATCHES "Windows") + # mio crate (dependency of tokio) requires winternl.h, found in ntdll find_package(NTDLL REQUIRED) target_link_libraries(chronik NTDLL::ntdll) + + # rocksdb requires items from rpcdce.h, found in rpcrt4 + find_package(RPCRT4 REQUIRED) + target_link_libraries(chronik RPCRT4::rpcrt4) +endif() + +# Rocksdb requires "atomic" +include(AddCompilerFlags) +custom_check_linker_flag(LINKER_HAS_ATOMIC, "-latomic") +if(LINKER_HAS_ATOMIC) + target_link_libraries(chronik atomic) endif() # Add chronik to server diff --git a/chronik/chronik-cpp/chronik.cpp b/chronik/chronik-cpp/chronik.cpp --- a/chronik/chronik-cpp/chronik.cpp +++ b/chronik/chronik-cpp/chronik.cpp @@ -24,6 +24,7 @@ bool Start([[maybe_unused]] const Config &config, [[maybe_unused]] const node::NodeContext &node) { return chronik_bridge::setup_chronik({ + .datadir_net = gArgs.GetDataDirNet().u8string(), .hosts = ToRustVec<rust::String>(gArgs.IsArgSet("-chronikbind") ? gArgs.GetArgs("-chronikbind") : DEFAULT_BINDS), diff --git a/chronik/chronik-cpp/chronik_validationinterface.cpp b/chronik/chronik-cpp/chronik_validationinterface.cpp --- a/chronik/chronik-cpp/chronik_validationinterface.cpp +++ b/chronik/chronik-cpp/chronik_validationinterface.cpp @@ -2,11 +2,30 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <blockindex.h> +#include <chronik-cpp/util/hash.h> #include <chronik-lib/src/ffi.rs.h> +#include <primitives/block.h> #include <validationinterface.h> namespace chronik { +chronik_bridge::Block BridgeBlock(const CBlock &block, + const CBlockIndex *pindex) { + const CBlockHeader header = pindex->GetBlockHeader(); + return {.hash = chronik::util::HashToArray(header.GetHash()), + .prev_hash = chronik::util::HashToArray(header.hashPrevBlock), + .n_bits = header.nBits, + .timestamp = header.GetBlockTime(), + .height = pindex->nHeight, + .file_num = uint32_t(pindex->nFile), + .data_pos = pindex->nDataPos, + .undo_pos = pindex->nUndoPos}; +} + +/** + * CValidationInterface connecting bitcoind events to Chronik + */ class ChronikValidationInterface final : public CValidationInterface { public: ChronikValidationInterface(rust::Box<chronik_bridge::Chronik> chronik_box) @@ -32,12 +51,12 @@ void BlockConnected(const std::shared_ptr<const CBlock> &block, const CBlockIndex *pindex) override { - m_chronik->handle_block_connected(); + m_chronik->handle_block_connected(BridgeBlock(*block, pindex)); } void BlockDisconnected(const std::shared_ptr<const CBlock> &block, const CBlockIndex *pindex) override { - m_chronik->handle_block_disconnected(); + m_chronik->handle_block_disconnected(BridgeBlock(*block, pindex)); } }; diff --git a/chronik/chronik-db/Cargo.toml b/chronik/chronik-db/Cargo.toml --- a/chronik/chronik-db/Cargo.toml +++ b/chronik/chronik-db/Cargo.toml @@ -19,7 +19,7 @@ postcard = { version = "1.0", features = ["alloc"] } # Key-value database -rocksdb = { version = "0.19", default-features = false } +rocksdb = { git = "https://github.com/rust-rocksdb/rust-rocksdb.git", rev = "a6103ef", default-features = false } # Serialize structs serde = { version = "1.0", features = ["derive"] } diff --git a/chronik/chronik-http/Cargo.toml b/chronik/chronik-http/Cargo.toml --- a/chronik/chronik-http/Cargo.toml +++ b/chronik/chronik-http/Cargo.toml @@ -10,6 +10,7 @@ [dependencies] abc-rust-error = { path = "../abc-rust-error" } abc-rust-lint = { path = "../abc-rust-lint" } +chronik-indexer = { path = "../chronik-indexer" } chronik-util = { path = "../chronik-util" } # Allow traits to use async functions diff --git a/chronik/chronik-http/src/error.rs b/chronik/chronik-http/src/error.rs --- a/chronik/chronik-http/src/error.rs +++ b/chronik/chronik-http/src/error.rs @@ -9,7 +9,7 @@ use chronik_util::{log, log_chronik}; use hyper::StatusCode; -use crate::{proto, protobuf::Protobuf}; +use crate::{proto, protobuf::Protobuf, server::ChronikServerError}; /// Wrapper around [`Report`] which can be converted into a [`Response`]. #[derive(Debug)] @@ -21,6 +21,12 @@ } } +impl From<ChronikServerError> for ReportError { + fn from(err: ChronikServerError) -> Self { + ReportError(err.into()) + } +} + impl IntoResponse for ReportError { fn into_response(self) -> Response { let ReportError(report) = self; diff --git a/chronik/chronik-http/src/server.rs b/chronik/chronik-http/src/server.rs --- a/chronik/chronik-http/src/server.rs +++ b/chronik/chronik-http/src/server.rs @@ -4,12 +4,18 @@ //! Module for [`ChronikServer`]. -use std::net::SocketAddr; +use std::{net::SocketAddr, sync::Arc}; use abc_rust_error::Result; -use axum::Router; +use axum::{extract::Path, routing, Extension, Router}; +use chronik_indexer::indexer::ChronikIndexer; use hyper::server::conn::AddrIncoming; use thiserror::Error; +use tokio::sync::RwLock; + +use crate::{error::ReportError, proto, protobuf::Protobuf}; + +type ChronikIndexerRef = Arc<RwLock<ChronikIndexer>>; use crate::handlers::handle_not_found; @@ -18,6 +24,8 @@ pub struct ChronikServerParams { /// Host address (port + IP) where to serve Chronik at. pub hosts: Vec<SocketAddr>, + /// Indexer to read data from + pub indexer: ChronikIndexerRef, } /// Chronik HTTP server, holding all the data/handles required to serve an @@ -25,6 +33,7 @@ #[derive(Debug)] pub struct ChronikServer { server_builders: Vec<hyper::server::Builder<AddrIncoming>>, + indexer: ChronikIndexerRef, } /// Errors for [`ChronikServer`]. @@ -37,6 +46,10 @@ /// Serving Chronik failed #[error("Chronik failed serving: {0}")] ServingFailed(String), + + /// Block not found in DB + #[error("404: Block not found: {0}")] + BlockNotFound(String), } use self::ChronikServerError::*; @@ -53,12 +66,15 @@ }) }) .collect::<Result<Vec<_>>>()?; - Ok(ChronikServer { server_builders }) + Ok(ChronikServer { + server_builders, + indexer: params.indexer, + }) } /// Serve a Chronik HTTP endpoint with the given parameters. pub async fn serve(self) -> Result<()> { - let app = Self::make_router(); + let app = Self::make_router(self.indexer); let servers = self .server_builders .into_iter() @@ -76,7 +92,30 @@ Ok(()) } - fn make_router() -> Router { - Router::new().fallback(handle_not_found) + fn make_router(indexer: ChronikIndexerRef) -> Router { + Router::new() + .route("/block/:height", routing::get(handle_block)) + .fallback(handle_not_found) + .layer(Extension(indexer)) } } + +async fn handle_block( + Path(height): Path<i32>, + Extension(indexer): Extension<ChronikIndexerRef>, +) -> Result<Protobuf<proto::Block>, ReportError> { + let indexer = indexer.read().await; + let blocks = indexer.blocks()?; + let db_block = blocks + .by_height(height)? + .ok_or_else(|| BlockNotFound(height.to_string()))?; + Ok(Protobuf(proto::Block { + block_info: Some(proto::BlockInfo { + hash: db_block.hash.to_vec(), + prev_hash: db_block.prev_hash.to_vec(), + height: db_block.height, + n_bits: db_block.n_bits, + timestamp: db_block.timestamp, + }), + })) +} diff --git a/chronik/chronik-indexer/src/indexer.rs b/chronik/chronik-indexer/src/indexer.rs --- a/chronik/chronik-indexer/src/indexer.rs +++ b/chronik/chronik-indexer/src/indexer.rs @@ -9,7 +9,7 @@ use abc_rust_error::{Result, WrapErr}; use chronik_db::{ db::{Db, WriteBatch}, - io::{BlockWriter, DbBlock}, + io::{BlockReader, BlockWriter, DbBlock}, }; use chronik_util::log_chronik; use thiserror::Error; @@ -82,6 +82,11 @@ self.db.write_batch(batch)?; Ok(()) } + + /// Return [`BlockReader`] to read blocks from the DB. + pub fn blocks(&self) -> Result<BlockReader<'_>> { + BlockReader::new(&self.db) + } } #[cfg(test)] diff --git a/chronik/chronik-lib/Cargo.toml b/chronik/chronik-lib/Cargo.toml --- a/chronik/chronik-lib/Cargo.toml +++ b/chronik/chronik-lib/Cargo.toml @@ -15,8 +15,12 @@ abc-rust-lint = { path = "../abc-rust-lint" } abc-rust-error = { path = "../abc-rust-error" } +bitcoinsuite-core = { path = "../bitcoinsuite-core" } + chronik-bridge = { path = "../chronik-bridge" } +chronik-db = { path = "../chronik-db" } chronik-http = { path = "../chronik-http" } +chronik-indexer = { path = "../chronik-indexer" } chronik-util = { path = "../chronik-util" } # Bridge to C++ diff --git a/chronik/chronik-lib/src/bridge.rs b/chronik/chronik-lib/src/bridge.rs --- a/chronik/chronik-lib/src/bridge.rs +++ b/chronik/chronik-lib/src/bridge.rs @@ -4,13 +4,22 @@ //! Rust side of the bridge; these structs and functions are exposed to C++. -use std::net::{AddrParseError, IpAddr, SocketAddr}; +use std::{ + net::{AddrParseError, IpAddr, SocketAddr}, + sync::Arc, +}; use abc_rust_error::Result; +use bitcoinsuite_core::block::BlockHash; use chronik_bridge::ffi::init_error; +use chronik_db::io::DbBlock; use chronik_http::server::{ChronikServer, ChronikServerParams}; +use chronik_indexer::indexer::{ + ChronikBlock, ChronikIndexer, ChronikIndexerParams, +}; use chronik_util::{log, log_chronik}; use thiserror::Error; +use tokio::sync::RwLock; use crate::{ error::ok_or_abort_node, @@ -46,17 +55,27 @@ .map(|host| parse_socket_addr(host, params.default_port)) .collect::<Result<Vec<_>>>()?; log!("Starting Chronik bound to {:?}\n", hosts); + let indexer = ChronikIndexer::setup(ChronikIndexerParams { + datadir_net: params.datadir_net.into(), + })?; + let indexer = Arc::new(RwLock::new(indexer)); let runtime = tokio::runtime::Builder::new_multi_thread() .enable_all() .build()?; - let server = runtime.block_on(async move { - // try_bind requires a Runtime - ChronikServer::setup(ChronikServerParams { hosts }) + let server = runtime.block_on({ + let indexer = Arc::clone(&indexer); + async move { + // try_bind requires a Runtime + ChronikServer::setup(ChronikServerParams { hosts, indexer }) + } })?; runtime.spawn(async move { ok_or_abort_node("ChronikServer::serve", server.serve().await); }); - let chronik = Box::new(Chronik { _runtime: runtime }); + let chronik = Box::new(Chronik { + indexer, + _runtime: runtime, + }); StartChronikValidationInterface(chronik); Ok(()) } @@ -76,6 +95,7 @@ /// cleanly. #[derive(Debug)] pub struct Chronik { + indexer: Arc<RwLock<ChronikIndexer>>, // Having this here ensures HTTP server, outstanding requests etc. will get // stopped when `Chronik` is dropped. _runtime: tokio::runtime::Runtime, @@ -93,12 +113,35 @@ } /// Block connected to the longest chain - pub fn handle_block_connected(&self) { + pub fn handle_block_connected(&self, block: ffi::Block) { + let mut indexer = self.indexer.blocking_write(); + ok_or_abort_node( + "handle_block_connected", + indexer.handle_block_connected(make_chronik_block(block)), + ); log_chronik!("Chronik: block connected\n"); } /// Block disconnected from the longest chain - pub fn handle_block_disconnected(&self) { + pub fn handle_block_disconnected(&self, block: ffi::Block) { + let mut indexer = self.indexer.blocking_write(); + ok_or_abort_node( + "handle_block_disconnected", + indexer.handle_block_disconnected(make_chronik_block(block)), + ); log_chronik!("Chronik: block disconnected\n"); } } + +fn make_chronik_block(block: ffi::Block) -> ChronikBlock { + let db_block = DbBlock { + hash: BlockHash::from(block.hash), + prev_hash: BlockHash::from(block.prev_hash), + height: block.height, + n_bits: block.n_bits, + timestamp: block.timestamp, + file_num: block.file_num, + data_pos: block.data_pos, + }; + ChronikBlock { db_block } +} diff --git a/chronik/chronik-lib/src/ffi.rs b/chronik/chronik-lib/src/ffi.rs --- a/chronik/chronik-lib/src/ffi.rs +++ b/chronik/chronik-lib/src/ffi.rs @@ -13,20 +13,58 @@ /// Params for setting up Chronik #[derive(Debug)] pub struct SetupParams { + /// Where the data of the blockchain is stored, dependent on network + /// (mainnet, testnet, regtest) + pub datadir_net: String, /// Host addresses where the Chronik HTTP endpoint will be served pub hosts: Vec<String>, /// Default port for `hosts` if only an IP address is given pub default_port: u16, } + /// Block coming from bitcoind to Chronik. + /// + /// We don't index all fields (e.g. hashMerkleRoot), only those that are + /// needed when querying a range of blocks. + /// + /// Instead of storing all the block data for Chronik again, we only store + /// file_num, data_pos and undo_pos of the block data of the node. + /// + /// This makes the index relatively small, as it's mostly just pointing to + /// the data the node already stores. + /// + /// Note that this prohibits us from using Chronik in pruned mode. + #[derive(Debug, Clone, PartialEq, Eq)] + pub struct Block { + /// Block hash + pub hash: [u8; 32], + /// hashPrevBlock, hash of the previous block in the chain + pub prev_hash: [u8; 32], + /// nBits, difficulty of the header + pub n_bits: u32, + /// Timestamp of the block + pub timestamp: i64, + /// Height of the block in the chain. + pub height: i32, + /// File number of the block file this block is stored in. + /// This can be used to later slice out transactions, so we don't have + /// to index txs twice. + pub file_num: u32, + /// Position of the block within the block file, starting at the block + /// header. + pub data_pos: u32, + /// Position of the undo data within the undo file. + pub undo_pos: u32, + } + extern "Rust" { type Chronik; fn setup_chronik(params: SetupParams) -> bool; fn handle_tx_added_to_mempool(&self); fn handle_tx_removed_from_mempool(&self); - fn handle_block_connected(&self); - fn handle_block_disconnected(&self); + fn handle_block_connected(&self, block: Block); + fn handle_block_disconnected(&self, block: Block); } unsafe extern "C++" { diff --git a/chronik/proto/chronik.proto b/chronik/proto/chronik.proto --- a/chronik/proto/chronik.proto +++ b/chronik/proto/chronik.proto @@ -6,6 +6,26 @@ package chronik; +// Block on the blockchain +message Block { + // Info about the block + BlockInfo block_info = 1; +} + +// Info about a block +message BlockInfo { + // Hash (little-endian) + bytes hash = 1; + // Hash of the previous block (little-endian) + bytes prev_hash = 2; + // Height in the chain + int32 height = 3; + // nBits field encoding the target + uint32 n_bits = 4; + // Timestamp field of the block + int64 timestamp = 5; +} + // Error message returned from our APIs. message Error { // 2, as legacy chronik uses this for the message so we're still compatible. diff --git a/cmake/modules/FindRPCRT4.cmake b/cmake/modules/FindRPCRT4.cmake new file mode 100644 --- /dev/null +++ b/cmake/modules/FindRPCRT4.cmake @@ -0,0 +1,45 @@ +# Copyright (c) 2023 The Bitcoin developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +# .rst: +# FindRPCRT4 +# -------------- +# +# Find the RPCRT4 library. The following conponents are available:: +# rpcrt4 +# +# This will define the following variables:: +# +# RPCRT4_FOUND - True if the RPCRT4 library is found. +# RPCRT4_INCLUDE_DIRS - List of the header include directories. +# RPCRT4_LIBRARIES - List of the libraries. +# +# And the following imported targets:: +# +# RPCRT4::rpcrt4 + +find_path(RPCDCE_INCLUDE_DIR + NAMES rpcdce.h +) + +list(APPEND RPCRT4_INCLUDE_DIRS + "${RPCDCE_INCLUDE_DIR}" +) + +mark_as_advanced(RPCRT4_INCLUDE_DIRS) + +if(RPCDCE_INCLUDE_DIR) + include(ExternalLibraryHelper) + find_component(RPCRT4 rpcrt4 + NAMES rpcrt4 + INCLUDE_DIRS ${RPCRT4_INCLUDE_DIRS} + ) +endif() + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(RPCRT4 + REQUIRED_VARS + RPCDCE_INCLUDE_DIR + HANDLE_COMPONENTS +) diff --git a/test/functional/chronik_block.py b/test/functional/chronik_block.py new file mode 100644 --- /dev/null +++ b/test/functional/chronik_block.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +# Copyright (c) 2023 The Bitcoin developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +import http.client + +from test_framework.address import ADDRESS_ECREG_P2SH_OP_TRUE +from test_framework.blocktools import GENESIS_BLOCK_HASH, TIME_GENESIS_BLOCK +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal + + +class ChronikServeTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 1 + self.extra_args = [['-chronik']] + + def skip_test_if_missing_module(self): + self.skip_if_no_chronik() + + def run_test(self): + import chronik_pb2 as pb + + def query_block(block_height): + client = http.client.HTTPConnection('127.0.0.1', 18442, timeout=4) + client.request('GET', f'/block/{block_height}') + response = client.getresponse() + assert_equal(response.getheader('Content-Type'), + 'application/x-protobuf') + return response + + # Expected genesis block + genesis_block = pb.Block( + block_info=pb.BlockInfo( + hash=bytes.fromhex(GENESIS_BLOCK_HASH)[::-1], + prev_hash=bytes(32), + n_bits=0x207fffff, + timestamp=TIME_GENESIS_BLOCK, + ), + ) + + # Query genesis block + response = query_block(0) + assert_equal(response.status, 200) + proto_block = pb.Block() + proto_block.ParseFromString(response.read()) + assert_equal(proto_block, genesis_block) + + # Block 1 not found + response = query_block(1) + assert_equal(response.status, 404) + proto_error = pb.Error() + proto_error.ParseFromString(response.read()) + assert_equal(proto_error.msg, '404: Block not found: 1') + + # Generate 100 blocks, verify they form a chain + node = self.nodes[0] + self.generatetoaddress(node, 100, ADDRESS_ECREG_P2SH_OP_TRUE) + block_hashes = [genesis_block.block_info.hash] + for i in range(1, 101): + response = query_block(i) + assert_equal(response.status, 200) + proto_block = pb.Block() + proto_block.ParseFromString(response.read()) + assert_equal(proto_block.block_info.prev_hash, block_hashes[-1]) + block_hashes.append(proto_block.block_info.hash) + + # Invalidate in the middle of the chain + node.invalidateblock(block_hashes[50][::-1].hex()) + # Gives 404 for the invalidated blocks + for i in range(50, 101): + response = query_block(i) + assert_equal(response.status, 404) + proto_error = pb.Error() + proto_error.ParseFromString(response.read()) + assert_equal(proto_error.msg, f'404: Block not found: {i}') + # Previous blocks are still fine + for i in range(0, 50): + response = query_block(i) + assert_equal(response.status, 200) + + +if __name__ == '__main__': + ChronikServeTest().main() diff --git a/test/functional/test_framework/blocktools.py b/test/functional/test_framework/blocktools.py --- a/test/functional/test_framework/blocktools.py +++ b/test/functional/test_framework/blocktools.py @@ -36,6 +36,7 @@ # Genesis block time (regtest) TIME_GENESIS_BLOCK = 1296688602 +GENESIS_BLOCK_HASH = '0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206' MAX_FUTURE_BLOCK_TIME = 2 * 60 * 60