diff --git a/chronik/chronik-db/src/db.rs b/chronik/chronik-db/src/db.rs index 0d4c7eb36..86f728d75 100644 --- a/chronik/chronik-db/src/db.rs +++ b/chronik/chronik-db/src/db.rs @@ -1,171 +1,173 @@ // Copyright (c) 2022 The Bitcoin developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. //! Module containing [`Db`] and errors, which encapsulates a database. //! Read and write operations should exclusively be done with dedicated writers //! and readers, such as [`crate::io::BlockWriter`]. use std::path::Path; use abc_rust_error::Result; pub use rocksdb::WriteBatch; use rocksdb::{ColumnFamilyDescriptor, IteratorMode}; use thiserror::Error; use crate::{ groups::{ScriptHistoryWriter, ScriptUtxoWriter}, io::{BlockWriter, MetadataWriter, SpentByWriter, TxWriter}, }; // All column family names used by Chronik should be defined here /// Column family name for the block data. pub const CF_BLK: &str = "blk"; /// Column family for the first tx_num of the block. Used to get a list of the /// txs of the block. pub const CF_BLK_BY_FIRST_TX: &str = "blk_by_first_tx"; +/// Column family for stats about blocks. +pub const CF_BLK_STATS: &str = "blk_stats"; /// Column family for the block height of the first tx_num of that block. Used /// to get the block height of a tx. pub const CF_FIRST_TX_BY_BLK: &str = "first_tx_by_blk"; /// Column family to lookup a block by its hash. pub const CF_LOOKUP_BLK_BY_HASH: &str = "lookup_blk_by_hash"; /// Column family to lookup a tx by its hash. pub const CF_LOOKUP_TX_BY_HASH: &str = "lookup_tx_by_hash"; /// Column family name for db metadata. pub const CF_META: &str = "meta"; /// Column family to store tx history by script. pub const CF_SCRIPT_HISTORY: &str = "script_history"; /// Column family for utxos by script. pub const CF_SCRIPT_UTXO: &str = "script_utxo"; /// Column family to store which outputs have been spent by which tx inputs. pub const CF_SPENT_BY: &str = "spent_by"; /// Column family for the tx data. pub const CF_TX: &str = "tx"; pub(crate) type CF = rocksdb::ColumnFamily; /// Indexer database. /// Owns the underlying [`rocksdb::DB`] instance. #[derive(Debug)] pub struct Db { db: rocksdb::DB, cf_names: Vec, } /// Errors indicating something went wrong with the database itself. #[derive(Debug, Error)] pub enum DbError { /// Column family requested but not defined during `Db::open`. #[error("Column family {0} doesn't exist")] NoSuchColumnFamily(String), /// Error with RocksDB itself, e.g. db inconsistency. #[error("RocksDB error: {0}")] RocksDb(rocksdb::Error), } use self::DbError::*; impl Db { /// Opens the database under the specified path. /// Creates the database file and necessary column families if necessary. pub fn open(path: impl AsRef) -> Result { let mut cfs = Vec::new(); BlockWriter::add_cfs(&mut cfs); MetadataWriter::add_cfs(&mut cfs); TxWriter::add_cfs(&mut cfs); ScriptHistoryWriter::add_cfs(&mut cfs); ScriptUtxoWriter::add_cfs(&mut cfs); SpentByWriter::add_cfs(&mut cfs); Self::open_with_cfs(path, cfs) } pub(crate) fn open_with_cfs( path: impl AsRef, cfs: Vec, ) -> Result { let db_options = Self::db_options(); let cf_names = cfs.iter().map(|cf| cf.name().to_string()).collect(); let db = rocksdb::DB::open_cf_descriptors(&db_options, path, cfs) .map_err(RocksDb)?; Ok(Db { db, cf_names }) } fn db_options() -> rocksdb::Options { let mut db_options = rocksdb::Options::default(); db_options.create_if_missing(true); db_options.create_missing_column_families(true); db_options } /// Destroy the DB, i.e. delete all it's associated files. /// /// According to the RocksDB docs, this differs from removing the dir: /// DestroyDB() will take care of the case where the RocksDB database is /// stored in multiple directories. For instance, a single DB can be /// configured to store its data in multiple directories by specifying /// different paths to DBOptions::db_paths, DBOptions::db_log_dir, and /// DBOptions::wal_dir. pub fn destroy(path: impl AsRef) -> Result<()> { let db_options = Self::db_options(); rocksdb::DB::destroy(&db_options, path).map_err(RocksDb)?; Ok(()) } /// Return a column family handle with the given name. pub fn cf(&self, name: &str) -> Result<&CF> { Ok(self .db .cf_handle(name) .ok_or_else(|| NoSuchColumnFamily(name.to_string()))?) } pub(crate) fn get( &self, cf: &CF, key: impl AsRef<[u8]>, ) -> Result>> { Ok(self.db.get_pinned_cf(cf, key).map_err(RocksDb)?) } pub(crate) fn iterator_end( &self, cf: &CF, ) -> impl Iterator, Box<[u8]>)>> + '_ { self.db .iterator_cf(cf, IteratorMode::End) .map(|result| Ok(result.map_err(RocksDb)?)) } pub(crate) fn iterator( &self, cf: &CF, start: &[u8], direction: rocksdb::Direction, ) -> impl Iterator, Box<[u8]>)>> + '_ { self.db .iterator_cf(cf, IteratorMode::From(start, direction)) .map(|result| Ok(result.map_err(RocksDb)?)) } /// Writes the batch to the Db atomically. pub fn write_batch(&self, write_batch: WriteBatch) -> Result<()> { self.db.write(write_batch).map_err(RocksDb)?; Ok(()) } /// Whether any of the column families in the DB have any data. /// /// Note: RocksDB forbids not opening all column families, therefore, this /// will always iter through all column families. pub fn is_db_empty(&self) -> Result { for cf_name in &self.cf_names { let cf = self.cf(cf_name)?; let mut cf_iter = self.db.full_iterator_cf(cf, IteratorMode::Start); if cf_iter.next().is_some() { return Ok(false); } } Ok(true) } } diff --git a/chronik/chronik-db/src/io/block_stats.rs b/chronik/chronik-db/src/io/block_stats.rs new file mode 100644 index 000000000..8d9cd784d --- /dev/null +++ b/chronik/chronik-db/src/io/block_stats.rs @@ -0,0 +1,273 @@ +// Copyright (c) 2023 The Bitcoin developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +use abc_rust_error::Result; +#[cfg(test)] +use rocksdb::ColumnFamilyDescriptor; +use serde::{Deserialize, Serialize}; + +use crate::{ + db::{Db, CF, CF_BLK_STATS}, + index_tx::IndexTx, + io::{bh_to_bytes, BlockHeight}, + ser::{db_deserialize, db_serialize}, +}; + +/// Statistics about a block, like num txs, block size, etc. +#[derive( + Clone, Debug, Default, Deserialize, Eq, Hash, PartialEq, Serialize, +)] +pub struct BlockStats { + /// Block size of this block in bytes (including headers etc.) + pub block_size: u64, + /// Number of txs in this block + pub num_txs: u64, + /// Total number of tx inputs in block (including coinbase) + pub num_inputs: u64, + /// Total number of tx output in block (including coinbase) + pub num_outputs: u64, + /// Total number of satoshis spent by tx inputs + pub sum_input_sats: i64, + /// Block reward for this block + pub sum_coinbase_output_sats: i64, + /// Total number of satoshis in non-coinbase tx outputs + pub sum_normal_output_sats: i64, + /// Total number of satoshis burned using OP_RETURN + pub sum_burned_sats: i64, +} + +struct BlockStatsColumn<'a> { + db: &'a Db, + cf: &'a CF, +} + +/// Write [`BlockStats`] to the DB. +#[derive(Debug)] +pub struct BlockStatsWriter<'a> { + col: BlockStatsColumn<'a>, +} + +/// Read [`BlockStats`] from the DB. +#[derive(Debug)] +pub struct BlockStatsReader<'a> { + col: BlockStatsColumn<'a>, +} + +impl<'a> BlockStatsColumn<'a> { + fn new(db: &'a Db) -> Result { + let cf = db.cf(CF_BLK_STATS)?; + Ok(BlockStatsColumn { db, cf }) + } +} + +impl<'a> BlockStatsWriter<'a> { + /// Create a new [`BlockStatsWriter`]. + pub fn new(db: &'a Db) -> Result { + Ok(BlockStatsWriter { + col: BlockStatsColumn::new(db)?, + }) + } + + /// Measure the [`BlockStats`] of the block with the given the block txs and + /// add them to the `WriteBatch`. + pub fn insert( + &self, + batch: &mut rocksdb::WriteBatch, + block_height: BlockHeight, + block_size: u64, + txs: &[IndexTx<'_>], + ) -> Result<()> { + let mut num_inputs = 0; + let mut num_outputs = 0; + let mut sum_input_sats = 0; + let mut sum_normal_output_sats = 0; + let mut sum_coinbase_output_sats = 0; + let mut sum_burned_sats = 0; + for tx in txs { + for output in &tx.tx.outputs { + if output.script.is_opreturn() { + sum_burned_sats += output.value; + } + } + let tx_output_sats = + tx.tx.outputs.iter().map(|output| output.value).sum::(); + if tx.is_coinbase { + sum_coinbase_output_sats += tx_output_sats; + } else { + sum_normal_output_sats += tx_output_sats; + for input in &tx.tx.inputs { + if let Some(coin) = input.coin.as_ref() { + sum_input_sats += coin.output.value; + } + } + } + num_inputs += tx.tx.inputs.len(); + num_outputs += tx.tx.outputs.len(); + } + let stats = BlockStats { + block_size, + num_txs: txs.len() as u64, + num_inputs: num_inputs as u64, + num_outputs: num_outputs as u64, + sum_input_sats, + sum_coinbase_output_sats, + sum_normal_output_sats, + sum_burned_sats, + }; + batch.put_cf( + self.col.cf, + bh_to_bytes(block_height), + db_serialize(&stats)?, + ); + Ok(()) + } + + /// Delete the block stats for the block with the given height. + pub fn delete( + &self, + batch: &mut rocksdb::WriteBatch, + block_height: BlockHeight, + ) { + batch.delete_cf(self.col.cf, bh_to_bytes(block_height)); + } + + #[cfg(test)] + pub(crate) fn add_cfs(columns: &mut Vec) { + columns.push(ColumnFamilyDescriptor::new( + CF_BLK_STATS, + rocksdb::Options::default(), + )); + } +} + +impl<'a> BlockStatsReader<'a> { + /// Create a new [`BlockStatsReader`]. + pub fn new(db: &'a Db) -> Result { + Ok(BlockStatsReader { + col: BlockStatsColumn::new(db)?, + }) + } + + /// Read the [`BlockStats`] from the DB, or [`None`] if the block doesn't + /// exist. + pub fn by_height( + &self, + block_height: BlockHeight, + ) -> Result> { + match self.col.db.get(self.col.cf, bh_to_bytes(block_height))? { + Some(ser_block_stats) => { + Ok(Some(db_deserialize::(&ser_block_stats)?)) + } + None => Ok(None), + } + } +} + +impl std::fmt::Debug for BlockStatsColumn<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "BlockStatsColumn {{ .. }}") + } +} + +#[cfg(test)] +mod tests { + use abc_rust_error::Result; + use bitcoinsuite_core::{ + script::{opcode::*, ScriptMut}, + tx::{Tx, TxId, TxMut, TxOutput}, + }; + use pretty_assertions::assert_eq; + use rocksdb::WriteBatch; + + use crate::{ + db::Db, + index_tx::prepare_indexed_txs, + io::{ + BlockStats, BlockStatsReader, BlockStatsWriter, BlockTxs, TxEntry, + TxWriter, + }, + test::make_inputs_tx, + }; + + #[test] + fn test_block_stats() -> Result<()> { + let tempdir = tempdir::TempDir::new("chronik-db--block_stats")?; + let mut cfs = Vec::new(); + TxWriter::add_cfs(&mut cfs); + BlockStatsWriter::add_cfs(&mut cfs); + let db = Db::open_with_cfs(tempdir.path(), cfs)?; + let tx_writer = TxWriter::new(&db)?; + let stats_writer = BlockStatsWriter::new(&db)?; + let stats_reader = BlockStatsReader::new(&db)?; + + let block = vec![ + make_inputs_tx(0x01, [(0x00, u32::MAX, 0xffff)], [50, 20]), + make_inputs_tx(0x02, [(0x01, 0, 50)], [40, 10]), + make_inputs_tx( + 0x03, + [(0x02, 0, 40), (0x01, 1, 20), (0x02, 1, 10)], + [60, 5], + ), + Tx::with_txid( + TxId::from([0x05; 32]), + TxMut { + version: 1, + inputs: make_inputs_tx(0, [(0x03, 0, 60)], []) + .inputs + .clone(), + outputs: vec![TxOutput { + value: 60, + script: { + let mut script = ScriptMut::default(); + script.put_opcodes([OP_RETURN, OP_1]); + script.freeze() + }, + }], + locktime: 0, + }, + ), + ]; + + let block_txs = block + .iter() + .map(|tx| TxEntry { + txid: tx.txid(), + ..Default::default() + }) + .collect::>(); + let mut batch = WriteBatch::default(); + let first_tx_num = tx_writer.insert( + &mut batch, + &BlockTxs { + txs: block_txs, + block_height: 1, + }, + )?; + let index_txs = prepare_indexed_txs(&db, first_tx_num, &block)?; + stats_writer.insert(&mut batch, 1, 1337, &index_txs)?; + db.write_batch(batch)?; + + assert_eq!( + stats_reader.by_height(1)?, + Some(BlockStats { + block_size: 1337, + num_txs: 4, + num_inputs: 6, + num_outputs: 7, + sum_input_sats: 180, + sum_coinbase_output_sats: 70, + sum_normal_output_sats: 175, + sum_burned_sats: 60, + }), + ); + + let mut batch = WriteBatch::default(); + stats_writer.delete(&mut batch, 1); + db.write_batch(batch)?; + + assert_eq!(stats_reader.by_height(1)?, None); + + Ok(()) + } +} diff --git a/chronik/chronik-db/src/io/mod.rs b/chronik/chronik-db/src/io/mod.rs index 5de8f6c42..8aa0ded93 100644 --- a/chronik/chronik-db/src/io/mod.rs +++ b/chronik/chronik-db/src/io/mod.rs @@ -1,19 +1,21 @@ // Copyright (c) 2022 The Bitcoin developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. //! Module containing readers and writers for the database used by Chronik. +mod block_stats; mod blocks; mod group_history; mod group_utxos; mod metadata; mod spent_by; mod txs; +pub use self::block_stats::*; pub use self::blocks::*; pub use self::group_history::*; pub use self::group_utxos::*; pub use self::metadata::*; pub use self::spent_by::*; pub use self::txs::*;