diff --git a/chronik/chronik-db/src/db.rs b/chronik/chronik-db/src/db.rs --- a/chronik/chronik-db/src/db.rs +++ b/chronik/chronik-db/src/db.rs @@ -15,7 +15,9 @@ use crate::{ groups::{ScriptHistoryWriter, ScriptUtxoWriter}, - io::{BlockWriter, MetadataWriter, SpentByWriter, TxWriter}, + io::{ + BlockStatsWriter, BlockWriter, MetadataWriter, SpentByWriter, TxWriter, + }, }; // All column family names used by Chronik should be defined here @@ -74,6 +76,7 @@ pub fn open(path: impl AsRef) -> Result { let mut cfs = Vec::new(); BlockWriter::add_cfs(&mut cfs); + BlockStatsWriter::add_cfs(&mut cfs); MetadataWriter::add_cfs(&mut cfs); TxWriter::add_cfs(&mut cfs); ScriptHistoryWriter::add_cfs(&mut cfs); diff --git a/chronik/chronik-db/src/io/block_stats.rs b/chronik/chronik-db/src/io/block_stats.rs --- a/chronik/chronik-db/src/io/block_stats.rs +++ b/chronik/chronik-db/src/io/block_stats.rs @@ -3,7 +3,6 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. use abc_rust_error::Result; -#[cfg(test)] use rocksdb::ColumnFamilyDescriptor; use serde::{Deserialize, Serialize}; @@ -132,7 +131,6 @@ batch.delete_cf(self.col.cf, bh_to_bytes(block_height)); } - #[cfg(test)] pub(crate) fn add_cfs(columns: &mut Vec) { columns.push(ColumnFamilyDescriptor::new( CF_BLK_STATS, diff --git a/chronik/chronik-indexer/src/indexer.rs b/chronik/chronik-indexer/src/indexer.rs --- a/chronik/chronik-indexer/src/indexer.rs +++ b/chronik/chronik-indexer/src/indexer.rs @@ -19,9 +19,9 @@ }, index_tx::prepare_indexed_txs, io::{ - BlockHeight, BlockReader, BlockTxs, BlockWriter, DbBlock, - MetadataReader, MetadataWriter, SchemaVersion, SpentByWriter, TxEntry, - TxWriter, + BlockHeight, BlockReader, BlockStatsWriter, BlockTxs, BlockWriter, + DbBlock, MetadataReader, MetadataWriter, SchemaVersion, SpentByWriter, + TxEntry, TxWriter, }, mem::{Mempool, MempoolTx}, }; @@ -36,7 +36,7 @@ subs_group::TxMsgType, }; -const CURRENT_INDEXER_VERSION: SchemaVersion = 6; +const CURRENT_INDEXER_VERSION: SchemaVersion = 7; /// Params for setting up a [`ChronikIndexer`] instance. #[derive(Clone)] @@ -66,6 +66,8 @@ pub db_block: DbBlock, /// Txs in the block, with locations of where they are stored on disk. pub block_txs: BlockTxs, + /// Block size in bytes. + pub size: u64, /// Txs in the block, with inputs/outputs so we can group them. pub txs: Vec, } @@ -288,9 +290,11 @@ &mut self, block: ChronikBlock, ) -> Result<()> { + let height = block.db_block.height; let mut batch = WriteBatch::default(); let block_writer = BlockWriter::new(&self.db)?; let tx_writer = TxWriter::new(&self.db)?; + let block_stats_writer = BlockStatsWriter::new(&self.db)?; let script_history_writer = ScriptHistoryWriter::new(&self.db, self.script_group.clone())?; let script_utxo_writer = @@ -300,6 +304,8 @@ let first_tx_num = tx_writer.insert(&mut batch, &block.block_txs)?; let index_txs = prepare_indexed_txs(&self.db, first_tx_num, &block.txs)?; + block_stats_writer + .insert(&mut batch, height, block.size, &index_txs)?; script_history_writer.insert(&mut batch, &index_txs)?; script_utxo_writer.insert(&mut batch, &index_txs)?; spent_by_writer.insert(&mut batch, &index_txs)?; @@ -327,6 +333,7 @@ let mut batch = WriteBatch::default(); let block_writer = BlockWriter::new(&self.db)?; let tx_writer = TxWriter::new(&self.db)?; + let block_stats_writer = BlockStatsWriter::new(&self.db)?; let script_history_writer = ScriptHistoryWriter::new(&self.db, self.script_group.clone())?; let script_utxo_writer = @@ -336,6 +343,7 @@ let first_tx_num = tx_writer.delete(&mut batch, &block.block_txs)?; let index_txs = prepare_indexed_txs(&self.db, first_tx_num, &block.txs)?; + block_stats_writer.delete(&mut batch, block.db_block.height); script_history_writer.delete(&mut batch, &index_txs)?; script_utxo_writer.delete(&mut batch, &index_txs)?; spent_by_writer.delete(&mut batch, &index_txs)?; @@ -457,6 +465,7 @@ Ok(ChronikBlock { db_block, block_txs, + size: block.size, txs, }) } @@ -561,6 +570,7 @@ block_height: 0, txs: vec![], }, + size: 285, txs: vec![], }; diff --git a/chronik/chronik-indexer/src/query/blocks.rs b/chronik/chronik-indexer/src/query/blocks.rs --- a/chronik/chronik-indexer/src/query/blocks.rs +++ b/chronik/chronik-indexer/src/query/blocks.rs @@ -8,7 +8,7 @@ use bitcoinsuite_core::block::BlockHash; use chronik_db::{ db::Db, - io::{BlockHeight, BlockReader, DbBlock}, + io::{BlockHeight, BlockReader, BlockStats, BlockStatsReader, DbBlock}, }; use chronik_proto::proto; use thiserror::Error; @@ -51,6 +51,10 @@ MAX_BLOCKS_PAGE_SIZE )] BlocksPageSizeTooLarge(usize), + + /// DB is missing block stats + #[error("500: Inconsistent DB: Missing block stats for height {0}")] + MissingBlockStats(BlockHeight), } use self::QueryBlockError::*; @@ -65,6 +69,7 @@ hash_or_height: String, ) -> Result { let db_blocks = BlockReader::new(self.db)?; + let block_stats_reader = BlockStatsReader::new(self.db)?; let db_block = if let Ok(hash) = hash_or_height.parse::() { db_blocks.by_hash(&hash)? } else { @@ -77,8 +82,13 @@ db_blocks.by_height(height)? }; let db_block = db_block.ok_or(BlockNotFound(hash_or_height))?; + let block_stats = block_stats_reader + .by_height(db_block.height)? + .ok_or(MissingBlockStats(db_block.height))?; Ok(proto::Block { - block_info: Some(self.make_block_info_proto(&db_block)), + block_info: Some( + self.make_block_info_proto(&db_block, &block_stats), + ), }) } @@ -99,6 +109,7 @@ return Err(BlocksPageSizeTooLarge(num_blocks).into()); } let block_reader = BlockReader::new(self.db)?; + let block_stats_reader = BlockStatsReader::new(self.db)?; let mut blocks = Vec::with_capacity(num_blocks); for block_height in start_height..=end_height { let block = block_reader.by_height(block_height)?; @@ -106,7 +117,10 @@ Some(block) => block, None => break, }; - blocks.push(self.make_block_info_proto(&block)); + let block_stats = block_stats_reader + .by_height(block_height)? + .ok_or(MissingBlockStats(block_height))?; + blocks.push(self.make_block_info_proto(&block, &block_stats)); } Ok(proto::Blocks { blocks }) } @@ -126,7 +140,11 @@ } } - fn make_block_info_proto(&self, db_block: &DbBlock) -> proto::BlockInfo { + fn make_block_info_proto( + &self, + db_block: &DbBlock, + block_stats: &BlockStats, + ) -> proto::BlockInfo { proto::BlockInfo { hash: db_block.hash.to_vec(), prev_hash: db_block.prev_hash.to_vec(), @@ -134,6 +152,14 @@ n_bits: db_block.n_bits, timestamp: db_block.timestamp, is_final: self.avalanche.is_final_height(db_block.height), + block_size: block_stats.block_size, + num_txs: block_stats.num_txs, + num_inputs: block_stats.num_inputs, + num_outputs: block_stats.num_outputs, + sum_input_sats: block_stats.sum_input_sats, + sum_coinbase_output_sats: block_stats.sum_coinbase_output_sats, + sum_normal_output_sats: block_stats.sum_normal_output_sats, + sum_burned_sats: block_stats.sum_burned_sats, } } } diff --git a/chronik/chronik-proto/proto/chronik.proto b/chronik/chronik-proto/proto/chronik.proto --- a/chronik/chronik-proto/proto/chronik.proto +++ b/chronik/chronik-proto/proto/chronik.proto @@ -40,6 +40,22 @@ int64 timestamp = 5; // Whether the block has been finalized by Avalanche bool is_final = 14; + // Block size of this block in bytes (including headers etc.) + uint64 block_size = 6; + // Number of txs in this block + uint64 num_txs = 7; + // Total number of tx inputs in block (including coinbase) + uint64 num_inputs = 8; + // Total number of tx output in block (including coinbase) + uint64 num_outputs = 9; + // Total number of satoshis spent by tx inputs + int64 sum_input_sats = 10; + // Block reward for this block + int64 sum_coinbase_output_sats = 11; + // Total number of satoshis in non-coinbase tx outputs + int64 sum_normal_output_sats = 12; + // Total number of satoshis burned using OP_RETURN + int64 sum_burned_sats = 13; } // Details about a transaction diff --git a/test/functional/chronik_block.py b/test/functional/chronik_block.py --- a/test/functional/chronik_block.py +++ b/test/functional/chronik_block.py @@ -35,6 +35,14 @@ height=0, n_bits=0x207fffff, timestamp=TIME_GENESIS_BLOCK, + block_size=285, + num_txs=1, + num_inputs=1, + num_outputs=1, + sum_input_sats=0, + sum_coinbase_output_sats=5000000000, + sum_normal_output_sats=0, + sum_burned_sats=0, ), ) @@ -73,6 +81,14 @@ height=i, n_bits=0x207fffff, timestamp=proto_block.block_info.timestamp, + block_size=181, + num_txs=1, + num_inputs=1, + num_outputs=1, + sum_input_sats=0, + sum_coinbase_output_sats=5000000000, + sum_normal_output_sats=0, + sum_burned_sats=0, ), )) assert_equal(proto_block, chronik.block(block_hashes[i]).ok()) @@ -102,6 +118,14 @@ height=50, n_bits=0x207fffff, timestamp=proto_block.block_info.timestamp, + block_size=181, + num_txs=1, + num_inputs=1, + num_outputs=1, + sum_input_sats=0, + sum_coinbase_output_sats=5000000000, + sum_normal_output_sats=0, + sum_burned_sats=0, ), )) assert_equal(chronik.block(fork_hash).ok(), proto_block) diff --git a/test/functional/chronik_block_info.py b/test/functional/chronik_block_info.py new file mode 100644 --- /dev/null +++ b/test/functional/chronik_block_info.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +# Copyright (c) 2023 The Bitcoin developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +""" +Test if the `BlockInfo` fields are set correctly in Chronik. +""" + +from test_framework.address import ( + ADDRESS_ECREG_P2SH_OP_TRUE, + ADDRESS_ECREG_UNSPENDABLE, + P2SH_OP_TRUE, + SCRIPTSIG_OP_TRUE, +) +from test_framework.blocktools import ( + create_block, + create_coinbase, + make_conform_to_ctor, +) +from test_framework.messages import COutPoint, CTransaction, CTxIn, CTxOut +from test_framework.p2p import P2PDataStore +from test_framework.script import OP_RETURN, CScript +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal + + +class ChronikBlockInfoTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 1 + self.extra_args = [['-chronik']] + self.rpc_timeout = 240 + + def skip_test_if_missing_module(self): + self.skip_if_no_chronik() + + def run_test(self): + from test_framework.chronik.client import ChronikClient, pb + + node = self.nodes[0] + node.setmocktime(1300000000) + chronik = ChronikClient('127.0.0.1', node.chronik_port) + + peer = node.add_p2p_connection(P2PDataStore()) + + coinblockhash = self.generatetoaddress(node, 1, ADDRESS_ECREG_P2SH_OP_TRUE)[0] + coinblock = node.getblock(coinblockhash) + cointx = coinblock['tx'][0] + + prev_hash = self.generatetoaddress(node, 100, ADDRESS_ECREG_UNSPENDABLE)[-1] + + coinvalue = 5000000000 + tx = CTransaction() + tx.vin = [CTxIn(outpoint=COutPoint(int(cointx, 16), 0), + scriptSig=SCRIPTSIG_OP_TRUE)] + tx.vout = [ + CTxOut(coinvalue - 10000, P2SH_OP_TRUE), + CTxOut(1000, CScript([OP_RETURN, b'test'])), + ] + tx.rehash() + + txid = node.sendrawtransaction(tx.serialize().hex()) + + tip_hash = self.generatetoaddress(node, 1, ADDRESS_ECREG_UNSPENDABLE)[-1] + + assert_equal(chronik.block(tip_hash).ok(), pb.Block( + block_info=pb.BlockInfo( + hash=bytes.fromhex(tip_hash)[::-1], + prev_hash=bytes.fromhex(prev_hash)[::-1], + height=102, + n_bits=0x207fffff, + timestamp=1300000018, + block_size=281, + num_txs=2, + num_inputs=2, + num_outputs=3, + sum_input_sats=coinvalue, + sum_coinbase_output_sats=coinvalue + 9000, + sum_normal_output_sats=coinvalue - 9000, + sum_burned_sats=1000, + ), + )) + + node.invalidateblock(tip_hash) + chronik.block(tip_hash).err(404) + + tx2 = CTransaction() + tx2.vin = [CTxIn(outpoint=COutPoint(int(txid, 16), 0), + scriptSig=SCRIPTSIG_OP_TRUE)] + tx2.vout = [ + CTxOut(3000, CScript([OP_RETURN, b'test'])), + CTxOut(5000, CScript([OP_RETURN, b'test'])), + CTxOut(coinvalue - 20000, P2SH_OP_TRUE), + ] + tx2.rehash() + + block = create_block(int(prev_hash, 16), + create_coinbase(102, b'\x03' * 33), + 1300000500) + block.vtx += [tx, tx2] + make_conform_to_ctor(block) + block.hashMerkleRoot = block.calc_merkle_root() + block.solve() + peer.send_blocks_and_test([block], node) + + assert_equal(chronik.block(block.hash).ok(), pb.Block( + block_info=pb.BlockInfo( + hash=bytes.fromhex(block.hash)[::-1], + prev_hash=bytes.fromhex(prev_hash)[::-1], + height=102, + n_bits=0x207fffff, + timestamp=1300000500, + block_size=403, + num_txs=3, + num_inputs=3, + num_outputs=7, + sum_input_sats=coinvalue * 2 - 10000, + sum_coinbase_output_sats=coinvalue, + sum_normal_output_sats=coinvalue * 2 - 21000, + sum_burned_sats=9000, + ), + )) + + +if __name__ == '__main__': + ChronikBlockInfoTest().main() diff --git a/test/functional/chronik_blocks.py b/test/functional/chronik_blocks.py --- a/test/functional/chronik_blocks.py +++ b/test/functional/chronik_blocks.py @@ -48,6 +48,14 @@ height=0, n_bits=0x207fffff, timestamp=TIME_GENESIS_BLOCK, + block_size=285, + num_txs=1, + num_inputs=1, + num_outputs=1, + sum_input_sats=0, + sum_coinbase_output_sats=5000000000, + sum_normal_output_sats=0, + sum_burned_sats=0, ) assert_equal(chronik.blocks(0, 100).ok(), pb.Blocks(blocks=[genesis_info])) assert_equal(chronik.blocks(0, 0).ok(), pb.Blocks(blocks=[genesis_info])) @@ -68,6 +76,14 @@ height=height, n_bits=0x207fffff, timestamp=1300000003, + block_size=181, + num_txs=1, + num_inputs=1, + num_outputs=1, + sum_input_sats=0, + sum_coinbase_output_sats=5000000000, + sum_normal_output_sats=0, + sum_burned_sats=0, ) for height in range(8, 13) ]),