Page Menu
Home
Phabricator
Search
Configure Global Search
Log In
Files
F13711231
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Award Token
Flag For Later
Size
99 KB
Subscribers
None
View Options
diff --git a/Cargo.toml b/Cargo.toml
index ac859662c..a690b1f47 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,21 +1,21 @@
# Copyright (c) 2022 The Bitcoin developers
[workspace]
members = [
"chronik/abc-rust-error",
"chronik/abc-rust-lint",
"chronik/bitcoinsuite-core",
"chronik/bitcoinsuite-slp",
"chronik/chronik-bridge",
"chronik/chronik-db",
"chronik/chronik-http",
"chronik/chronik-indexer",
"chronik/chronik-lib",
"chronik/chronik-plugin",
"chronik/chronik-proto",
"chronik/chronik-util",
]
[workspace.package]
-rust-version = "1.72.0"
+rust-version = "1.76.0"
diff --git a/chronik/CMakeLists.txt b/chronik/CMakeLists.txt
index dd4d3de01..83fd25afc 100644
--- a/chronik/CMakeLists.txt
+++ b/chronik/CMakeLists.txt
@@ -1,210 +1,211 @@
# Copyright (c) 2022 The Bitcoin developers
set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
include(FetchContent)
FetchContent_Declare(
Corrosion
GIT_REPOSITORY https://github.com/corrosion-rs/corrosion.git
GIT_TAG v0.4.7
)
FetchContent_MakeAvailable(Corrosion)
-set(REQUIRED_RUST_VERSION "1.72.0")
+set(REQUIRED_RUST_VERSION "1.76.0")
if(Rust_VERSION VERSION_LESS REQUIRED_RUST_VERSION)
message(FATAL_ERROR "Minimum required Rust version is "
- "${REQUIRED_RUST_VERSION}, but found ${Rust_VERSION}")
+ "${REQUIRED_RUST_VERSION}, but found ${Rust_VERSION}. "
+ "Use `rustup update stable` to update.")
endif()
set(CARGO_BUILD_DIR "${CMAKE_BINARY_DIR}/cargo/build")
set_property(DIRECTORY "${CMAKE_SOURCE_DIR}"
APPEND PROPERTY
ADDITIONAL_CLEAN_FILES "${CARGO_BUILD_DIR}"
)
get_property(
RUSTC_EXECUTABLE
TARGET Rust::Rustc PROPERTY IMPORTED_LOCATION
)
get_filename_component(RUST_BIN_DIR ${RUSTC_EXECUTABLE} DIRECTORY)
include(DoOrFail)
find_program_or_fail(RUSTDOC_EXECUTABLE rustdoc
PATHS "${RUST_BIN_DIR}"
)
set(CHRONIK_CARGO_FLAGS --locked)
if(BUILD_BITCOIN_CHRONIK_PLUGINS)
set(CHRONIK_FEATURE_FLAGS --features plugins)
endif()
function(add_cargo_custom_target TARGET)
add_custom_target(${TARGET}
COMMAND
"${CMAKE_COMMAND}"
-E env
CARGO_TARGET_DIR="${CARGO_BUILD_DIR}"
CARGO_BUILD_RUSTC="$<TARGET_FILE:Rust::Rustc>"
CARGO_BUILD_RUSTDOC="${RUSTDOC_EXECUTABLE}"
"$<TARGET_FILE:Rust::Cargo>"
${CHRONIK_CARGO_FLAGS}
${ARGN}
WORKING_DIRECTORY
"${CMAKE_SOURCE_DIR}"
)
endfunction()
function(add_crate_test_targets CRATE)
set(CRATE_TEST_TARGET "check-crate-${CRATE}")
add_custom_target("${CRATE_TEST_TARGET}")
set(CLIPPY_TARGET "${CRATE_TEST_TARGET}-clippy")
add_cargo_custom_target("${CLIPPY_TARGET}"
clippy
--package "${CRATE}-*"
-- -D warnings
)
set(TEST_TARGET "${CRATE_TEST_TARGET}-test")
add_cargo_custom_target("${TEST_TARGET}"
test
--package "${CRATE}-*"
)
add_dependencies("${CRATE_TEST_TARGET}"
"${CLIPPY_TARGET}"
"${TEST_TARGET}"
)
add_dependencies("check-crates"
"${CRATE_TEST_TARGET}"
)
endfunction()
add_custom_target("check-crates")
add_crate_test_targets(abc-rust)
add_crate_test_targets(bitcoinsuite)
add_crate_test_targets(chronik)
# Compile Rust, generates chronik-lib
corrosion_import_crate(
MANIFEST_PATH "chronik-lib/Cargo.toml"
FLAGS
${CHRONIK_CARGO_FLAGS}
${CHRONIK_FEATURE_FLAGS}
)
set(Rust_TRIPLE
"${Rust_CARGO_TARGET_ARCH}"
"${Rust_CARGO_TARGET_VENDOR}"
"${Rust_CARGO_TARGET_OS}"
)
if (Rust_CARGO_TARGET_ENV)
list(APPEND Rust_TRIPLE "${Rust_CARGO_TARGET_ENV}")
endif()
list(JOIN Rust_TRIPLE "-" Rust_CARGO_TARGET)
# cxx crate generates some source files at this location
set(CXXBRIDGE_GENERATED_FOLDER
"${CARGO_BUILD_DIR}/${Rust_CARGO_TARGET}/cxxbridge")
set(CHRONIK_BRIDGE_GENERATED_CPP_FILES
"${CXXBRIDGE_GENERATED_FOLDER}/chronik-bridge/src/ffi.rs.cc")
set(CHRONIK_LIB_GENERATED_CPP_FILES
"${CXXBRIDGE_GENERATED_FOLDER}/chronik-lib/src/ffi.rs.cc")
add_custom_command(
OUTPUT
${CHRONIK_BRIDGE_GENERATED_CPP_FILES}
${CHRONIK_LIB_GENERATED_CPP_FILES}
COMMAND
"${CMAKE_COMMAND}"
-E env
"echo" "Generating cxx bridge files"
DEPENDS $<TARGET_PROPERTY:chronik-lib,INTERFACE_LINK_LIBRARIES>
)
# Chronik-bridge library
# Contains all the C++ functions used by Rust, and the code bridging both
add_library(chronik-bridge
chronik-cpp/chronik_bridge.cpp
chronik-cpp/util/hash.cpp
${CHRONIK_BRIDGE_GENERATED_CPP_FILES}
)
target_include_directories(chronik-bridge
PUBLIC
"${CMAKE_CURRENT_SOURCE_DIR}"
"${CXXBRIDGE_GENERATED_FOLDER}"
)
target_link_libraries(chronik-bridge
util
leveldb
)
# Chronik library
# Compiles and links all the Chronik code, and exposes chronik::Start and
# chronik::Stop to run the indexer from C++.
add_library(chronik
chronik-cpp/chronik.cpp
chronik-cpp/chronik_validationinterface.cpp
${CHRONIK_LIB_GENERATED_CPP_FILES}
)
target_link_libraries(chronik
chronik-bridge
chronik-lib
)
# Plugins require us to link agains libpython
if(BUILD_BITCOIN_CHRONIK_PLUGINS)
find_package(Python COMPONENTS Interpreter Development)
message("Adding Python_LIBRARIES: ${Python_LIBRARIES}")
target_link_libraries(chronik ${Python_LIBRARIES})
endif()
if(${CMAKE_SYSTEM_NAME} MATCHES "Windows")
# mio crate (dependency of tokio) requires winternl.h, found in ntdll
find_package(NTDLL REQUIRED)
target_link_libraries(chronik NTDLL::ntdll)
# rocksdb requires items from rpcdce.h, found in rpcrt4
find_package(RPCRT4 REQUIRED)
target_link_libraries(chronik RPCRT4::rpcrt4)
endif()
# Rocksdb requires "atomic"
include(AddCompilerFlags)
custom_check_linker_flag(LINKER_HAS_ATOMIC "-latomic")
if(LINKER_HAS_ATOMIC)
target_link_libraries(chronik atomic)
endif()
# Add chronik to server
target_link_libraries(server
chronik
# TODO: We need to add the library again, otherwise gcc linking fails.
# It's not clear yet why this is the case.
chronik-bridge
)
# Install the directory containing the proto files. The trailing slash ensures
# the directory is not duplicated (see
# https://cmake.org/cmake/help/v3.16/command/install.html#installing-directories)
set(CHRONIK_PROTO_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/chronik-proto/proto/")
set(CHRONIK_PROTO_COMPONENT "chronik-proto")
install(
DIRECTORY "${CHRONIK_PROTO_DIRECTORY}"
DESTINATION "proto"
COMPONENT "${CHRONIK_PROTO_COMPONENT}"
)
add_custom_target("install-${CHRONIK_PROTO_COMPONENT}"
COMMENT "Installing component ${CHRONIK_PROTO_COMPONENT}"
COMMAND
"${CMAKE_COMMAND}"
-E env CMAKE_INSTALL_ALWAYS=ON
"${CMAKE_COMMAND}"
-DCOMPONENT="${CHRONIK_PROTO_COMPONENT}"
-DCMAKE_INSTALL_PREFIX="${CMAKE_INSTALL_PREFIX}"
-P cmake_install.cmake
WORKING_DIRECTORY "${CMAKE_BINARY_DIR}"
)
diff --git a/chronik/bitcoinsuite-slp/src/color.rs b/chronik/bitcoinsuite-slp/src/color.rs
index a2c9b3a78..67fe44478 100644
--- a/chronik/bitcoinsuite-slp/src/color.rs
+++ b/chronik/bitcoinsuite-slp/src/color.rs
@@ -1,596 +1,596 @@
// Copyright (c) 2023 The Bitcoin developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
//! Module for [`ColoredTx`].
use bitcoinsuite_core::tx::{Tx, TxId};
use bytes::Bytes;
use thiserror::Error;
use crate::{
alp,
color::ColorError::*,
empp,
parsed::{ParsedData, ParsedGenesis, ParsedMintData, ParsedTxType},
slp,
structs::{
Amount, GenesisInfo, Token, TokenMeta, TokenOutput, TokenVariant,
TxType,
},
token_id::TokenId,
token_type::{SlpTokenType, TokenType},
};
/// A parsed SLP or ALP tx with outputs colored according to the tokens
/// specified in the `OP_RETURN`.
#[derive(Clone, Debug, Default, PartialEq)]
pub struct ColoredTx {
/// Parsed sections defining where tokens should go.
/// Can be multiple for ALP, at most 1 for SLP.
pub sections: Vec<ColoredTxSection>,
/// Intentional burns, specifying how many tokens are supposed to be burned
/// of which type.
pub intentional_burns: Vec<IntentionalBurn>,
/// Outputs colored with the tokens as specified in the `OP_RETURN`.
pub outputs: Vec<Option<TokenOutput>>,
/// Reports of failed parsing attempts
pub failed_parsings: Vec<FailedParsing>,
/// Reports of failed coloring attempts
pub failed_colorings: Vec<FailedColoring>,
}
/// Section defining where tokens should go as specified in the `OP_RETURN`.
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct ColoredTxSection {
/// [`TokenMeta`] specified in the section.
pub meta: TokenMeta,
/// [`TxType`] specified in the section.
pub tx_type: TxType,
/// Minimum required sum of input tokens of this token meta.
/// Note that this may be different from the `outputs` in [`ColoredTx`] of
/// the respective token meta for SLP, as SLP allows sending tokens to
/// nonexistent outputs, whereas in ALP this would be a failed coloring.
pub required_input_sum: u128,
/// SLP allows coloring non-existent outputs, but this usually is a
/// mistake, so we keep track of it here.
pub has_colored_out_of_range: bool,
/// [`GenesisInfo`] introduced in this section. Only present for GENESIS.
pub genesis_info: Option<GenesisInfo>,
}
/// Any kind of parse error that can occur when processing a tx.
#[derive(Clone, Debug, Eq, PartialEq)]
pub enum ParseError {
/// Parsing the OP_RETURN as eMPP failed
Empp(empp::ParseError),
/// Parsing the OP_RETURN as SLP failed
Slp(slp::ParseError),
/// Parsing a pushdata in an OP_RETURN as ALP failed
Alp(alp::ParseError),
}
/// Report of a failed parsing attempt
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct FailedParsing {
/// Which pushdata in the OP_RETURN failed to parse, or None if the entire
/// OP_RETURN is the culprit.
pub pushdata_idx: Option<usize>,
/// The actual bytes that failed to parse.
pub bytes: Bytes,
/// Error explaining why the parsing failed.
pub error: ParseError,
}
/// Report of a failed coloring attempt
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct FailedColoring {
/// Which pushdata in the OP_RETURN failed to color the tx.
pub pushdata_idx: usize,
/// Parsed data which failed to color.
pub parsed: ParsedData,
/// Error explaining why the coloring failed.
pub error: ColorError,
}
/// Intentional burn, allowing users to specify cleanly and precisely how tokens
/// should be removed from supply.
///
/// This prevents the bells and whistles of indexers and wallets to reject a tx
/// whose intent is to remove tokens from supply.
#[derive(Clone, Debug, Eq, Hash, PartialEq)]
pub struct IntentionalBurn {
/// Which token meta should be burned
pub meta: TokenMeta,
/// How many tokens should be burned
pub amount: Amount,
}
/// Error when trying to color a parsed section.
#[derive(Clone, Debug, Error, Eq, PartialEq)]
pub enum ColorError {
/// ALP disallows coloring non-existent outputs
#[error("Too few outputs, expected {expected} but got {actual}")]
TooFewOutputs {
/// Expected number of outputs for the coloring to succeed
expected: usize,
/// Actual number of outputs
actual: usize,
},
/// GENESIS must be first
#[error("GENESIS must be the first pushdata")]
GenesisMustBeFirst,
/// Token types must be ascending, to allow clean upgrades when introducing
/// new token types.
#[error(
"Descending token type: {before} > {after}, token types must be in \
ascending order"
)]
DescendingTokenType {
/// Larger token type coming before
before: u8,
/// Smaller token type coming after
after: u8,
},
/// Tried coloring using the same token ID twice, which is not allowed.
/// Only the first coloring counts.
#[error(
"Duplicate token_id {token_id}, found in section {prev_section_idx}"
)]
DuplicateTokenId {
/// Valid section that first colored the tx
prev_section_idx: usize,
/// Token ID that was colored twice
token_id: TokenId,
},
/// Tried doing an intentional burn of the same token ID twice, which is
/// not allowed. Only the first intentional burn counts.
#[error(
"Duplicate intentional burn token_id {token_id}, found in burn \
#{prev_burn_idx} and #{burn_idx}"
)]
DuplicateIntentionalBurnTokenId {
/// Valid previous intentional burn
prev_burn_idx: usize,
/// Invalid later duplicate burn
burn_idx: usize,
/// Token ID burned
token_id: TokenId,
},
/// Outputs cannot be colored twice by different sections
#[error(
"Overlapping amount when trying to color {amount} at index \
{output_idx}, output is already colored with {prev_token}"
)]
OverlappingAmount {
/// Previous token the output is already colored with
prev_token: Token,
/// Index of the output that we tried to color twice
output_idx: usize,
/// Amount that tried to color an output twice
amount: Amount,
},
/// Outputs cannot be colored twice by different sections
#[error(
"Overlapping mint baton when trying to color mint baton at index \
{output_idx}, output is already colored with {prev_token}"
)]
OverlappingMintBaton {
/// Previous token the output is already colored with
prev_token: Token,
/// Index of the output that we tried tot color twice
output_idx: usize,
},
}
impl ColoredTx {
/// Parse the OP_RETURN of the tx and color its outputs
pub fn color_tx(tx: &Tx) -> Option<ColoredTx> {
- let op_return = match tx.outputs.get(0) {
+ let op_return = match tx.outputs.first() {
Some(output) if output.script.is_opreturn() => &output.script,
_ => return None,
};
let mut colored = ColoredTx {
sections: vec![],
intentional_burns: vec![],
outputs: vec![None; tx.outputs.len()],
failed_colorings: vec![],
failed_parsings: vec![],
};
// First, try to parse and color as SLP tx
match slp::parse(tx.txid_ref(), op_return) {
Ok(Some(parsed)) => {
colored
.color_section(0, parsed, &mut 0)
.expect("Coloring SLP always succeeds");
return Some(colored);
}
Ok(None) => {}
Err(slp_error) => {
colored.failed_parsings.push(FailedParsing {
pushdata_idx: None,
bytes: op_return.bytecode().clone(),
error: ParseError::Slp(slp_error),
});
return Some(colored);
}
}
// Then, try to parse as eMPP tx
let pushdata = match empp::parse(op_return) {
Ok(Some(pushdata)) => pushdata,
Ok(None) => return None,
Err(empp_err) => {
colored.failed_parsings.push(FailedParsing {
pushdata_idx: None,
bytes: op_return.bytecode().clone(),
error: ParseError::Empp(empp_err),
});
return Some(colored);
}
};
// Color all the pushdata as ALP, and if we encountered any ALP
// sections, return the colored tx.
if colored.color_all_alp_pushdata(pushdata, tx.txid_ref()) {
return Some(colored);
}
// We found an eMPP OP_RETURN but no ALP sections
None
}
// Color all the pushdata as ALP one-by-one and return whether we
// encountered anything ALP-like.
fn color_all_alp_pushdata(
&mut self,
pushdata: Vec<Bytes>,
txid: &TxId,
) -> bool {
let mut max_token_type = 0;
let mut has_any_alp = false;
for (pushdata_idx, pushdata) in pushdata.into_iter().enumerate() {
let parsed = match alp::parse_section(txid, pushdata.clone()) {
Ok(Some(parsed)) => parsed,
Ok(None) => continue,
Err(alp_error) => {
self.failed_parsings.push(FailedParsing {
pushdata_idx: Some(pushdata_idx),
bytes: pushdata,
error: ParseError::Alp(alp_error),
});
has_any_alp = true;
continue;
}
};
has_any_alp = true;
let color_result = self.color_section(
pushdata_idx,
parsed.clone(),
&mut max_token_type,
);
if let Err(error) = color_result {
self.failed_colorings.push(FailedColoring {
pushdata_idx,
parsed,
error,
});
}
}
has_any_alp
}
fn color_section(
&mut self,
pushdata_idx: usize,
parsed: ParsedData,
max_token_type: &mut u8,
) -> Result<(), ColorError> {
let meta = parsed.meta;
// token_type must be in ascending order
if *max_token_type > meta.token_type.to_u8() {
return Err(DescendingTokenType {
before: *max_token_type,
after: meta.token_type.to_u8(),
});
}
*max_token_type = meta.token_type.to_u8();
// Only report duplicate token IDs on MINT and SEND, burn and GENESIS
// are handled separately
if matches!(parsed.tx_type.tx_type(), TxType::MINT | TxType::SEND) {
for (prev_section_idx, prev_section) in
self.sections.iter().enumerate()
{
if prev_section.meta.token_id == meta.token_id {
return Err(DuplicateTokenId {
prev_section_idx,
token_id: meta.token_id,
});
}
}
}
match parsed.tx_type {
ParsedTxType::Genesis(genesis) => {
self.color_genesis(pushdata_idx, meta, genesis)
}
ParsedTxType::Mint(mint) => self.color_mint(meta, mint),
ParsedTxType::Send(send) => self.color_send(meta, send),
ParsedTxType::Burn(amount) => self.color_burn(meta, amount),
ParsedTxType::Unknown => self.color_unknown(meta),
}
}
fn color_genesis(
&mut self,
pushdata_idx: usize,
meta: TokenMeta,
genesis: ParsedGenesis,
) -> Result<(), ColorError> {
// GENESIS must be the very first section in the pushdata.
// This prevents assigning the same token ID to different tokens, even
// if we introduced a new LOKAD ID, as long as it also upholds this
// rule.
if pushdata_idx != 0 {
return Err(GenesisMustBeFirst);
}
let has_colored_out_of_range =
self.color_mint_data(&meta, &genesis.mint_data)?;
self.sections.push(ColoredTxSection {
meta,
tx_type: TxType::GENESIS,
required_input_sum: 0,
genesis_info: Some(genesis.info),
has_colored_out_of_range,
});
Ok(())
}
fn color_mint(
&mut self,
meta: TokenMeta,
mint: ParsedMintData,
) -> Result<(), ColorError> {
let has_colored_out_of_range = self.color_mint_data(&meta, &mint)?;
self.sections.push(ColoredTxSection {
meta,
tx_type: TxType::MINT,
required_input_sum: 0,
genesis_info: None,
has_colored_out_of_range,
});
Ok(())
}
fn color_mint_data(
&mut self,
meta: &TokenMeta,
mint_data: &ParsedMintData,
) -> Result<bool, ColorError> {
let token_idx = self.sections.len();
let mut out_of_range_idx = None;
// Verify no outputs have been colored already
for (output_idx, &amount) in
mint_data.amounts_range().zip(&mint_data.amounts)
{
if amount != 0 {
match self.outputs.get(output_idx) {
Some(Some(token)) => {
return Err(OverlappingAmount {
prev_token: self.token(token),
output_idx,
amount,
});
}
Some(None) => {}
None => out_of_range_idx = Some(output_idx),
}
}
}
for output_idx in mint_data.batons_range() {
match self.outputs.get(output_idx) {
Some(Some(token)) => {
return Err(OverlappingMintBaton {
prev_token: self.token(token),
output_idx,
})
}
Some(None) => {}
None => out_of_range_idx = Some(output_idx),
}
}
if let Some(output_idx) = out_of_range_idx {
// ALP forbids amounts and batons for nonexistent outputs
if meta.token_type.is_alp() {
return Err(TooFewOutputs {
expected: output_idx + 1,
actual: self.outputs.len(),
});
}
}
// Now, color all outputs
for (output_idx, &amount) in
mint_data.amounts_range().zip(&mint_data.amounts)
{
if output_idx >= self.outputs.len() {
break;
}
if amount > 0 {
self.outputs[output_idx] = Some(TokenOutput {
token_idx,
variant: TokenVariant::Amount(amount),
});
}
}
for output_idx in mint_data.batons_range() {
if output_idx >= self.outputs.len() {
break;
}
self.outputs[output_idx] = Some(TokenOutput {
token_idx,
variant: TokenVariant::MintBaton,
});
}
Ok(out_of_range_idx.is_some())
}
fn color_send(
&mut self,
meta: TokenMeta,
amounts: Vec<Amount>,
) -> Result<(), ColorError> {
// Verify no outputs have been colored already
let mut out_of_range_idx = None;
for (idx, &amount) in amounts.iter().enumerate() {
if amount != 0 {
match self.outputs.get(idx + 1) {
Some(Some(token)) => {
return Err(OverlappingAmount {
prev_token: self.token(token),
output_idx: idx + 1,
amount,
})
}
Some(None) => {}
None => out_of_range_idx = Some(idx + 1),
}
}
}
if let Some(output_idx) = out_of_range_idx {
// ALP forbids amounts and batons for nonexistent outputs
if meta.token_type.is_alp() {
return Err(TooFewOutputs {
expected: output_idx + 1,
actual: self.outputs.len(),
});
}
}
// Color outputs and also calculate the required input sum
let mut required_input_sum = 0u128;
for (idx, &amount) in amounts.iter().enumerate() {
if amount == 0 {
continue;
}
required_input_sum += u128::from(amount);
if let Some(output) = self.outputs.get_mut(idx + 1) {
*output = Some(TokenOutput {
token_idx: self.sections.len(),
variant: TokenVariant::Amount(amount),
});
}
}
self.sections.push(ColoredTxSection {
meta,
tx_type: TxType::SEND,
required_input_sum,
genesis_info: None,
has_colored_out_of_range: out_of_range_idx.is_some(),
});
Ok(())
}
fn color_burn(
&mut self,
meta: TokenMeta,
amount: Amount,
) -> Result<(), ColorError> {
for (prev_burn_idx, prev_burn) in
self.intentional_burns.iter().enumerate()
{
if prev_burn.meta.token_id == meta.token_id {
return Err(DuplicateIntentionalBurnTokenId {
prev_burn_idx,
burn_idx: self.intentional_burns.len(),
token_id: meta.token_id,
});
}
}
self.intentional_burns
.push(IntentionalBurn { meta, amount });
Ok(())
}
fn color_unknown(&mut self, meta: TokenMeta) -> Result<(), ColorError> {
// Color all outputs (except the OP_RETURN) that haven't been colored
// yet as "unknown"
for token_data in self.outputs.iter_mut().skip(1) {
if token_data.is_none() {
*token_data = Some(TokenOutput {
token_idx: self.sections.len(),
variant: TokenVariant::Unknown(meta.token_type.to_u8()),
});
}
}
self.sections.push(ColoredTxSection {
meta,
tx_type: TxType::UNKNOWN,
required_input_sum: 0,
genesis_info: None,
has_colored_out_of_range: false,
});
Ok(())
}
/// Turn a [`TokenOutput`] of this [`ColoredTx`] into a [`Token`].
pub fn token(&self, token_output: &TokenOutput) -> Token {
let section = &self.sections[token_output.token_idx];
Token {
meta: section.meta,
variant: token_output.variant,
}
}
}
impl ColoredTxSection {
/// Whether the section has SLP V2 (MintVault) token type and a MINT tx
/// type.
pub fn is_mint_vault_mint(&self) -> bool {
if self.tx_type != TxType::MINT {
return false;
}
matches!(
self.meta.token_type,
TokenType::Slp(SlpTokenType::MintVault),
)
}
}
impl std::fmt::Display for FailedParsing {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "Parsing failed")?;
if let Some(pushdata_idx) = self.pushdata_idx {
write!(f, " at pushdata idx {pushdata_idx}")?;
}
write!(f, ": {}", self.error)
}
}
impl std::fmt::Display for ParseError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match &self {
ParseError::Empp(err) => write!(f, "eMPP error: {err}"),
ParseError::Slp(err) => write!(f, "SLP error: {err}"),
ParseError::Alp(err) => write!(f, "ALP error: {err}"),
}
}
}
diff --git a/chronik/bitcoinsuite-slp/src/verify.rs b/chronik/bitcoinsuite-slp/src/verify.rs
index 9f98fe19b..d05cc5dd2 100644
--- a/chronik/bitcoinsuite-slp/src/verify.rs
+++ b/chronik/bitcoinsuite-slp/src/verify.rs
@@ -1,451 +1,451 @@
// Copyright (c) 2023 The Bitcoin developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
//! Module for [`VerifyContext`].
use std::collections::BTreeMap;
use bitcoinsuite_core::script::Script;
use serde::{Deserialize, Serialize};
use thiserror::Error;
use crate::{
alp::consts::MAX_TX_INPUTS,
color::{ColoredTx, ColoredTxSection},
parsed::ParsedTxType,
structs::{GenesisInfo, Token, TokenMeta, TokenVariant, TxType},
token_tx::{TokenTx, TokenTxEntry},
token_type::{SlpTokenType, TokenType},
verify::BurnError::*,
};
/// Error how token verification based on inputs failed
#[derive(Clone, Debug, Deserialize, Eq, Error, Hash, PartialEq, Serialize)]
pub enum BurnError {
/// ALP has an upper limit on the number tx inputs.
/// Note that given current consensus rules, having this many inputs is not
/// possible, so this is a failsafe.
#[error("Too many tx inputs, got {0} but only {} allowed", MAX_TX_INPUTS)]
TooManyTxInputs(usize),
/// NFT1 CHILD GENESIS requires an NFT1 GROUP token in the first input
#[error("Invalid NFT1 CHILD GENESIS: No GROUP token")]
MissingNft1Group,
/// MINT requires a mint baton in the inputs
#[error("Missing MINT baton")]
MissingMintBaton,
/// MINT requires a mint vault input
#[error("Missing MINT vault")]
MissingMintVault,
/// SEND transfers cannot have more tokens in the outputs than are supplied
/// in the inputs.
#[error("Insufficient token input output sum: {actual} < {required}")]
InsufficientInputSum {
/// Required minimum inputs as specified in the outputs
required: u128,
/// Actual supplied token amount
actual: u128,
},
}
/// Token spent as an input
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct SpentToken {
/// Input token
pub token: Token,
/// GROUP token ID and type of the input, if any
pub group_token_meta: Option<TokenMeta>,
}
/// Context under which to verify a [`ColoredTx`].
#[derive(Debug)]
pub struct VerifyContext<'a> {
/// Input tokens of the tx
pub spent_tokens: &'a [Option<SpentToken>],
/// scriptPubKeys of the inputs of the tx, required only for SLP V2 MINT
/// txs
pub spent_scripts: Option<&'a [Script]>,
/// [`GenesisInfo`] of the tx's token ID, required only for SLP V2 MINT txs
pub genesis_info: Option<&'a GenesisInfo>,
/// Override whether a mint vault input is present by setting this to
/// `true`.
pub override_has_mint_vault: Option<bool>,
}
struct BareBurn {
burn_amount: u128,
burns_mint_batons: bool,
group_token_meta: Option<TokenMeta>,
is_invalid: bool,
}
impl VerifyContext<'_> {
/// Verify the [`ColoredTx`] under the given context and return a verified
/// [`TokenTx`].
pub fn verify(&self, tx: ColoredTx) -> TokenTx {
let mut entries = Vec::new();
for section in &tx.sections {
entries.push(self.verify_section(&tx, section));
}
// Add entries for standalone intentional burns without any
// corresponding existing sections
for intentional_burn in &tx.intentional_burns {
if !entries.iter().any(|burn| {
burn.meta.token_id == intentional_burn.meta.token_id
}) {
entries.push(TokenTxEntry {
meta: intentional_burn.meta,
tx_type: Some(TxType::BURN),
genesis_info: None,
group_token_meta: None,
is_invalid: false,
intentional_burn_amount: Some(intentional_burn.amount),
actual_burn_amount: 0,
burns_mint_batons: false,
burn_error: None,
has_colored_out_of_range: false,
failed_colorings: vec![],
});
}
}
let bare_burns = self.calc_bare_burns(&tx, &entries);
// Add failed colorings to the matching entry, or add a new one
for failed_coloring in tx.failed_colorings {
if let Some(entry) = entries
.iter_mut()
.find(|entry| entry.meta == failed_coloring.parsed.meta)
{
entry.failed_colorings.push(failed_coloring);
continue;
}
entries.push(TokenTxEntry {
meta: failed_coloring.parsed.meta,
tx_type: Some(failed_coloring.parsed.tx_type.tx_type()),
genesis_info: match &failed_coloring.parsed.tx_type {
ParsedTxType::Genesis(genesis) => {
Some(genesis.info.clone())
}
_ => None,
},
group_token_meta: None,
is_invalid: true,
intentional_burn_amount: None,
actual_burn_amount: 0,
burns_mint_batons: false,
burn_error: None,
has_colored_out_of_range: false,
failed_colorings: vec![failed_coloring],
});
}
// Update entries for bare burn or add them
for (burn_meta, bare_burn) in bare_burns {
if let Some(entry) =
entries.iter_mut().find(|entry| entry.meta == *burn_meta)
{
if bare_burn.burns_mint_batons {
entry.is_invalid = true;
}
entry.actual_burn_amount = bare_burn.burn_amount;
entry.burns_mint_batons = bare_burn.burns_mint_batons;
entry.group_token_meta = bare_burn.group_token_meta;
continue;
}
entries.push(TokenTxEntry {
meta: *burn_meta,
tx_type: None,
genesis_info: None,
group_token_meta: bare_burn.group_token_meta,
is_invalid: bare_burn.is_invalid,
intentional_burn_amount: None,
actual_burn_amount: bare_burn.burn_amount,
burns_mint_batons: bare_burn.burns_mint_batons,
burn_error: None,
has_colored_out_of_range: false,
failed_colorings: vec![],
});
}
let outputs = tx
.outputs
.into_iter()
.map(|output| -> Option<_> {
let entry = &entries[output.as_ref()?.token_idx];
if entry.is_invalid {
return None;
}
output
})
.collect::<Vec<_>>();
TokenTx {
entries,
outputs,
failed_parsings: tx.failed_parsings,
}
}
fn verify_section(
&self,
tx: &ColoredTx,
section: &ColoredTxSection,
) -> TokenTxEntry {
let input_sum = self.calc_input_sum(§ion.meta);
// Template entry with either zero defaults or copied over from the
// colored section.
let entry = TokenTxEntry {
meta: section.meta,
tx_type: Some(section.tx_type),
genesis_info: section.genesis_info.clone(),
group_token_meta: self.inherited_group_token_meta(§ion.meta),
is_invalid: false,
intentional_burn_amount: self
.intentional_burn_amount(tx, §ion.meta),
actual_burn_amount: 0,
burns_mint_batons: false,
burn_error: None,
has_colored_out_of_range: section.has_colored_out_of_range,
failed_colorings: vec![],
};
// ALP only allows up to 2**15 inputs
if section.meta.token_type.is_alp()
&& self.spent_tokens.len() > MAX_TX_INPUTS
{
return TokenTxEntry {
is_invalid: true,
actual_burn_amount: input_sum,
burns_mint_batons: self.has_mint_baton(§ion.meta),
burn_error: Some(TooManyTxInputs(self.spent_tokens.len())),
..entry
};
}
match section.tx_type {
// NFT1 CHILD tokens have to an NFT1 GROUP token at the 1st input
TxType::GENESIS
if section.meta.token_type
== TokenType::Slp(SlpTokenType::Nft1Child) =>
{
- match self.spent_tokens.get(0) {
+ match self.spent_tokens.first() {
Some(Some(spent_token))
if spent_token.token.meta.token_type
== TokenType::Slp(SlpTokenType::Nft1Group)
&& spent_token.token.variant.amount() > 0 =>
{
TokenTxEntry {
group_token_meta: Some(spent_token.token.meta),
..entry
}
}
_ => TokenTxEntry {
is_invalid: true,
burn_error: Some(MissingNft1Group),
..entry
},
}
}
// All other GENESIS txs are self-evident
TxType::GENESIS => entry,
// SLP V2 Mint Vault must have a given Script as input
TxType::MINT if section.is_mint_vault_mint() => {
if self.has_mint_vault() {
return TokenTxEntry {
actual_burn_amount: input_sum,
..entry
};
}
TokenTxEntry {
is_invalid: true,
actual_burn_amount: input_sum,
burn_error: Some(MissingMintVault),
..entry
}
}
// All other MINTs must have a matching mint baton
TxType::MINT => {
if self.has_mint_baton(§ion.meta) {
return TokenTxEntry {
actual_burn_amount: input_sum,
..entry
};
}
TokenTxEntry {
is_invalid: true,
actual_burn_amount: input_sum,
burn_error: Some(MissingMintBaton),
..entry
}
}
// SEND cannot spent more than came into the tx as inputs
TxType::SEND if input_sum < section.required_input_sum => {
TokenTxEntry {
is_invalid: true,
actual_burn_amount: input_sum,
burns_mint_batons: self.has_mint_baton(§ion.meta),
burn_error: Some(InsufficientInputSum {
required: section.required_input_sum,
actual: input_sum,
}),
..entry
}
}
// Valid SEND
TxType::SEND => {
let output_sum = self.calc_output_sum(tx, §ion.meta);
let actual_burn_amount = input_sum - output_sum;
TokenTxEntry {
actual_burn_amount,
burns_mint_batons: self.has_mint_baton(§ion.meta),
..entry
}
}
// UNKNOWN txs are self-evident
TxType::UNKNOWN => entry,
TxType::BURN => unreachable!(
"BURNs are only in intentional_burns, not in sections"
),
}
}
fn has_mint_baton(&self, meta: &TokenMeta) -> bool {
self.spent_tokens.iter().flatten().any(|spent_token| {
&spent_token.token.meta == meta
&& spent_token.token.variant.is_mint_baton()
})
}
fn has_mint_vault(&self) -> bool {
if let Some(override_has_mint_vault) = self.override_has_mint_vault {
return override_has_mint_vault;
}
let Some(spent_scripts) = self.spent_scripts else {
panic!(
"VerifyContext used incorrectly; spent_scripts must be \
present for SLP V2 Mint Vault token types"
);
};
let Some(genesis_info) = self.genesis_info else {
return false;
};
let Some(scripthash) = &genesis_info.mint_vault_scripthash else {
return false;
};
let script = Script::p2sh(scripthash);
spent_scripts
.iter()
.any(|spent_script| spent_script == &script)
}
fn calc_input_sum(&self, meta: &TokenMeta) -> u128 {
self.spent_tokens
.iter()
.flatten()
.filter(|token| &token.token.meta == meta)
.map(|token| token.token.variant.amount() as u128)
.sum()
}
fn calc_output_sum(&self, tx: &ColoredTx, meta: &TokenMeta) -> u128 {
tx.outputs
.iter()
.flatten()
.filter(|token| &tx.sections[token.token_idx].meta == meta)
.map(|token| token.variant.amount() as u128)
.sum()
}
fn inherited_group_token_meta(
&self,
meta: &TokenMeta,
) -> Option<TokenMeta> {
self.spent_tokens
.iter()
.flatten()
.find(|token| &token.token.meta == meta)
.and_then(|token| token.group_token_meta)
}
fn intentional_burn_amount(
&self,
tx: &ColoredTx,
meta: &TokenMeta,
) -> Option<u64> {
tx.intentional_burns
.iter()
.find(|burn| &burn.meta == meta)
.map(|burn| burn.amount)
}
// Bare burns: spent tokens without a corresponding section
fn calc_bare_burns(
&self,
tx: &ColoredTx,
entries: &[TokenTxEntry],
) -> BTreeMap<&TokenMeta, BareBurn> {
let mut bare_burns = BTreeMap::new();
for (input_idx, input) in self.spent_tokens.iter().enumerate() {
let Some(input) = input else { continue };
// Input has a corresponding mentioned section, not a bare burn
if tx
.sections
.iter()
.any(|section| section.meta == input.token.meta)
{
continue;
}
let bare_burn =
bare_burns.entry(&input.token.meta).or_insert(BareBurn {
burn_amount: 0,
burns_mint_batons: false,
group_token_meta: input.group_token_meta,
is_invalid: false,
});
// We don't consider NFT1 GROUP inputs for NFT1 CHILD GENESIS a burn
// At this stage the validation that the first input is an NFT1
// GROUP token is already done, otherwise is_invalid would be true.
// We still create a bare burn entry so that we get a TokenTxEntry,
// but leave is_invalid at false and don't increment the burned
// amount.
if input_idx == 0 {
if let Some(first_entry) = entries.first() {
if first_entry.meta.token_type
== TokenType::Slp(SlpTokenType::Nft1Child)
&& first_entry.tx_type == Some(TxType::GENESIS)
&& !first_entry.is_invalid
{
continue;
}
}
}
// All other bare burns are invalid
bare_burn.is_invalid = true;
match input.token.variant {
TokenVariant::Amount(amount) => {
bare_burn.burn_amount += u128::from(amount)
}
TokenVariant::MintBaton => bare_burn.burns_mint_batons = true,
TokenVariant::Unknown(_) => {}
}
}
bare_burns
}
}
diff --git a/chronik/chronik-db/src/io/spent_by.rs b/chronik/chronik-db/src/io/spent_by.rs
index 7d6fb01b3..619e19bfa 100644
--- a/chronik/chronik-db/src/io/spent_by.rs
+++ b/chronik/chronik-db/src/io/spent_by.rs
@@ -1,681 +1,678 @@
// Copyright (c) 2023 The Bitcoin developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
use std::{
collections::{hash_map::Entry, HashMap},
time::Instant,
};
use abc_rust_error::Result;
use rocksdb::{ColumnFamilyDescriptor, Options};
use serde::{Deserialize, Serialize};
use thiserror::Error;
use crate::{
db::{Db, WriteBatch, CF, CF_SPENT_BY},
index_tx::IndexTx,
io::{merge::catch_merge_errors, TxNum},
ser::{db_deserialize, db_deserialize_vec, db_serialize, db_serialize_vec},
};
/// Indicates an output has been spent by an input in a tx.
/// This is an entry in the list of spent outputs of a tx.
#[derive(
Clone,
Debug,
Default,
Deserialize,
Eq,
Hash,
Ord,
PartialEq,
PartialOrd,
Serialize,
)]
pub struct SpentByEntry {
/// Which output has been spent.
pub out_idx: u32,
/// Which tx spent the output.
pub tx_num: TxNum,
/// Which input of the spending tx spent the output.
pub input_idx: u32,
}
struct SpentByColumn<'a> {
db: &'a Db,
cf: &'a CF,
}
/// Write to the DB which outputs of a tx have been spent by which tx (and
/// input).
///
/// For each tx that has any output spent, there will be a list of entries in
/// the DB. Each entry tells us which output has been spent, which tx_num spent
/// it, and which input of that tx.
///
/// Note: While TxWriter writes keys using 8-byte big-endian numbers, this
/// writes it using [`db_serialize`], because unlike TxWriter, we don't rely on
/// any ordering. Since [`db_serialize`] is more compact, this saves some space.
#[derive(Debug)]
pub struct SpentByWriter<'a> {
col: SpentByColumn<'a>,
}
/// Read from the DB which outputs of a tx have been spent by which tx (and
/// input).
#[derive(Debug)]
pub struct SpentByReader<'a> {
col: SpentByColumn<'a>,
}
/// In-memory data for spent-by data.
#[derive(Debug, Default)]
pub struct SpentByMemData {
/// Stats about cache hits, num requests etc.
pub stats: SpentByStats,
}
/// Stats about cache hits, num requests etc.
#[derive(Clone, Debug, Default)]
pub struct SpentByStats {
/// Total number of txs updated.
pub n_total: usize,
/// Time [s] for insert/delete.
pub t_total: f64,
/// Time [s] for fetching txs.
pub t_fetch: f64,
}
/// Error indicating that something went wrong with writing spent-by data.
#[derive(Debug, Error, PartialEq, Eq)]
pub enum SpentByError {
/// Tried adding a spent-by entry, but it's already marked as spent by
/// another entry.
#[error(
"Inconsistent DB: Duplicate spend by entry for tx_num = {tx_num}: \
{existing:?} already exists, but tried to add {new:?}"
)]
DuplicateSpentByEntry {
/// An output of this tx_num has been spent
tx_num: TxNum,
/// Entry already present in the DB.
existing: SpentByEntry,
/// Entry we tried to add.
new: SpentByEntry,
},
/// Tried removing a spent-by entry, but it doesn't match what we got from
/// the disconnected block.
#[error(
"Inconsistent DB: Mismatched spend by entry for tx_num = {tx_num}: \
Expected {expected:?} to be present, but got {actual:?}"
)]
MismatchedSpentByEntry {
/// Tried removing a spent-by entry of an output of this tx_num.
tx_num: TxNum,
/// Entry we expected based on the disconnected block.
expected: SpentByEntry,
/// Entry actually found in the DB.
actual: SpentByEntry,
},
/// Tried removing a spent-by entry, but it doesn't exist.
#[error(
"Inconsistent DB: Missing spend by entry for tx_num = {tx_num}: \
Expected {expected:?} to be present, but none found"
)]
MissingSpentByEntry {
/// Tried removing a spent-by entry of an output of this tx_num.
tx_num: TxNum,
/// Entry we expected to be present based on the disconnected block.
expected: SpentByEntry,
},
}
use self::SpentByError::*;
fn ser_tx_num(tx_num: TxNum) -> Result<Vec<u8>> {
db_serialize(&tx_num)
}
fn deser_tx_num(bytes: &[u8]) -> Result<TxNum> {
db_deserialize(bytes)
}
fn init_merge_spent_by(
_key: &[u8],
existing_value: Option<&[u8]>,
_operands: &rocksdb::MergeOperands,
) -> Result<Vec<SpentByEntry>> {
match existing_value {
Some(bytes) => db_deserialize_vec::<SpentByEntry>(bytes),
None => Ok(vec![]),
}
}
fn apply_merge_spent_by(
key: &[u8],
entries: &mut Vec<SpentByEntry>,
operand: &[u8],
) -> Result<()> {
let extra_entries = db_deserialize_vec::<SpentByEntry>(operand)?;
entries.reserve(extra_entries.len());
for spent_by in extra_entries {
let search_idx = entries
.binary_search_by_key(&spent_by.out_idx, |entry| entry.out_idx);
match search_idx {
Ok(idx) => {
let input_tx_num = deser_tx_num(key)?;
// Output already spent by another tx -> corrupted DB?
return Err(DuplicateSpentByEntry {
tx_num: input_tx_num,
existing: entries[idx].clone(),
new: spent_by,
}
.into());
}
Err(insert_idx) => {
// No entry found -> insert it
entries.insert(insert_idx, spent_by);
}
}
}
Ok(())
}
fn ser_merge_spent_by(
_key: &[u8],
entries: Vec<SpentByEntry>,
) -> Result<Vec<u8>> {
db_serialize_vec::<SpentByEntry>(entries)
}
impl<'a> SpentByColumn<'a> {
fn new(db: &'a Db) -> Result<Self> {
let cf = db.cf(CF_SPENT_BY)?;
Ok(SpentByColumn { db, cf })
}
}
impl<'a> SpentByWriter<'a> {
/// Create a new [`SpentByWriter`].
pub fn new(db: &'a Db) -> Result<Self> {
let col = SpentByColumn::new(db)?;
Ok(SpentByWriter { col })
}
/// Add spent-by entries to txs spent in the txs.
/// For each tx output spent in `txs`, add which tx spent it.
pub fn insert(
&self,
batch: &mut WriteBatch,
txs: &[IndexTx<'_>],
mem_data: &mut SpentByMemData,
) -> Result<()> {
let stats = &mut mem_data.stats;
let t_start = Instant::now();
stats.n_total += txs.len();
let mut spent_by_map = HashMap::<TxNum, Vec<SpentByEntry>>::new();
for tx in txs {
if tx.is_coinbase {
// a coinbase doesn't spend anything
continue;
}
for (input_idx, (input, &input_tx_num)) in
tx.tx.inputs.iter().zip(tx.input_nums.iter()).enumerate()
{
let spent_by = SpentByEntry {
out_idx: input.prev_out.out_idx,
tx_num: tx.tx_num,
input_idx: input_idx as u32,
};
- spent_by_map
- .entry(input_tx_num)
- .or_insert(vec![])
- .push(spent_by);
+ spent_by_map.entry(input_tx_num).or_default().push(spent_by);
}
}
for (tx_num, entries) in spent_by_map {
batch.merge_cf(
self.col.cf,
ser_tx_num(tx_num)?,
db_serialize_vec::<SpentByEntry>(entries)?,
);
}
stats.t_total += t_start.elapsed().as_secs_f64();
Ok(())
}
/// Remove spent-by entries from txs spent in the txs.
/// For each tx output spent in `txs`, remove which tx spent it.
pub fn delete(
&self,
batch: &mut WriteBatch,
txs: &[IndexTx<'_>],
mem_data: &mut SpentByMemData,
) -> Result<()> {
let stats = &mut mem_data.stats;
let t_start = Instant::now();
stats.n_total += txs.len();
let mut spent_by_map = HashMap::<TxNum, Vec<SpentByEntry>>::new();
for tx in txs {
if tx.is_coinbase {
// a coinbase doesn't spend anything
continue;
}
for (input_idx, (input, &input_tx_num)) in
tx.tx.inputs.iter().zip(tx.input_nums.iter()).enumerate()
{
let spent_by = SpentByEntry {
out_idx: input.prev_out.out_idx,
tx_num: tx.tx_num,
input_idx: input_idx as u32,
};
let t_fetch = Instant::now();
let spent_by_entries =
self.get_or_fetch(&mut spent_by_map, input_tx_num)?;
stats.t_fetch += t_fetch.elapsed().as_secs_f64();
let search_idx = spent_by_entries
.binary_search_by_key(&spent_by.out_idx, |entry| {
entry.out_idx
});
match search_idx {
Ok(idx) => {
// Found the spent-by entry -> remove it.
if spent_by_entries[idx] != spent_by {
// Existing entry doesn't match what's in the DB.
return Err(MismatchedSpentByEntry {
tx_num: input_tx_num,
expected: spent_by,
actual: spent_by_entries[idx].clone(),
}
.into());
}
spent_by_entries.remove(idx);
}
Err(_) => {
// Spent-by entry not found, but should be there.
return Err(MissingSpentByEntry {
tx_num: input_tx_num,
expected: spent_by,
}
.into());
}
}
}
}
for (tx_num, entries) in spent_by_map {
let ser_num = ser_tx_num(tx_num)?;
if entries.is_empty() {
batch.delete_cf(self.col.cf, ser_num);
} else {
batch.put_cf(self.col.cf, ser_num, db_serialize_vec(entries)?);
}
}
stats.t_total += t_start.elapsed().as_secs_f64();
Ok(())
}
fn get_or_fetch<'b>(
&self,
spent_by_map: &'b mut HashMap<TxNum, Vec<SpentByEntry>>,
tx_num: TxNum,
) -> Result<&'b mut Vec<SpentByEntry>> {
match spent_by_map.entry(tx_num) {
Entry::Occupied(entry) => Ok(entry.into_mut()),
Entry::Vacant(entry) => {
let db_entries =
match self.col.db.get(self.col.cf, ser_tx_num(tx_num)?)? {
Some(data) => {
db_deserialize_vec::<SpentByEntry>(&data)?
}
None => vec![],
};
Ok(entry.insert(db_entries))
}
}
}
pub(crate) fn add_cfs(columns: &mut Vec<ColumnFamilyDescriptor>) {
let mut options = Options::default();
options.set_merge_operator(
"spent_by::merge_op",
catch_merge_errors(
init_merge_spent_by,
apply_merge_spent_by,
ser_merge_spent_by,
),
|_, _, _| None,
);
columns.push(ColumnFamilyDescriptor::new(CF_SPENT_BY, options));
}
}
impl<'a> SpentByReader<'a> {
/// Create a new [`SpentByReader`].
pub fn new(db: &'a Db) -> Result<Self> {
let col = SpentByColumn::new(db)?;
Ok(SpentByReader { col })
}
/// Query the spent-by entries by [`TxNum`].
pub fn by_tx_num(
&self,
tx_num: TxNum,
) -> Result<Option<Vec<SpentByEntry>>> {
match self.col.db.get(self.col.cf, ser_tx_num(tx_num)?)? {
Some(data) => Ok(Some(db_deserialize_vec::<SpentByEntry>(&data)?)),
None => Ok(None),
}
}
}
impl std::fmt::Debug for SpentByColumn<'_> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("SpentByColumn")
.field("db", &self.db)
.field("cf", &"..")
.finish()
}
}
#[cfg(test)]
mod tests {
use std::cell::RefCell;
use abc_rust_error::Result;
use bitcoinsuite_core::tx::Tx;
use crate::{
db::{Db, WriteBatch},
index_tx::prepare_indexed_txs,
io::{
merge::{check_for_errors, MERGE_ERROR_LOCK},
BlockTxs, SpentByEntry, SpentByError, SpentByMemData,
SpentByReader, SpentByWriter, TxEntry, TxWriter, TxsMemData,
},
test::make_inputs_tx,
};
#[test]
fn test_spent_by() -> Result<()> {
let _guard = MERGE_ERROR_LOCK.lock().unwrap();
abc_rust_error::install();
let tempdir = tempdir::TempDir::new("chronik-db--spent_by")?;
let mut cfs = Vec::new();
TxWriter::add_cfs(&mut cfs);
SpentByWriter::add_cfs(&mut cfs);
let db = Db::open_with_cfs(tempdir.path(), cfs)?;
let tx_writer = TxWriter::new(&db)?;
let spent_by_writer = SpentByWriter::new(&db)?;
let spent_by_reader = SpentByReader::new(&db)?;
let mem_data = RefCell::new(SpentByMemData::default());
let txs_mem_data = RefCell::new(TxsMemData::default());
let block_height = RefCell::new(-1);
let txs_batch = |txs: &[Tx]| BlockTxs {
txs: txs
.iter()
.map(|tx| TxEntry {
txid: tx.txid(),
..Default::default()
})
.collect(),
block_height: *block_height.borrow(),
};
let connect_block = |txs: &[Tx]| -> Result<()> {
let mut batch = WriteBatch::default();
*block_height.borrow_mut() += 1;
let first_tx_num = tx_writer.insert(
&mut batch,
&txs_batch(txs),
&mut txs_mem_data.borrow_mut(),
)?;
let index_txs = prepare_indexed_txs(&db, first_tx_num, txs)?;
spent_by_writer.insert(
&mut batch,
&index_txs,
&mut mem_data.borrow_mut(),
)?;
db.write_batch(batch)?;
for tx in &index_txs {
for &input_tx_num in &tx.input_nums {
spent_by_reader.by_tx_num(input_tx_num)?;
check_for_errors()?;
}
}
Ok(())
};
let disconnect_block = |txs: &[Tx]| -> Result<()> {
let mut batch = WriteBatch::default();
let first_tx_num = tx_writer.delete(
&mut batch,
&txs_batch(txs),
&mut txs_mem_data.borrow_mut(),
)?;
let index_txs = prepare_indexed_txs(&db, first_tx_num, txs)?;
spent_by_writer.delete(
&mut batch,
&index_txs,
&mut mem_data.borrow_mut(),
)?;
db.write_batch(batch)?;
*block_height.borrow_mut() -= 1;
Ok(())
};
macro_rules! spent_by {
(out_idx=$out_idx:literal -> tx_num=$tx_num:literal,
input_idx=$input_idx:literal) => {
SpentByEntry {
out_idx: $out_idx,
tx_num: $tx_num,
input_idx: $input_idx,
}
};
}
let block0 =
vec![make_inputs_tx(0, [(0x00, u32::MAX, -1)], [-1, -1, -1, -1])];
connect_block(&block0)?;
let block1 = vec![
make_inputs_tx(1, [(0x00, u32::MAX, -1)], [-1, -1]),
// spend 3rd output of tx_num=0
make_inputs_tx(2, [(0, 3, -1)], [-1, -1, -1]),
];
connect_block(&block1)?;
assert_eq!(
spent_by_reader.by_tx_num(0)?,
Some(vec![spent_by!(out_idx=3 -> tx_num=2, input_idx=0)]),
);
// Remove block1
disconnect_block(&block1)?;
assert_eq!(spent_by_reader.by_tx_num(0)?, None);
// Add block1 again
connect_block(&block1)?;
let block2 = vec![
make_inputs_tx(3, [(0x00, u32::MAX, -1)], [-1]),
// spend 2nd output of tx_num=0, and 0th output of tx_num=1
make_inputs_tx(4, [(0, 2, -1), (1, 0, -1)], [-1, -1, -1]),
// spend 1st output of tx_num=1, and 1st output of tx_num=0
make_inputs_tx(5, [(1, 1, -1), (0, 1, -1)], [-1, -1]),
];
connect_block(&block2)?;
assert_eq!(
spent_by_reader.by_tx_num(0)?,
Some(vec![
spent_by!(out_idx=1 -> tx_num=5, input_idx=1),
spent_by!(out_idx=2 -> tx_num=4, input_idx=0),
spent_by!(out_idx=3 -> tx_num=2, input_idx=0),
]),
);
assert_eq!(
spent_by_reader.by_tx_num(1)?,
Some(vec![
spent_by!(out_idx=0 -> tx_num=4, input_idx=1),
spent_by!(out_idx=1 -> tx_num=5, input_idx=0),
]),
);
// More complex block
let block3 = vec![
make_inputs_tx(6, [(0x00, u32::MAX, -1)], [-1]),
make_inputs_tx(7, [(4, 0, -1), (2, 0, -1)], [-1, -1]),
make_inputs_tx(8, [(9, 0, -1), (5, 1, -1)], [-1, -1]),
make_inputs_tx(9, [(7, 1, -1), (8, 1, -1)], [-1]),
];
connect_block(&block3)?;
assert_eq!(
spent_by_reader.by_tx_num(2)?,
Some(vec![spent_by!(out_idx=0 -> tx_num=7, input_idx=1)]),
);
assert_eq!(
spent_by_reader.by_tx_num(4)?,
Some(vec![spent_by!(out_idx=0 -> tx_num=7, input_idx=0)]),
);
assert_eq!(
spent_by_reader.by_tx_num(5)?,
Some(vec![spent_by!(out_idx=1 -> tx_num=8, input_idx=1)]),
);
assert_eq!(spent_by_reader.by_tx_num(6)?, None);
assert_eq!(
spent_by_reader.by_tx_num(7)?,
Some(vec![spent_by!(out_idx=1 -> tx_num=9, input_idx=0)]),
);
assert_eq!(
spent_by_reader.by_tx_num(8)?,
Some(vec![spent_by!(out_idx=1 -> tx_num=9, input_idx=1)]),
);
assert_eq!(
spent_by_reader.by_tx_num(9)?,
Some(vec![spent_by!(out_idx=0 -> tx_num=8, input_idx=0)]),
);
disconnect_block(&block3)?;
assert_eq!(
spent_by_reader.by_tx_num(0)?,
Some(vec![
spent_by!(out_idx=1 -> tx_num=5, input_idx=1),
spent_by!(out_idx=2 -> tx_num=4, input_idx=0),
spent_by!(out_idx=3 -> tx_num=2, input_idx=0),
]),
);
assert_eq!(
spent_by_reader.by_tx_num(1)?,
Some(vec![
spent_by!(out_idx=0 -> tx_num=4, input_idx=1),
spent_by!(out_idx=1 -> tx_num=5, input_idx=0),
]),
);
assert_eq!(spent_by_reader.by_tx_num(2)?, None);
assert_eq!(spent_by_reader.by_tx_num(4)?, None);
assert_eq!(spent_by_reader.by_tx_num(5)?, None);
assert_eq!(spent_by_reader.by_tx_num(6)?, None);
assert_eq!(spent_by_reader.by_tx_num(7)?, None);
assert_eq!(spent_by_reader.by_tx_num(8)?, None);
assert_eq!(spent_by_reader.by_tx_num(9)?, None);
// failed connect: duplicate entry
let block_duplicate_spend = vec![
make_inputs_tx(10, [(0x00, u32::MAX, -1)], []),
make_inputs_tx(11, [(0, 1, -1)], []),
];
assert_eq!(
connect_block(&block_duplicate_spend)
.unwrap_err()
.downcast::<SpentByError>()?,
SpentByError::DuplicateSpentByEntry {
tx_num: 0,
existing: spent_by!(out_idx=1 -> tx_num=5, input_idx=1),
new: spent_by!(out_idx=1 -> tx_num=7, input_idx=0),
},
);
// Ensure failed connect didn't have any side-effects on spent-by data
assert_eq!(
spent_by_reader.by_tx_num(0)?,
Some(vec![
spent_by!(out_idx=1 -> tx_num=5, input_idx=1),
spent_by!(out_idx=2 -> tx_num=4, input_idx=0),
spent_by!(out_idx=3 -> tx_num=2, input_idx=0),
]),
);
assert_eq!(
spent_by_reader.by_tx_num(1)?,
Some(vec![
spent_by!(out_idx=0 -> tx_num=4, input_idx=1),
spent_by!(out_idx=1 -> tx_num=5, input_idx=0),
]),
);
// Undo side effects introduced by failed connect
disconnect_block(&[
make_inputs_tx(10, [(0x00, u32::MAX, -1)], []),
// Note the missing input values
make_inputs_tx(11, [], []),
])?;
// failed disconnect: mismatched entry
let block_mismatched_spend = vec![
make_inputs_tx(3, [(0x00, u32::MAX, -1)], []),
make_inputs_tx(4, [(0, 1, -1)], []),
];
assert_eq!(
disconnect_block(&block_mismatched_spend)
.unwrap_err()
.downcast::<SpentByError>()?,
SpentByError::MismatchedSpentByEntry {
tx_num: 0,
expected: spent_by!(out_idx=1 -> tx_num=4, input_idx=0),
actual: spent_by!(out_idx=1 -> tx_num=5, input_idx=1),
},
);
// failed disconnect: missing entry
let block_missing_spend = vec![
make_inputs_tx(3, [(0x00, u32::MAX, -1)], []),
make_inputs_tx(4, [(3, 1, -1)], []),
];
assert_eq!(
disconnect_block(&block_missing_spend)
.unwrap_err()
.downcast::<SpentByError>()?,
SpentByError::MissingSpentByEntry {
tx_num: 3,
expected: spent_by!(out_idx=1 -> tx_num=4, input_idx=0),
},
);
// disconnect blocks
disconnect_block(&block2)?;
assert_eq!(
spent_by_reader.by_tx_num(0)?,
Some(vec![spent_by!(out_idx=3 -> tx_num=2, input_idx=0)]),
);
assert_eq!(spent_by_reader.by_tx_num(1)?, None);
assert_eq!(spent_by_reader.by_tx_num(2)?, None);
assert_eq!(spent_by_reader.by_tx_num(3)?, None);
assert_eq!(spent_by_reader.by_tx_num(4)?, None);
assert_eq!(spent_by_reader.by_tx_num(5)?, None);
disconnect_block(&block1)?;
assert_eq!(spent_by_reader.by_tx_num(0)?, None);
assert_eq!(spent_by_reader.by_tx_num(1)?, None);
assert_eq!(spent_by_reader.by_tx_num(2)?, None);
assert_eq!(spent_by_reader.by_tx_num(3)?, None);
disconnect_block(&block0)?;
assert_eq!(spent_by_reader.by_tx_num(0)?, None);
assert_eq!(spent_by_reader.by_tx_num(1)?, None);
drop(db);
rocksdb::DB::destroy(&rocksdb::Options::default(), tempdir.path())?;
let _ = check_for_errors();
Ok(())
}
}
diff --git a/chronik/chronik-db/src/io/token/batch.rs b/chronik/chronik-db/src/io/token/batch.rs
index e914c19bb..b6910fbed 100644
--- a/chronik/chronik-db/src/io/token/batch.rs
+++ b/chronik/chronik-db/src/io/token/batch.rs
@@ -1,463 +1,463 @@
// Copyright (c) 2024 The Bitcoin developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
use std::collections::{BTreeMap, HashMap, HashSet};
use abc_rust_error::Result;
use bimap::BiMap;
use bitcoinsuite_core::{script::Script, tx::OutPoint};
use bitcoinsuite_slp::{
color::ColoredTx,
structs::{GenesisInfo, Token, TokenMeta, TokenVariant, TxType},
token_tx::TokenTx,
token_type::TokenType,
verify::{SpentToken, VerifyContext},
};
use itertools::{Either, Itertools};
use thiserror::Error;
use topo_sort::TopoSort;
use crate::{
index_tx::IndexTx,
io::{
token::{BatchError::*, DbToken, DbTokenTx, FLAGS_HAS_MINT_VAULT},
TxNum,
},
};
/// Tx that has token data encoded in it and is ready for token validation
#[derive(Debug)]
pub struct PreparedTx<'tx> {
/// Tx with index data (tx_num etc.)
pub tx: &'tx IndexTx<'tx>,
/// Parsed & colored tx. Note that in this context we only store txs with a
/// non-empty list of sections.
pub colored: ColoredTx,
}
/// Struct bundling all the data necessary to process a batch (i.e. a block) of
/// token txs.
#[derive(Debug)]
pub struct BatchProcessor<'tx> {
/// Tx that have any token info attached
pub prepared_txs: HashMap<TxNum, PreparedTx<'tx>>,
/// Non-token txs may still have token inputs
pub non_token_txs: Vec<&'tx IndexTx<'tx>>,
/// Whether the batch has any GENESIS txs; if the token index is empty and
/// we have no GENESIS, we can safely ignore the batch.
pub has_any_genesis: bool,
}
/// DB data required to process a batch of txs
#[derive(Debug)]
pub struct BatchDbData {
/// Token tx data coming from the DB
pub token_txs: BTreeMap<TxNum, DbTokenTx>,
/// token_nums assigned for each [`TokenMeta`]. This is a bi-map so we can
/// lookup either direction.
pub token_metas: BiMap<TxNum, TokenMeta>,
/// [`GenesisInfo`] from the database; required for SLP V2 Mint Vault txs
pub genesis_infos: HashMap<TokenMeta, GenesisInfo>,
}
#[derive(Debug, Default)]
/// Result of the batch verification before inserting
pub struct ProcessedTokenTxBatch {
/// New tokens to be added to the DB
pub new_tokens: Vec<(TxNum, TokenMeta, GenesisInfo)>,
/// New DB data for txs to be added to the DB.
pub db_token_txs: HashMap<TxNum, DbTokenTx>,
/// Validated token txs in the batch.
pub valid_txs: HashMap<TxNum, TokenTx>,
/// Validated spent tokens in the batch, can have entries not it valid_txs
pub spent_tokens: HashMap<TxNum, Vec<Option<SpentToken>>>,
/// True if validation of txs was performed, or false if validation was
/// safely skipped because no tokens are in the DB and the batch contained
/// no token txs.
pub did_validation: bool,
}
/// Error when batch-processing txs, usually implies a critical failure
#[derive(Debug, Error, PartialEq)]
pub enum BatchError {
/// Transactions couldn't be ordered topologically because of a cycle
/// dependency. Note: This is cryptographically impossible in practice,
/// because of the irreversability of SHA256.
#[error("Cycle in SLP txs")]
Cycle,
/// A token_num that should be in the DB was not found
#[error("Inconsistent BatchDbData: Missing TokenId for token tx num {0}")]
MissingTokenTxNum(TxNum),
/// GenesisInfo that should be in the DB was not found
#[error("Inconsistent Tx: Missing coin for TxInput {0:?}")]
MissingTxInputCoin(OutPoint),
}
impl<'tx> BatchProcessor<'tx> {
/// Prepare the given indexed txs as token txs.
pub fn prepare(txs: &'tx [IndexTx<'tx>]) -> Self {
let (prepared_txs, non_token_txs): (HashMap<_, _>, Vec<_>) = txs
.iter()
.partition_map(|tx| match ColoredTx::color_tx(tx.tx) {
Some(colored) if !colored.sections.is_empty() => {
Either::Left((tx.tx_num, PreparedTx { tx, colored }))
}
_ => Either::Right(tx),
});
// Coloring step ensures all GENESIS are at the first section
let has_any_genesis = prepared_txs
.values()
.any(|tx| tx.colored.sections[0].tx_type == TxType::GENESIS);
BatchProcessor {
prepared_txs,
non_token_txs,
has_any_genesis,
}
}
/// Collect all [`TokenMeta`]s of the SLP V2 Mint Vault txs in the batch
pub fn collect_mint_vault_metas(&self) -> HashSet<TokenMeta> {
self.prepared_txs
.values()
.filter(|tx| tx.colored.sections[0].is_mint_vault_mint())
.map(|tx| tx.colored.sections[0].meta)
.collect()
}
/// Verify the entire batch of txs. It updates the DB data with some token
/// data during validation, but that can be discarded. The result of the
/// verification is returned as [`ProcessedTokenTxBatch`].
pub fn verify(
mut self,
mut db_data: BatchDbData,
) -> Result<ProcessedTokenTxBatch> {
// Build a DAG of tx nums so we can sort topologically
let mut topo_sort = TopoSort::with_capacity(self.prepared_txs.len());
for (&tx_num, batch_tx) in &self.prepared_txs {
topo_sort.insert_from_slice(tx_num, &batch_tx.tx.input_nums);
}
// Iterate txs in topological order
let mut processed_batch = ProcessedTokenTxBatch {
did_validation: true,
..Default::default()
};
for tx_num in topo_sort.into_nodes() {
let tx_num = tx_num.map_err(|_| Cycle)?;
let prepared_tx = self.prepared_txs.remove(&tx_num).unwrap();
self.verify_token_tx(
prepared_tx,
&mut db_data,
&mut processed_batch,
)?;
}
// Non-token txs can still contain token inputs, add those to the index
// too
for non_token_tx in &self.non_token_txs {
self.process_non_token_tx(
non_token_tx,
&mut db_data,
&mut processed_batch,
)?;
}
Ok(processed_batch)
}
fn verify_token_tx(
&self,
prepared_tx: PreparedTx<'_>,
db_data: &mut BatchDbData,
processed_batch: &mut ProcessedTokenTxBatch,
) -> Result<()> {
let tx_num = prepared_tx.tx.tx_num;
let spent_tokens = self.tx_token_inputs(
prepared_tx.tx,
db_data,
&processed_batch.valid_txs,
)?;
let first_section = &prepared_tx.colored.sections[0];
let is_genesis = first_section.genesis_info.is_some();
let is_mint_vault_mint = first_section.is_mint_vault_mint();
// MINT txs on SLP V2 tokens need spent scripts and genesis data
let mut spent_scripts = None;
let mut genesis_info = None;
if is_mint_vault_mint {
spent_scripts = Some(Self::tx_spent_scripts(prepared_tx.tx)?);
genesis_info = db_data.genesis_infos.get(&first_section.meta);
}
let context = VerifyContext {
spent_tokens: &spent_tokens,
spent_scripts: spent_scripts.as_deref(),
genesis_info,
override_has_mint_vault: None,
};
let valid_tx = context.verify(prepared_tx.colored);
let has_any_inputs = spent_tokens.iter().any(|input| input.is_some());
let has_any_outputs =
valid_tx.outputs.iter().any(|token| token.is_some());
// Don't store txs that have no actual token inputs or outputs
if !has_any_outputs && !has_any_inputs && !is_genesis {
return Ok(());
}
// Add new tokens from GENESIS txs
- if let Some(entry) = valid_tx.entries.get(0) {
+ if let Some(entry) = valid_tx.entries.first() {
// Skip invalid GENESIS txs
if !entry.is_invalid {
if let Some(info) = &entry.genesis_info {
db_data.token_metas.insert(tx_num, entry.meta);
processed_batch.new_tokens.push((
tx_num,
entry.meta,
info.clone(),
));
// Note: Don't update db_data.genesis_info, because SLP V2
// GENESIS txs require a confirmation before they take
// effect.
}
}
}
let mut token_tx_nums = Vec::new();
let mut token_metas = Vec::new();
let mut group_token_metas = BTreeMap::new();
for entry in &valid_tx.entries {
let Some(&token_tx_num) =
db_data.token_metas.get_by_right(&entry.meta)
else {
continue;
};
if !token_metas.iter().contains(&entry.meta) {
token_tx_nums.push(token_tx_num);
token_metas.push(entry.meta);
}
entry.group_token_meta.and_then(|group_meta| {
let &tx_num = db_data.token_metas.get_by_right(&group_meta)?;
if !token_metas.iter().contains(&group_meta) {
token_tx_nums.push(tx_num);
token_metas.push(group_meta);
group_token_metas.insert(entry.meta, group_meta);
}
Some(())
});
}
let mut flags = 0;
if is_mint_vault_mint {
- let first_entry = valid_tx.entries.get(0);
+ let first_entry = valid_tx.entries.first();
let has_mint_vault =
first_entry.map_or(false, |entry| !entry.is_invalid);
if has_mint_vault {
flags |= FLAGS_HAS_MINT_VAULT;
}
}
let db_token_tx = DbTokenTx {
token_tx_nums,
group_token_indices: group_token_metas
.iter()
.map(|(meta, group_meta)| {
(
meta_idx(meta, &token_metas),
meta_idx(group_meta, &token_metas),
)
})
.collect(),
inputs: spent_tokens
.iter()
.map(|input| {
to_db_token(
input.as_ref().map(|input| &input.token),
&token_metas,
)
})
.collect::<Vec<_>>(),
outputs: valid_tx
.outputs
.iter()
.map(|output| {
to_db_token(
output
.as_ref()
.map(|output| valid_tx.token(output))
.as_ref(),
&token_metas,
)
})
.collect::<Vec<_>>(),
flags,
};
processed_batch.db_token_txs.insert(tx_num, db_token_tx);
processed_batch.valid_txs.insert(tx_num, valid_tx);
processed_batch.spent_tokens.insert(tx_num, spent_tokens);
Ok(())
}
fn process_non_token_tx(
&self,
tx: &IndexTx<'_>,
db_data: &mut BatchDbData,
processed_batch: &mut ProcessedTokenTxBatch,
) -> Result<()> {
let mut db_token_tx_nums = Vec::new();
let mut db_inputs = Vec::with_capacity(tx.input_nums.len());
let mut db_group_token_indices = BTreeMap::new();
for (&input_tx_num, input) in tx.input_nums.iter().zip(&tx.tx.inputs) {
let out_idx = input.prev_out.out_idx as usize;
let db_token_tx = processed_batch
.db_token_txs
.get(&input_tx_num)
.or_else(|| db_data.token_txs.get(&input_tx_num));
let Some(db_token_tx) = db_token_tx else {
continue;
};
let db_token = &db_token_tx.outputs[out_idx];
let Some(token_tx_num) = db_token_tx.token_tx_num(db_token) else {
db_inputs.push(*db_token);
continue;
};
let token_num_idx = db_token_tx_nums
.iter()
.position(|&tx_num| tx_num == token_tx_num)
.unwrap_or_else(|| {
db_token_tx_nums.push(token_tx_num);
db_token_tx_nums.len() - 1
});
if let Some(group_token_tx_num) =
db_token_tx.group_token_tx_num(db_token)
{
let group_token_num_idx = db_token_tx_nums
.iter()
.position(|&tx_num| tx_num == group_token_tx_num)
.unwrap_or_else(|| {
db_token_tx_nums.push(group_token_tx_num);
db_token_tx_nums.len() - 1
});
db_group_token_indices
.insert(token_num_idx as u32, group_token_num_idx as u32);
}
db_inputs.push(db_token.with_idx(token_num_idx as u32));
}
// Skip non-token tx if we don't have any token inputs
if db_inputs.iter().any(|&input| input != DbToken::NoToken) {
processed_batch.db_token_txs.insert(
tx.tx_num,
DbTokenTx {
token_tx_nums: db_token_tx_nums,
group_token_indices: db_group_token_indices,
inputs: db_inputs,
outputs: vec![DbToken::NoToken; tx.tx.outputs.len()],
flags: 0,
},
);
processed_batch.spent_tokens.insert(
tx.tx_num,
self.tx_token_inputs(tx, db_data, &processed_batch.valid_txs)?,
);
}
Ok(())
}
fn tx_spent_scripts(tx: &IndexTx<'_>) -> Result<Vec<Script>> {
let mut spent_scripts = Vec::with_capacity(tx.tx.inputs.len());
for tx_input in &tx.tx.inputs {
let coin = tx_input
.coin
.as_ref()
.ok_or(MissingTxInputCoin(tx_input.prev_out))?;
spent_scripts.push(coin.output.script.clone());
}
Ok(spent_scripts)
}
fn tx_token_inputs(
&self,
tx: &IndexTx<'_>,
db_data: &BatchDbData,
valid_txs: &HashMap<TxNum, TokenTx>,
) -> Result<Vec<Option<SpentToken>>> {
if tx.is_coinbase {
Ok(vec![])
} else {
let mut inputs = Vec::with_capacity(tx.input_nums.len());
for (&input_num, input) in tx.input_nums.iter().zip(&tx.tx.inputs) {
inputs.push(self.token_output(
input_num,
input.prev_out.out_idx as usize,
db_data,
valid_txs,
)?);
}
Ok(inputs)
}
}
fn token_output(
&self,
tx_num: TxNum,
out_idx: usize,
db_data: &BatchDbData,
valid_txs: &HashMap<TxNum, TokenTx>,
) -> Result<Option<SpentToken>> {
// Output is from this batch
if let Some(token_tx) = valid_txs.get(&tx_num) {
let Some(Some(token_output)) = token_tx.outputs.get(out_idx) else {
return Ok(None);
};
return Ok(Some(token_tx.spent_token(token_output)));
}
// Output is from the DB
let Some(db_token_tx) = db_data.token_txs.get(&tx_num) else {
return Ok(None);
};
Ok(db_token_tx.spent_token(
&db_token_tx.outputs[out_idx],
|tx_num| {
db_data
.token_metas
.get_by_left(&tx_num)
.cloned()
.ok_or(MissingTokenTxNum(tx_num))
},
)?)
}
}
fn meta_idx(needle_meta: &TokenMeta, metas: &[TokenMeta]) -> u32 {
metas
.iter()
.position(|meta| meta == needle_meta)
.expect("TokenMeta should be in the list") as u32
}
fn to_db_token(token: Option<&Token>, metas: &[TokenMeta]) -> DbToken {
let Some(token) = token else {
return DbToken::NoToken;
};
match token.variant {
TokenVariant::Amount(amount) => {
DbToken::Amount(meta_idx(&token.meta, metas), amount)
}
TokenVariant::MintBaton => {
DbToken::MintBaton(meta_idx(&token.meta, metas))
}
TokenVariant::Unknown(token_type) => match token.meta.token_type {
TokenType::Slp(_) => DbToken::UnknownSlp(token_type),
TokenType::Alp(_) => DbToken::UnknownAlp(token_type),
},
}
}
diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml
index 133fb4759..1ba2fb332 100644
--- a/contrib/gitian-descriptors/gitian-linux.yml
+++ b/contrib/gitian-descriptors/gitian-linux.yml
@@ -1,245 +1,245 @@
---
name: "bitcoin-abc-linux"
enable_cache: true
distro: "debian"
suites:
- "bullseye"
architectures:
- "amd64"
packages:
- "autoconf"
- "automake"
- "binutils-aarch64-linux-gnu"
- "binutils-arm-linux-gnueabihf"
- "binutils-gold"
- "bison"
- "bsdmainutils"
- "ca-certificates"
- "clang"
- "cmake"
- "curl"
- "faketime"
# Use gcc/g++ 9 to avoid introducing the new pthread_cond_clockwait from glibc
# 2.30, which would make our release binary incompatible with systems using an
# older glibc version.
- "g++-9"
- "g++-9-aarch64-linux-gnu"
- "g++-9-arm-linux-gnueabihf"
- "gcc-9"
- "gcc-9-aarch64-linux-gnu"
- "gcc-9-arm-linux-gnueabihf"
- "git"
- "gperf"
# Needed for Rocksdb
- "libc6-dev-i386"
- "libtool"
- "ninja-build"
- "pkg-config"
- "protobuf-compiler"
- "python3"
- "python3-pip"
remotes:
- "url": "https://github.com/Bitcoin-ABC/bitcoin-abc.git"
"dir": "bitcoin"
files: []
script: |
WRAP_DIR=$HOME/wrapped
HOSTS=(
x86_64-linux-gnu
arm-linux-gnueabihf
aarch64-linux-gnu
)
# CMake toolchain file name differ from host name
declare -A CMAKE_TOOLCHAIN_FILE
CMAKE_TOOLCHAIN_FILE[x86_64-linux-gnu]=Linux64.cmake
CMAKE_TOOLCHAIN_FILE[arm-linux-gnueabihf]=LinuxARM.cmake
CMAKE_TOOLCHAIN_FILE[aarch64-linux-gnu]=LinuxAArch64.cmake
# Allow extra cmake option to be specified for each host
declare -A CMAKE_EXTRA_OPTIONS
# ARM assembly is supported but experimental, disable it for the release
CMAKE_EXTRA_OPTIONS[arm-linux-gnueabihf]="-DSECP256K1_USE_ASM=OFF"
FAKETIME_HOST_PROGS=""
FAKETIME_PROGS="date ar ranlib nm"
HOST_CFLAGS="-O2 -g"
HOST_CXXFLAGS="-O2 -g"
HOST_LDFLAGS=-static-libstdc++
export TZ="UTC"
export BUILD_DIR=`pwd`
mkdir -p ${WRAP_DIR}
if test -n "$GBUILD_CACHE_ENABLED"; then
export SOURCES_PATH=${GBUILD_COMMON_CACHE}
export BASE_CACHE=${GBUILD_PACKAGE_CACHE}
mkdir -p ${BASE_CACHE} ${SOURCES_PATH}
fi
function create_global_faketime_wrappers {
for prog in ${FAKETIME_PROGS}; do
echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog}
echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog}
echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog}
echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog}
echo "\$REAL \$@" >> $WRAP_DIR/${prog}
chmod +x ${WRAP_DIR}/${prog}
done
}
function create_per-host_faketime_wrappers {
for i in ${HOSTS[@]}; do
for prog in ${FAKETIME_HOST_PROGS}; do
echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog}
echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog}
echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog}
echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog}
echo "\$REAL \"\$@\"" >> $WRAP_DIR/${i}-${prog}
chmod +x ${WRAP_DIR}/${i}-${prog}
done
done
}
function create_per-host_compiler_wrapper {
for i in ${HOSTS[@]}; do
for prog in gcc g++; do
echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog}
echo "REAL=\`which -a ${i}-${prog}-9 | head -1\`" >> ${WRAP_DIR}/${i}-${prog}
echo "\$REAL \"\$@\"" >> $WRAP_DIR/${i}-${prog}
chmod +x ${WRAP_DIR}/${i}-${prog}
done
done
}
function create_native_compiler_wrapper {
for prog in gcc g++; do
echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog}
echo "REAL=\`which -a x86_64-linux-gnu-${prog}-9 | head -1\`" >> ${WRAP_DIR}/${prog}
echo "\$REAL \"\$@\"" >> $WRAP_DIR/${prog}
chmod +x ${WRAP_DIR}/${prog}
done
}
pip3 install lief==0.13.2
# Faketime for depends so intermediate results are comparable
export PATH_orig=${PATH}
create_global_faketime_wrappers "2000-01-01 12:00:00"
create_per-host_faketime_wrappers "2000-01-01 12:00:00"
# Wrap the compiler <host>-gcc-9 and <host>-g++-9 into <host>-gcc and
# <host>-g++
create_per-host_compiler_wrapper
# For the current host platform also wrap into regular gcc and g++, assume
# x86_64
create_native_compiler_wrapper
export PATH=${WRAP_DIR}:${PATH}
cd bitcoin
SOURCEDIR=`pwd`
BASEPREFIX=`pwd`/depends
# Build dependencies for each host
for i in ${HOSTS[@]}; do
make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}"
done
# Faketime for binaries
export PATH=${PATH_orig}
create_global_faketime_wrappers "${REFERENCE_DATETIME}"
create_per-host_faketime_wrappers "${REFERENCE_DATETIME}"
export PATH=${WRAP_DIR}:${PATH}
mkdir -p source_package
pushd source_package
# Any toolchain file will work for building the source package, just pick the
# first one
cmake -GNinja .. \
-DCMAKE_TOOLCHAIN_FILE=${SOURCEDIR}/cmake/platforms/${CMAKE_TOOLCHAIN_FILE[${HOSTS[0]}]}
ninja package_source
SOURCEDIST=`echo bitcoin-abc-*.tar.gz`
mv ${SOURCEDIST} ..
popd
DISTNAME=`echo ${SOURCEDIST} | sed 's/.tar.*//'`
# Correct tar file order
mkdir -p temp
pushd temp
tar -xf ../$SOURCEDIST
find bitcoin-abc-* | sort | tar --mtime="${REFERENCE_DATETIME}" --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ../$SOURCEDIST
popd
ORIGPATH="$PATH"
# Install chronik dependencies
# Rust
curl -sSf https://static.rust-lang.org/rustup/archive/1.26.0/x86_64-unknown-linux-gnu/rustup-init -o rustup-init
echo "0b2f6c8f85a3d02fde2efc0ced4657869d73fccfce59defb4e8d29233116e6db rustup-init" | sha256sum -c
chmod +x rustup-init
- ./rustup-init -y --default-toolchain=1.72.0
+ ./rustup-init -y --default-toolchain=1.76.0
# Rust target name differs from our host name, let's map
declare -A RUST_TARGET
RUST_TARGET[x86_64-linux-gnu]=x86_64-unknown-linux-gnu
RUST_TARGET[arm-linux-gnueabihf]=arm-unknown-linux-gnueabihf
RUST_TARGET[aarch64-linux-gnu]=aarch64-unknown-linux-gnu
for i in ${HOSTS[@]}; do
$HOME/.cargo/bin/rustup target add ${RUST_TARGET[${i}]}
done
# Cleanup
rm -f rustup-init
# Extend the hosts to include an experimental chronik build.
# Despite not being a new host per se, it makes it easy to reuse the same code
# and prevent errors.
# TODO Remove after chronik is made part of the main release.
for i in ${HOSTS[@]}; do
HOSTS+=($i-chronik-experimental)
CMAKE_TOOLCHAIN_FILE[$i-chronik-experimental]=${CMAKE_TOOLCHAIN_FILE[$i]}
CMAKE_EXTRA_OPTIONS[$i-chronik-experimental]="${CMAKE_EXTRA_OPTIONS[$i]} -DBUILD_BITCOIN_CHRONIK=ON"
done
# Extract the release tarball into a dir for each host and build
for i in ${HOSTS[@]}; do
export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH}
mkdir -p distsrc-${i}
cd distsrc-${i}
INSTALLPATH=`pwd`/installed/${DISTNAME}
mkdir -p ${INSTALLPATH}
cmake -GNinja .. \
-DCMAKE_TOOLCHAIN_FILE=${SOURCEDIR}/cmake/platforms/${CMAKE_TOOLCHAIN_FILE[${i}]} \
-DCLIENT_VERSION_IS_RELEASE=ON \
-DENABLE_CLANG_TIDY=OFF \
-DENABLE_REDUCE_EXPORTS=ON \
-DENABLE_STATIC_LIBSTDCXX=ON \
-DENABLE_GLIBC_BACK_COMPAT=ON \
-DCMAKE_INSTALL_PREFIX=${INSTALLPATH} \
-DCCACHE=OFF \
-DUSE_LINKER= \
${CMAKE_EXTRA_OPTIONS[${i}]}
ninja
ninja security-check
# TODO Rust pulls several symbols from GLIBC 2.30, this needs to be fixed.
# Since it is still in an experimental state, ignore for now.
if [[ "${i}" != *"chronik-experimental" ]]; then
ninja symbol-check
else
# Install the chronik protobuf files
ninja install-chronik-proto
fi
ninja install-debug
cd installed
find ${DISTNAME} -not -name "*.dbg" | sort | tar --mtime="${REFERENCE_DATETIME}" --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}.tar.gz
find ${DISTNAME} -name "*.dbg" | sort | tar --mtime="${REFERENCE_DATETIME}" --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}-debug.tar.gz
cd ../../
rm -rf distsrc-${i}
done
mkdir -p $OUTDIR/src
mv $SOURCEDIST $OUTDIR/src
diff --git a/contrib/utils/install-dependencies-bullseye.sh b/contrib/utils/install-dependencies-bullseye.sh
index 2be4f5d6a..e6c2ed07b 100755
--- a/contrib/utils/install-dependencies-bullseye.sh
+++ b/contrib/utils/install-dependencies-bullseye.sh
@@ -1,212 +1,212 @@
#!/usr/bin/env bash
export LC_ALL=C.UTF-8
set -euxo pipefail
dpkg --add-architecture i386
PACKAGES=(
arcanist
automake
autotools-dev
binutils
bison
bsdmainutils
build-essential
ccache
cmake
curl
default-jdk
devscripts
doxygen
dput
g++-9
g++-9-aarch64-linux-gnu
g++-9-arm-linux-gnueabihf
g++-9-multilib
g++-mingw-w64
gcc-9
gcc-9-aarch64-linux-gnu
gcc-9-arm-linux-gnueabihf
gcc-9-multilib
gettext-base
git
golang
gnupg
graphviz
gperf
help2man
jq
lcov
less
lib32stdc++-10-dev
libboost-dev
libbz2-dev
libc6-dev:i386
libcap-dev
libdb++-dev
libdb-dev
libevent-dev
libjemalloc-dev
libminiupnpc-dev
libnatpmp-dev
libprotobuf-dev
libpcsclite-dev
libqrencode-dev
libqt5core5a
libqt5dbus5
libqt5gui5
libsqlite3-dev
libssl-dev
libtinfo5
libtool
libzmq3-dev
lld
make
ninja-build
nsis
pandoc
php-codesniffer
pkg-config
protobuf-compiler
python3
python3-pip
python3-setuptools
python3-yaml
python3-zmq
qttools5-dev
qttools5-dev-tools
shellcheck
software-properties-common
swig
tar
wget
xorriso
xvfb
yamllint
)
function join_by() {
local IFS="$1"
shift
echo "$*"
}
apt-get update
DEBIAN_FRONTEND=noninteractive apt-get install -y $(join_by ' ' "${PACKAGES[@]}")
BACKPORTS=(
git-filter-repo
qemu-user-static
)
echo "deb http://deb.debian.org/debian bullseye-backports main" | tee -a /etc/apt/sources.list
apt-get update
DEBIAN_FRONTEND=noninteractive apt-get -t bullseye-backports install -y $(join_by ' ' "${BACKPORTS[@]}")
# Install llvm and clang
apt-key add "$(dirname "$0")"/llvm.pub
add-apt-repository "deb https://apt.llvm.org/bullseye/ llvm-toolchain-bullseye-12 main"
apt-get update
LLVM_PACKAGES=(
clang-12
clang-format-12
clang-tidy-12
clang-tools-12
)
DEBIAN_FRONTEND=noninteractive apt-get install -y $(join_by ' ' "${LLVM_PACKAGES[@]}")
# Make sure our specific llvm and clang versions have highest priority
update-alternatives --install /usr/bin/clang clang "$(command -v clang-12)" 100
update-alternatives --install /usr/bin/clang++ clang++ "$(command -v clang++-12)" 100
update-alternatives --install /usr/bin/llvm-symbolizer llvm-symbolizer "$(command -v llvm-symbolizer-12)" 100
# Use gcc-9/g++-9 by default so it uses libstdc++-9. This prevents from pulling
# the new pthread_cond_clockwait symbol from GLIBC_30 and ensure we are testing
# under the same condition our release it built.
update-alternatives --install /usr/bin/gcc gcc "$(command -v gcc-9)" 100
update-alternatives --install /usr/bin/g++ g++ "$(command -v g++-9)" 100
update-alternatives --install /usr/bin/aarch64-linux-gnu-gcc aarch64-linux-gnu-gcc "$(command -v aarch64-linux-gnu-gcc-9)" 100
update-alternatives --install /usr/bin/aarch64-linux-gnu-g++ aarch64-linux-gnu-g++ "$(command -v aarch64-linux-gnu-g++-9)" 100
update-alternatives --install /usr/bin/arm-linux-gnueabihf-gcc arm-linux-gnueabihf-gcc "$(command -v arm-linux-gnueabihf-gcc-9)" 100
update-alternatives --install /usr/bin/arm-linux-gnueabihf-g++ arm-linux-gnueabihf-g++ "$(command -v arm-linux-gnueabihf-g++-9)" 100
# Use the mingw posix variant
update-alternatives --set x86_64-w64-mingw32-g++ $(command -v x86_64-w64-mingw32-g++-posix)
update-alternatives --set x86_64-w64-mingw32-gcc $(command -v x86_64-w64-mingw32-gcc-posix)
# Python library for merging nested structures
pip3 install deepmerge
# For running Python test suites
pip3 install pytest
# For en/-decoding protobuf messages
# This version is compatible with Debian's "protobuf-compiler" package
pip3 install "protobuf<=3.20"
# For security-check.py and symbol-check.py
pip3 install "lief==0.13.2"
# For Chronik WebSocket endpoint
pip3 install websocket-client
# Required python linters
pip3 install black==23.3.0 isort==5.6.4 mypy==0.910 flynt==0.78 flake8==6.0.0
echo "export PATH=\"$(python3 -m site --user-base)/bin:\$PATH\"" >> ~/.bashrc
# shellcheck source=/dev/null
source ~/.bashrc
# Install npm v10.x and nodejs v20.x
wget https://deb.nodesource.com/setup_20.x -O nodesetup.sh
echo "f8fb478685fb916cc70858200595a4f087304bcde1e69aa713bf2eb41695afc1 nodesetup.sh" | sha256sum -c
chmod +x nodesetup.sh
./nodesetup.sh
apt-get install -y nodejs
# Install nyc for mocha unit test reporting
npm i -g nyc
-# Install Rust stable 1.72.0 and nightly from the 2023-12-29
+# Install Rust stable 1.76.0 and nightly from the 2023-12-29
curl -sSf https://static.rust-lang.org/rustup/archive/1.26.0/x86_64-unknown-linux-gnu/rustup-init -o rustup-init
echo "0b2f6c8f85a3d02fde2efc0ced4657869d73fccfce59defb4e8d29233116e6db rustup-init" | sha256sum -c
chmod +x rustup-init
-./rustup-init -y --default-toolchain=1.72.0
+./rustup-init -y --default-toolchain=1.76.0
RUST_HOME="${HOME}/.cargo/bin"
RUST_NIGHTLY_DATE=2023-12-29
"${RUST_HOME}/rustup" install nightly-${RUST_NIGHTLY_DATE}
"${RUST_HOME}/rustup" component add rustfmt --toolchain nightly-${RUST_NIGHTLY_DATE}
# Name the nightly toolchain "abc-nightly"
"${RUST_HOME}/rustup" toolchain link abc-nightly "$(${RUST_HOME}/rustc +nightly-${RUST_NIGHTLY_DATE} --print sysroot)"
# Install required compile platform targets on stable
"${RUST_HOME}/rustup" target add "i686-unknown-linux-gnu" \
"x86_64-unknown-linux-gnu" \
"aarch64-unknown-linux-gnu" \
"arm-unknown-linux-gnueabihf" \
"x86_64-apple-darwin" \
"x86_64-pc-windows-gnu"
# Install Electrum ABC test dependencies
here=$(dirname -- "$(readlink -f -- "${BASH_SOURCE[0]}")")
pip3 install -r "${here}/../../electrum/contrib/requirements/requirements.txt"
pip3 install -r "${here}/../../electrum/contrib/requirements/requirements-regtest.txt"
pip3 install -r "${here}/../../electrum/contrib/requirements/requirements-hw.txt"
# Install the winehq-staging version of wine that doesn't suffer from the memory
# limitations of the previous versions. Installation instructions are from
# https://wiki.winehq.org/Debian
mkdir -p /etc/apt/keyrings
wget -O /etc/apt/keyrings/winehq-archive.key https://dl.winehq.org/wine-builds/winehq.key
wget -NP /etc/apt/sources.list.d/ https://dl.winehq.org/wine-builds/debian/dists/bullseye/winehq-bullseye.sources
apt-get update
WINE_VERSION=8.19~bullseye-1
# We need all the packages and dependencies to use a pinpointed vesion
WINE_PACKAGES=(
winehq-staging
wine-staging
wine-staging-amd64
wine-staging-i386
)
# Pinpoint the version so we get consistent results on CI
DEBIAN_FRONTEND=noninteractive apt-get install -y $(join_by ' ' "${WINE_PACKAGES[@]/%/=${WINE_VERSION}}")
File Metadata
Details
Attached
Mime Type
text/x-diff
Expires
Sun, Apr 27, 11:04 (1 d, 10 h)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
5573354
Default Alt Text
(99 KB)
Attached To
rABC Bitcoin ABC
Event Timeline
Log In to Comment