diff --git a/Cargo.toml b/Cargo.toml index ac859662c..a690b1f47 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,21 +1,21 @@ # Copyright (c) 2022 The Bitcoin developers [workspace] members = [ "chronik/abc-rust-error", "chronik/abc-rust-lint", "chronik/bitcoinsuite-core", "chronik/bitcoinsuite-slp", "chronik/chronik-bridge", "chronik/chronik-db", "chronik/chronik-http", "chronik/chronik-indexer", "chronik/chronik-lib", "chronik/chronik-plugin", "chronik/chronik-proto", "chronik/chronik-util", ] [workspace.package] -rust-version = "1.72.0" +rust-version = "1.76.0" diff --git a/chronik/CMakeLists.txt b/chronik/CMakeLists.txt index dd4d3de01..83fd25afc 100644 --- a/chronik/CMakeLists.txt +++ b/chronik/CMakeLists.txt @@ -1,210 +1,211 @@ # Copyright (c) 2022 The Bitcoin developers set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) include(FetchContent) FetchContent_Declare( Corrosion GIT_REPOSITORY https://github.com/corrosion-rs/corrosion.git GIT_TAG v0.4.7 ) FetchContent_MakeAvailable(Corrosion) -set(REQUIRED_RUST_VERSION "1.72.0") +set(REQUIRED_RUST_VERSION "1.76.0") if(Rust_VERSION VERSION_LESS REQUIRED_RUST_VERSION) message(FATAL_ERROR "Minimum required Rust version is " - "${REQUIRED_RUST_VERSION}, but found ${Rust_VERSION}") + "${REQUIRED_RUST_VERSION}, but found ${Rust_VERSION}. " + "Use `rustup update stable` to update.") endif() set(CARGO_BUILD_DIR "${CMAKE_BINARY_DIR}/cargo/build") set_property(DIRECTORY "${CMAKE_SOURCE_DIR}" APPEND PROPERTY ADDITIONAL_CLEAN_FILES "${CARGO_BUILD_DIR}" ) get_property( RUSTC_EXECUTABLE TARGET Rust::Rustc PROPERTY IMPORTED_LOCATION ) get_filename_component(RUST_BIN_DIR ${RUSTC_EXECUTABLE} DIRECTORY) include(DoOrFail) find_program_or_fail(RUSTDOC_EXECUTABLE rustdoc PATHS "${RUST_BIN_DIR}" ) set(CHRONIK_CARGO_FLAGS --locked) if(BUILD_BITCOIN_CHRONIK_PLUGINS) set(CHRONIK_FEATURE_FLAGS --features plugins) endif() function(add_cargo_custom_target TARGET) add_custom_target(${TARGET} COMMAND "${CMAKE_COMMAND}" -E env CARGO_TARGET_DIR="${CARGO_BUILD_DIR}" CARGO_BUILD_RUSTC="$" CARGO_BUILD_RUSTDOC="${RUSTDOC_EXECUTABLE}" "$" ${CHRONIK_CARGO_FLAGS} ${ARGN} WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" ) endfunction() function(add_crate_test_targets CRATE) set(CRATE_TEST_TARGET "check-crate-${CRATE}") add_custom_target("${CRATE_TEST_TARGET}") set(CLIPPY_TARGET "${CRATE_TEST_TARGET}-clippy") add_cargo_custom_target("${CLIPPY_TARGET}" clippy --package "${CRATE}-*" -- -D warnings ) set(TEST_TARGET "${CRATE_TEST_TARGET}-test") add_cargo_custom_target("${TEST_TARGET}" test --package "${CRATE}-*" ) add_dependencies("${CRATE_TEST_TARGET}" "${CLIPPY_TARGET}" "${TEST_TARGET}" ) add_dependencies("check-crates" "${CRATE_TEST_TARGET}" ) endfunction() add_custom_target("check-crates") add_crate_test_targets(abc-rust) add_crate_test_targets(bitcoinsuite) add_crate_test_targets(chronik) # Compile Rust, generates chronik-lib corrosion_import_crate( MANIFEST_PATH "chronik-lib/Cargo.toml" FLAGS ${CHRONIK_CARGO_FLAGS} ${CHRONIK_FEATURE_FLAGS} ) set(Rust_TRIPLE "${Rust_CARGO_TARGET_ARCH}" "${Rust_CARGO_TARGET_VENDOR}" "${Rust_CARGO_TARGET_OS}" ) if (Rust_CARGO_TARGET_ENV) list(APPEND Rust_TRIPLE "${Rust_CARGO_TARGET_ENV}") endif() list(JOIN Rust_TRIPLE "-" Rust_CARGO_TARGET) # cxx crate generates some source files at this location set(CXXBRIDGE_GENERATED_FOLDER "${CARGO_BUILD_DIR}/${Rust_CARGO_TARGET}/cxxbridge") set(CHRONIK_BRIDGE_GENERATED_CPP_FILES "${CXXBRIDGE_GENERATED_FOLDER}/chronik-bridge/src/ffi.rs.cc") set(CHRONIK_LIB_GENERATED_CPP_FILES "${CXXBRIDGE_GENERATED_FOLDER}/chronik-lib/src/ffi.rs.cc") add_custom_command( OUTPUT ${CHRONIK_BRIDGE_GENERATED_CPP_FILES} ${CHRONIK_LIB_GENERATED_CPP_FILES} COMMAND "${CMAKE_COMMAND}" -E env "echo" "Generating cxx bridge files" DEPENDS $ ) # Chronik-bridge library # Contains all the C++ functions used by Rust, and the code bridging both add_library(chronik-bridge chronik-cpp/chronik_bridge.cpp chronik-cpp/util/hash.cpp ${CHRONIK_BRIDGE_GENERATED_CPP_FILES} ) target_include_directories(chronik-bridge PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}" "${CXXBRIDGE_GENERATED_FOLDER}" ) target_link_libraries(chronik-bridge util leveldb ) # Chronik library # Compiles and links all the Chronik code, and exposes chronik::Start and # chronik::Stop to run the indexer from C++. add_library(chronik chronik-cpp/chronik.cpp chronik-cpp/chronik_validationinterface.cpp ${CHRONIK_LIB_GENERATED_CPP_FILES} ) target_link_libraries(chronik chronik-bridge chronik-lib ) # Plugins require us to link agains libpython if(BUILD_BITCOIN_CHRONIK_PLUGINS) find_package(Python COMPONENTS Interpreter Development) message("Adding Python_LIBRARIES: ${Python_LIBRARIES}") target_link_libraries(chronik ${Python_LIBRARIES}) endif() if(${CMAKE_SYSTEM_NAME} MATCHES "Windows") # mio crate (dependency of tokio) requires winternl.h, found in ntdll find_package(NTDLL REQUIRED) target_link_libraries(chronik NTDLL::ntdll) # rocksdb requires items from rpcdce.h, found in rpcrt4 find_package(RPCRT4 REQUIRED) target_link_libraries(chronik RPCRT4::rpcrt4) endif() # Rocksdb requires "atomic" include(AddCompilerFlags) custom_check_linker_flag(LINKER_HAS_ATOMIC "-latomic") if(LINKER_HAS_ATOMIC) target_link_libraries(chronik atomic) endif() # Add chronik to server target_link_libraries(server chronik # TODO: We need to add the library again, otherwise gcc linking fails. # It's not clear yet why this is the case. chronik-bridge ) # Install the directory containing the proto files. The trailing slash ensures # the directory is not duplicated (see # https://cmake.org/cmake/help/v3.16/command/install.html#installing-directories) set(CHRONIK_PROTO_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/chronik-proto/proto/") set(CHRONIK_PROTO_COMPONENT "chronik-proto") install( DIRECTORY "${CHRONIK_PROTO_DIRECTORY}" DESTINATION "proto" COMPONENT "${CHRONIK_PROTO_COMPONENT}" ) add_custom_target("install-${CHRONIK_PROTO_COMPONENT}" COMMENT "Installing component ${CHRONIK_PROTO_COMPONENT}" COMMAND "${CMAKE_COMMAND}" -E env CMAKE_INSTALL_ALWAYS=ON "${CMAKE_COMMAND}" -DCOMPONENT="${CHRONIK_PROTO_COMPONENT}" -DCMAKE_INSTALL_PREFIX="${CMAKE_INSTALL_PREFIX}" -P cmake_install.cmake WORKING_DIRECTORY "${CMAKE_BINARY_DIR}" ) diff --git a/chronik/bitcoinsuite-slp/src/color.rs b/chronik/bitcoinsuite-slp/src/color.rs index a2c9b3a78..67fe44478 100644 --- a/chronik/bitcoinsuite-slp/src/color.rs +++ b/chronik/bitcoinsuite-slp/src/color.rs @@ -1,596 +1,596 @@ // Copyright (c) 2023 The Bitcoin developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. //! Module for [`ColoredTx`]. use bitcoinsuite_core::tx::{Tx, TxId}; use bytes::Bytes; use thiserror::Error; use crate::{ alp, color::ColorError::*, empp, parsed::{ParsedData, ParsedGenesis, ParsedMintData, ParsedTxType}, slp, structs::{ Amount, GenesisInfo, Token, TokenMeta, TokenOutput, TokenVariant, TxType, }, token_id::TokenId, token_type::{SlpTokenType, TokenType}, }; /// A parsed SLP or ALP tx with outputs colored according to the tokens /// specified in the `OP_RETURN`. #[derive(Clone, Debug, Default, PartialEq)] pub struct ColoredTx { /// Parsed sections defining where tokens should go. /// Can be multiple for ALP, at most 1 for SLP. pub sections: Vec, /// Intentional burns, specifying how many tokens are supposed to be burned /// of which type. pub intentional_burns: Vec, /// Outputs colored with the tokens as specified in the `OP_RETURN`. pub outputs: Vec>, /// Reports of failed parsing attempts pub failed_parsings: Vec, /// Reports of failed coloring attempts pub failed_colorings: Vec, } /// Section defining where tokens should go as specified in the `OP_RETURN`. #[derive(Clone, Debug, Eq, PartialEq)] pub struct ColoredTxSection { /// [`TokenMeta`] specified in the section. pub meta: TokenMeta, /// [`TxType`] specified in the section. pub tx_type: TxType, /// Minimum required sum of input tokens of this token meta. /// Note that this may be different from the `outputs` in [`ColoredTx`] of /// the respective token meta for SLP, as SLP allows sending tokens to /// nonexistent outputs, whereas in ALP this would be a failed coloring. pub required_input_sum: u128, /// SLP allows coloring non-existent outputs, but this usually is a /// mistake, so we keep track of it here. pub has_colored_out_of_range: bool, /// [`GenesisInfo`] introduced in this section. Only present for GENESIS. pub genesis_info: Option, } /// Any kind of parse error that can occur when processing a tx. #[derive(Clone, Debug, Eq, PartialEq)] pub enum ParseError { /// Parsing the OP_RETURN as eMPP failed Empp(empp::ParseError), /// Parsing the OP_RETURN as SLP failed Slp(slp::ParseError), /// Parsing a pushdata in an OP_RETURN as ALP failed Alp(alp::ParseError), } /// Report of a failed parsing attempt #[derive(Clone, Debug, Eq, PartialEq)] pub struct FailedParsing { /// Which pushdata in the OP_RETURN failed to parse, or None if the entire /// OP_RETURN is the culprit. pub pushdata_idx: Option, /// The actual bytes that failed to parse. pub bytes: Bytes, /// Error explaining why the parsing failed. pub error: ParseError, } /// Report of a failed coloring attempt #[derive(Clone, Debug, Eq, PartialEq)] pub struct FailedColoring { /// Which pushdata in the OP_RETURN failed to color the tx. pub pushdata_idx: usize, /// Parsed data which failed to color. pub parsed: ParsedData, /// Error explaining why the coloring failed. pub error: ColorError, } /// Intentional burn, allowing users to specify cleanly and precisely how tokens /// should be removed from supply. /// /// This prevents the bells and whistles of indexers and wallets to reject a tx /// whose intent is to remove tokens from supply. #[derive(Clone, Debug, Eq, Hash, PartialEq)] pub struct IntentionalBurn { /// Which token meta should be burned pub meta: TokenMeta, /// How many tokens should be burned pub amount: Amount, } /// Error when trying to color a parsed section. #[derive(Clone, Debug, Error, Eq, PartialEq)] pub enum ColorError { /// ALP disallows coloring non-existent outputs #[error("Too few outputs, expected {expected} but got {actual}")] TooFewOutputs { /// Expected number of outputs for the coloring to succeed expected: usize, /// Actual number of outputs actual: usize, }, /// GENESIS must be first #[error("GENESIS must be the first pushdata")] GenesisMustBeFirst, /// Token types must be ascending, to allow clean upgrades when introducing /// new token types. #[error( "Descending token type: {before} > {after}, token types must be in \ ascending order" )] DescendingTokenType { /// Larger token type coming before before: u8, /// Smaller token type coming after after: u8, }, /// Tried coloring using the same token ID twice, which is not allowed. /// Only the first coloring counts. #[error( "Duplicate token_id {token_id}, found in section {prev_section_idx}" )] DuplicateTokenId { /// Valid section that first colored the tx prev_section_idx: usize, /// Token ID that was colored twice token_id: TokenId, }, /// Tried doing an intentional burn of the same token ID twice, which is /// not allowed. Only the first intentional burn counts. #[error( "Duplicate intentional burn token_id {token_id}, found in burn \ #{prev_burn_idx} and #{burn_idx}" )] DuplicateIntentionalBurnTokenId { /// Valid previous intentional burn prev_burn_idx: usize, /// Invalid later duplicate burn burn_idx: usize, /// Token ID burned token_id: TokenId, }, /// Outputs cannot be colored twice by different sections #[error( "Overlapping amount when trying to color {amount} at index \ {output_idx}, output is already colored with {prev_token}" )] OverlappingAmount { /// Previous token the output is already colored with prev_token: Token, /// Index of the output that we tried to color twice output_idx: usize, /// Amount that tried to color an output twice amount: Amount, }, /// Outputs cannot be colored twice by different sections #[error( "Overlapping mint baton when trying to color mint baton at index \ {output_idx}, output is already colored with {prev_token}" )] OverlappingMintBaton { /// Previous token the output is already colored with prev_token: Token, /// Index of the output that we tried tot color twice output_idx: usize, }, } impl ColoredTx { /// Parse the OP_RETURN of the tx and color its outputs pub fn color_tx(tx: &Tx) -> Option { - let op_return = match tx.outputs.get(0) { + let op_return = match tx.outputs.first() { Some(output) if output.script.is_opreturn() => &output.script, _ => return None, }; let mut colored = ColoredTx { sections: vec![], intentional_burns: vec![], outputs: vec![None; tx.outputs.len()], failed_colorings: vec![], failed_parsings: vec![], }; // First, try to parse and color as SLP tx match slp::parse(tx.txid_ref(), op_return) { Ok(Some(parsed)) => { colored .color_section(0, parsed, &mut 0) .expect("Coloring SLP always succeeds"); return Some(colored); } Ok(None) => {} Err(slp_error) => { colored.failed_parsings.push(FailedParsing { pushdata_idx: None, bytes: op_return.bytecode().clone(), error: ParseError::Slp(slp_error), }); return Some(colored); } } // Then, try to parse as eMPP tx let pushdata = match empp::parse(op_return) { Ok(Some(pushdata)) => pushdata, Ok(None) => return None, Err(empp_err) => { colored.failed_parsings.push(FailedParsing { pushdata_idx: None, bytes: op_return.bytecode().clone(), error: ParseError::Empp(empp_err), }); return Some(colored); } }; // Color all the pushdata as ALP, and if we encountered any ALP // sections, return the colored tx. if colored.color_all_alp_pushdata(pushdata, tx.txid_ref()) { return Some(colored); } // We found an eMPP OP_RETURN but no ALP sections None } // Color all the pushdata as ALP one-by-one and return whether we // encountered anything ALP-like. fn color_all_alp_pushdata( &mut self, pushdata: Vec, txid: &TxId, ) -> bool { let mut max_token_type = 0; let mut has_any_alp = false; for (pushdata_idx, pushdata) in pushdata.into_iter().enumerate() { let parsed = match alp::parse_section(txid, pushdata.clone()) { Ok(Some(parsed)) => parsed, Ok(None) => continue, Err(alp_error) => { self.failed_parsings.push(FailedParsing { pushdata_idx: Some(pushdata_idx), bytes: pushdata, error: ParseError::Alp(alp_error), }); has_any_alp = true; continue; } }; has_any_alp = true; let color_result = self.color_section( pushdata_idx, parsed.clone(), &mut max_token_type, ); if let Err(error) = color_result { self.failed_colorings.push(FailedColoring { pushdata_idx, parsed, error, }); } } has_any_alp } fn color_section( &mut self, pushdata_idx: usize, parsed: ParsedData, max_token_type: &mut u8, ) -> Result<(), ColorError> { let meta = parsed.meta; // token_type must be in ascending order if *max_token_type > meta.token_type.to_u8() { return Err(DescendingTokenType { before: *max_token_type, after: meta.token_type.to_u8(), }); } *max_token_type = meta.token_type.to_u8(); // Only report duplicate token IDs on MINT and SEND, burn and GENESIS // are handled separately if matches!(parsed.tx_type.tx_type(), TxType::MINT | TxType::SEND) { for (prev_section_idx, prev_section) in self.sections.iter().enumerate() { if prev_section.meta.token_id == meta.token_id { return Err(DuplicateTokenId { prev_section_idx, token_id: meta.token_id, }); } } } match parsed.tx_type { ParsedTxType::Genesis(genesis) => { self.color_genesis(pushdata_idx, meta, genesis) } ParsedTxType::Mint(mint) => self.color_mint(meta, mint), ParsedTxType::Send(send) => self.color_send(meta, send), ParsedTxType::Burn(amount) => self.color_burn(meta, amount), ParsedTxType::Unknown => self.color_unknown(meta), } } fn color_genesis( &mut self, pushdata_idx: usize, meta: TokenMeta, genesis: ParsedGenesis, ) -> Result<(), ColorError> { // GENESIS must be the very first section in the pushdata. // This prevents assigning the same token ID to different tokens, even // if we introduced a new LOKAD ID, as long as it also upholds this // rule. if pushdata_idx != 0 { return Err(GenesisMustBeFirst); } let has_colored_out_of_range = self.color_mint_data(&meta, &genesis.mint_data)?; self.sections.push(ColoredTxSection { meta, tx_type: TxType::GENESIS, required_input_sum: 0, genesis_info: Some(genesis.info), has_colored_out_of_range, }); Ok(()) } fn color_mint( &mut self, meta: TokenMeta, mint: ParsedMintData, ) -> Result<(), ColorError> { let has_colored_out_of_range = self.color_mint_data(&meta, &mint)?; self.sections.push(ColoredTxSection { meta, tx_type: TxType::MINT, required_input_sum: 0, genesis_info: None, has_colored_out_of_range, }); Ok(()) } fn color_mint_data( &mut self, meta: &TokenMeta, mint_data: &ParsedMintData, ) -> Result { let token_idx = self.sections.len(); let mut out_of_range_idx = None; // Verify no outputs have been colored already for (output_idx, &amount) in mint_data.amounts_range().zip(&mint_data.amounts) { if amount != 0 { match self.outputs.get(output_idx) { Some(Some(token)) => { return Err(OverlappingAmount { prev_token: self.token(token), output_idx, amount, }); } Some(None) => {} None => out_of_range_idx = Some(output_idx), } } } for output_idx in mint_data.batons_range() { match self.outputs.get(output_idx) { Some(Some(token)) => { return Err(OverlappingMintBaton { prev_token: self.token(token), output_idx, }) } Some(None) => {} None => out_of_range_idx = Some(output_idx), } } if let Some(output_idx) = out_of_range_idx { // ALP forbids amounts and batons for nonexistent outputs if meta.token_type.is_alp() { return Err(TooFewOutputs { expected: output_idx + 1, actual: self.outputs.len(), }); } } // Now, color all outputs for (output_idx, &amount) in mint_data.amounts_range().zip(&mint_data.amounts) { if output_idx >= self.outputs.len() { break; } if amount > 0 { self.outputs[output_idx] = Some(TokenOutput { token_idx, variant: TokenVariant::Amount(amount), }); } } for output_idx in mint_data.batons_range() { if output_idx >= self.outputs.len() { break; } self.outputs[output_idx] = Some(TokenOutput { token_idx, variant: TokenVariant::MintBaton, }); } Ok(out_of_range_idx.is_some()) } fn color_send( &mut self, meta: TokenMeta, amounts: Vec, ) -> Result<(), ColorError> { // Verify no outputs have been colored already let mut out_of_range_idx = None; for (idx, &amount) in amounts.iter().enumerate() { if amount != 0 { match self.outputs.get(idx + 1) { Some(Some(token)) => { return Err(OverlappingAmount { prev_token: self.token(token), output_idx: idx + 1, amount, }) } Some(None) => {} None => out_of_range_idx = Some(idx + 1), } } } if let Some(output_idx) = out_of_range_idx { // ALP forbids amounts and batons for nonexistent outputs if meta.token_type.is_alp() { return Err(TooFewOutputs { expected: output_idx + 1, actual: self.outputs.len(), }); } } // Color outputs and also calculate the required input sum let mut required_input_sum = 0u128; for (idx, &amount) in amounts.iter().enumerate() { if amount == 0 { continue; } required_input_sum += u128::from(amount); if let Some(output) = self.outputs.get_mut(idx + 1) { *output = Some(TokenOutput { token_idx: self.sections.len(), variant: TokenVariant::Amount(amount), }); } } self.sections.push(ColoredTxSection { meta, tx_type: TxType::SEND, required_input_sum, genesis_info: None, has_colored_out_of_range: out_of_range_idx.is_some(), }); Ok(()) } fn color_burn( &mut self, meta: TokenMeta, amount: Amount, ) -> Result<(), ColorError> { for (prev_burn_idx, prev_burn) in self.intentional_burns.iter().enumerate() { if prev_burn.meta.token_id == meta.token_id { return Err(DuplicateIntentionalBurnTokenId { prev_burn_idx, burn_idx: self.intentional_burns.len(), token_id: meta.token_id, }); } } self.intentional_burns .push(IntentionalBurn { meta, amount }); Ok(()) } fn color_unknown(&mut self, meta: TokenMeta) -> Result<(), ColorError> { // Color all outputs (except the OP_RETURN) that haven't been colored // yet as "unknown" for token_data in self.outputs.iter_mut().skip(1) { if token_data.is_none() { *token_data = Some(TokenOutput { token_idx: self.sections.len(), variant: TokenVariant::Unknown(meta.token_type.to_u8()), }); } } self.sections.push(ColoredTxSection { meta, tx_type: TxType::UNKNOWN, required_input_sum: 0, genesis_info: None, has_colored_out_of_range: false, }); Ok(()) } /// Turn a [`TokenOutput`] of this [`ColoredTx`] into a [`Token`]. pub fn token(&self, token_output: &TokenOutput) -> Token { let section = &self.sections[token_output.token_idx]; Token { meta: section.meta, variant: token_output.variant, } } } impl ColoredTxSection { /// Whether the section has SLP V2 (MintVault) token type and a MINT tx /// type. pub fn is_mint_vault_mint(&self) -> bool { if self.tx_type != TxType::MINT { return false; } matches!( self.meta.token_type, TokenType::Slp(SlpTokenType::MintVault), ) } } impl std::fmt::Display for FailedParsing { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "Parsing failed")?; if let Some(pushdata_idx) = self.pushdata_idx { write!(f, " at pushdata idx {pushdata_idx}")?; } write!(f, ": {}", self.error) } } impl std::fmt::Display for ParseError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match &self { ParseError::Empp(err) => write!(f, "eMPP error: {err}"), ParseError::Slp(err) => write!(f, "SLP error: {err}"), ParseError::Alp(err) => write!(f, "ALP error: {err}"), } } } diff --git a/chronik/bitcoinsuite-slp/src/verify.rs b/chronik/bitcoinsuite-slp/src/verify.rs index 9f98fe19b..d05cc5dd2 100644 --- a/chronik/bitcoinsuite-slp/src/verify.rs +++ b/chronik/bitcoinsuite-slp/src/verify.rs @@ -1,451 +1,451 @@ // Copyright (c) 2023 The Bitcoin developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. //! Module for [`VerifyContext`]. use std::collections::BTreeMap; use bitcoinsuite_core::script::Script; use serde::{Deserialize, Serialize}; use thiserror::Error; use crate::{ alp::consts::MAX_TX_INPUTS, color::{ColoredTx, ColoredTxSection}, parsed::ParsedTxType, structs::{GenesisInfo, Token, TokenMeta, TokenVariant, TxType}, token_tx::{TokenTx, TokenTxEntry}, token_type::{SlpTokenType, TokenType}, verify::BurnError::*, }; /// Error how token verification based on inputs failed #[derive(Clone, Debug, Deserialize, Eq, Error, Hash, PartialEq, Serialize)] pub enum BurnError { /// ALP has an upper limit on the number tx inputs. /// Note that given current consensus rules, having this many inputs is not /// possible, so this is a failsafe. #[error("Too many tx inputs, got {0} but only {} allowed", MAX_TX_INPUTS)] TooManyTxInputs(usize), /// NFT1 CHILD GENESIS requires an NFT1 GROUP token in the first input #[error("Invalid NFT1 CHILD GENESIS: No GROUP token")] MissingNft1Group, /// MINT requires a mint baton in the inputs #[error("Missing MINT baton")] MissingMintBaton, /// MINT requires a mint vault input #[error("Missing MINT vault")] MissingMintVault, /// SEND transfers cannot have more tokens in the outputs than are supplied /// in the inputs. #[error("Insufficient token input output sum: {actual} < {required}")] InsufficientInputSum { /// Required minimum inputs as specified in the outputs required: u128, /// Actual supplied token amount actual: u128, }, } /// Token spent as an input #[derive(Clone, Debug, Eq, PartialEq)] pub struct SpentToken { /// Input token pub token: Token, /// GROUP token ID and type of the input, if any pub group_token_meta: Option, } /// Context under which to verify a [`ColoredTx`]. #[derive(Debug)] pub struct VerifyContext<'a> { /// Input tokens of the tx pub spent_tokens: &'a [Option], /// scriptPubKeys of the inputs of the tx, required only for SLP V2 MINT /// txs pub spent_scripts: Option<&'a [Script]>, /// [`GenesisInfo`] of the tx's token ID, required only for SLP V2 MINT txs pub genesis_info: Option<&'a GenesisInfo>, /// Override whether a mint vault input is present by setting this to /// `true`. pub override_has_mint_vault: Option, } struct BareBurn { burn_amount: u128, burns_mint_batons: bool, group_token_meta: Option, is_invalid: bool, } impl VerifyContext<'_> { /// Verify the [`ColoredTx`] under the given context and return a verified /// [`TokenTx`]. pub fn verify(&self, tx: ColoredTx) -> TokenTx { let mut entries = Vec::new(); for section in &tx.sections { entries.push(self.verify_section(&tx, section)); } // Add entries for standalone intentional burns without any // corresponding existing sections for intentional_burn in &tx.intentional_burns { if !entries.iter().any(|burn| { burn.meta.token_id == intentional_burn.meta.token_id }) { entries.push(TokenTxEntry { meta: intentional_burn.meta, tx_type: Some(TxType::BURN), genesis_info: None, group_token_meta: None, is_invalid: false, intentional_burn_amount: Some(intentional_burn.amount), actual_burn_amount: 0, burns_mint_batons: false, burn_error: None, has_colored_out_of_range: false, failed_colorings: vec![], }); } } let bare_burns = self.calc_bare_burns(&tx, &entries); // Add failed colorings to the matching entry, or add a new one for failed_coloring in tx.failed_colorings { if let Some(entry) = entries .iter_mut() .find(|entry| entry.meta == failed_coloring.parsed.meta) { entry.failed_colorings.push(failed_coloring); continue; } entries.push(TokenTxEntry { meta: failed_coloring.parsed.meta, tx_type: Some(failed_coloring.parsed.tx_type.tx_type()), genesis_info: match &failed_coloring.parsed.tx_type { ParsedTxType::Genesis(genesis) => { Some(genesis.info.clone()) } _ => None, }, group_token_meta: None, is_invalid: true, intentional_burn_amount: None, actual_burn_amount: 0, burns_mint_batons: false, burn_error: None, has_colored_out_of_range: false, failed_colorings: vec![failed_coloring], }); } // Update entries for bare burn or add them for (burn_meta, bare_burn) in bare_burns { if let Some(entry) = entries.iter_mut().find(|entry| entry.meta == *burn_meta) { if bare_burn.burns_mint_batons { entry.is_invalid = true; } entry.actual_burn_amount = bare_burn.burn_amount; entry.burns_mint_batons = bare_burn.burns_mint_batons; entry.group_token_meta = bare_burn.group_token_meta; continue; } entries.push(TokenTxEntry { meta: *burn_meta, tx_type: None, genesis_info: None, group_token_meta: bare_burn.group_token_meta, is_invalid: bare_burn.is_invalid, intentional_burn_amount: None, actual_burn_amount: bare_burn.burn_amount, burns_mint_batons: bare_burn.burns_mint_batons, burn_error: None, has_colored_out_of_range: false, failed_colorings: vec![], }); } let outputs = tx .outputs .into_iter() .map(|output| -> Option<_> { let entry = &entries[output.as_ref()?.token_idx]; if entry.is_invalid { return None; } output }) .collect::>(); TokenTx { entries, outputs, failed_parsings: tx.failed_parsings, } } fn verify_section( &self, tx: &ColoredTx, section: &ColoredTxSection, ) -> TokenTxEntry { let input_sum = self.calc_input_sum(§ion.meta); // Template entry with either zero defaults or copied over from the // colored section. let entry = TokenTxEntry { meta: section.meta, tx_type: Some(section.tx_type), genesis_info: section.genesis_info.clone(), group_token_meta: self.inherited_group_token_meta(§ion.meta), is_invalid: false, intentional_burn_amount: self .intentional_burn_amount(tx, §ion.meta), actual_burn_amount: 0, burns_mint_batons: false, burn_error: None, has_colored_out_of_range: section.has_colored_out_of_range, failed_colorings: vec![], }; // ALP only allows up to 2**15 inputs if section.meta.token_type.is_alp() && self.spent_tokens.len() > MAX_TX_INPUTS { return TokenTxEntry { is_invalid: true, actual_burn_amount: input_sum, burns_mint_batons: self.has_mint_baton(§ion.meta), burn_error: Some(TooManyTxInputs(self.spent_tokens.len())), ..entry }; } match section.tx_type { // NFT1 CHILD tokens have to an NFT1 GROUP token at the 1st input TxType::GENESIS if section.meta.token_type == TokenType::Slp(SlpTokenType::Nft1Child) => { - match self.spent_tokens.get(0) { + match self.spent_tokens.first() { Some(Some(spent_token)) if spent_token.token.meta.token_type == TokenType::Slp(SlpTokenType::Nft1Group) && spent_token.token.variant.amount() > 0 => { TokenTxEntry { group_token_meta: Some(spent_token.token.meta), ..entry } } _ => TokenTxEntry { is_invalid: true, burn_error: Some(MissingNft1Group), ..entry }, } } // All other GENESIS txs are self-evident TxType::GENESIS => entry, // SLP V2 Mint Vault must have a given Script as input TxType::MINT if section.is_mint_vault_mint() => { if self.has_mint_vault() { return TokenTxEntry { actual_burn_amount: input_sum, ..entry }; } TokenTxEntry { is_invalid: true, actual_burn_amount: input_sum, burn_error: Some(MissingMintVault), ..entry } } // All other MINTs must have a matching mint baton TxType::MINT => { if self.has_mint_baton(§ion.meta) { return TokenTxEntry { actual_burn_amount: input_sum, ..entry }; } TokenTxEntry { is_invalid: true, actual_burn_amount: input_sum, burn_error: Some(MissingMintBaton), ..entry } } // SEND cannot spent more than came into the tx as inputs TxType::SEND if input_sum < section.required_input_sum => { TokenTxEntry { is_invalid: true, actual_burn_amount: input_sum, burns_mint_batons: self.has_mint_baton(§ion.meta), burn_error: Some(InsufficientInputSum { required: section.required_input_sum, actual: input_sum, }), ..entry } } // Valid SEND TxType::SEND => { let output_sum = self.calc_output_sum(tx, §ion.meta); let actual_burn_amount = input_sum - output_sum; TokenTxEntry { actual_burn_amount, burns_mint_batons: self.has_mint_baton(§ion.meta), ..entry } } // UNKNOWN txs are self-evident TxType::UNKNOWN => entry, TxType::BURN => unreachable!( "BURNs are only in intentional_burns, not in sections" ), } } fn has_mint_baton(&self, meta: &TokenMeta) -> bool { self.spent_tokens.iter().flatten().any(|spent_token| { &spent_token.token.meta == meta && spent_token.token.variant.is_mint_baton() }) } fn has_mint_vault(&self) -> bool { if let Some(override_has_mint_vault) = self.override_has_mint_vault { return override_has_mint_vault; } let Some(spent_scripts) = self.spent_scripts else { panic!( "VerifyContext used incorrectly; spent_scripts must be \ present for SLP V2 Mint Vault token types" ); }; let Some(genesis_info) = self.genesis_info else { return false; }; let Some(scripthash) = &genesis_info.mint_vault_scripthash else { return false; }; let script = Script::p2sh(scripthash); spent_scripts .iter() .any(|spent_script| spent_script == &script) } fn calc_input_sum(&self, meta: &TokenMeta) -> u128 { self.spent_tokens .iter() .flatten() .filter(|token| &token.token.meta == meta) .map(|token| token.token.variant.amount() as u128) .sum() } fn calc_output_sum(&self, tx: &ColoredTx, meta: &TokenMeta) -> u128 { tx.outputs .iter() .flatten() .filter(|token| &tx.sections[token.token_idx].meta == meta) .map(|token| token.variant.amount() as u128) .sum() } fn inherited_group_token_meta( &self, meta: &TokenMeta, ) -> Option { self.spent_tokens .iter() .flatten() .find(|token| &token.token.meta == meta) .and_then(|token| token.group_token_meta) } fn intentional_burn_amount( &self, tx: &ColoredTx, meta: &TokenMeta, ) -> Option { tx.intentional_burns .iter() .find(|burn| &burn.meta == meta) .map(|burn| burn.amount) } // Bare burns: spent tokens without a corresponding section fn calc_bare_burns( &self, tx: &ColoredTx, entries: &[TokenTxEntry], ) -> BTreeMap<&TokenMeta, BareBurn> { let mut bare_burns = BTreeMap::new(); for (input_idx, input) in self.spent_tokens.iter().enumerate() { let Some(input) = input else { continue }; // Input has a corresponding mentioned section, not a bare burn if tx .sections .iter() .any(|section| section.meta == input.token.meta) { continue; } let bare_burn = bare_burns.entry(&input.token.meta).or_insert(BareBurn { burn_amount: 0, burns_mint_batons: false, group_token_meta: input.group_token_meta, is_invalid: false, }); // We don't consider NFT1 GROUP inputs for NFT1 CHILD GENESIS a burn // At this stage the validation that the first input is an NFT1 // GROUP token is already done, otherwise is_invalid would be true. // We still create a bare burn entry so that we get a TokenTxEntry, // but leave is_invalid at false and don't increment the burned // amount. if input_idx == 0 { if let Some(first_entry) = entries.first() { if first_entry.meta.token_type == TokenType::Slp(SlpTokenType::Nft1Child) && first_entry.tx_type == Some(TxType::GENESIS) && !first_entry.is_invalid { continue; } } } // All other bare burns are invalid bare_burn.is_invalid = true; match input.token.variant { TokenVariant::Amount(amount) => { bare_burn.burn_amount += u128::from(amount) } TokenVariant::MintBaton => bare_burn.burns_mint_batons = true, TokenVariant::Unknown(_) => {} } } bare_burns } } diff --git a/chronik/chronik-db/src/io/spent_by.rs b/chronik/chronik-db/src/io/spent_by.rs index 7d6fb01b3..619e19bfa 100644 --- a/chronik/chronik-db/src/io/spent_by.rs +++ b/chronik/chronik-db/src/io/spent_by.rs @@ -1,681 +1,678 @@ // Copyright (c) 2023 The Bitcoin developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. use std::{ collections::{hash_map::Entry, HashMap}, time::Instant, }; use abc_rust_error::Result; use rocksdb::{ColumnFamilyDescriptor, Options}; use serde::{Deserialize, Serialize}; use thiserror::Error; use crate::{ db::{Db, WriteBatch, CF, CF_SPENT_BY}, index_tx::IndexTx, io::{merge::catch_merge_errors, TxNum}, ser::{db_deserialize, db_deserialize_vec, db_serialize, db_serialize_vec}, }; /// Indicates an output has been spent by an input in a tx. /// This is an entry in the list of spent outputs of a tx. #[derive( Clone, Debug, Default, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, )] pub struct SpentByEntry { /// Which output has been spent. pub out_idx: u32, /// Which tx spent the output. pub tx_num: TxNum, /// Which input of the spending tx spent the output. pub input_idx: u32, } struct SpentByColumn<'a> { db: &'a Db, cf: &'a CF, } /// Write to the DB which outputs of a tx have been spent by which tx (and /// input). /// /// For each tx that has any output spent, there will be a list of entries in /// the DB. Each entry tells us which output has been spent, which tx_num spent /// it, and which input of that tx. /// /// Note: While TxWriter writes keys using 8-byte big-endian numbers, this /// writes it using [`db_serialize`], because unlike TxWriter, we don't rely on /// any ordering. Since [`db_serialize`] is more compact, this saves some space. #[derive(Debug)] pub struct SpentByWriter<'a> { col: SpentByColumn<'a>, } /// Read from the DB which outputs of a tx have been spent by which tx (and /// input). #[derive(Debug)] pub struct SpentByReader<'a> { col: SpentByColumn<'a>, } /// In-memory data for spent-by data. #[derive(Debug, Default)] pub struct SpentByMemData { /// Stats about cache hits, num requests etc. pub stats: SpentByStats, } /// Stats about cache hits, num requests etc. #[derive(Clone, Debug, Default)] pub struct SpentByStats { /// Total number of txs updated. pub n_total: usize, /// Time [s] for insert/delete. pub t_total: f64, /// Time [s] for fetching txs. pub t_fetch: f64, } /// Error indicating that something went wrong with writing spent-by data. #[derive(Debug, Error, PartialEq, Eq)] pub enum SpentByError { /// Tried adding a spent-by entry, but it's already marked as spent by /// another entry. #[error( "Inconsistent DB: Duplicate spend by entry for tx_num = {tx_num}: \ {existing:?} already exists, but tried to add {new:?}" )] DuplicateSpentByEntry { /// An output of this tx_num has been spent tx_num: TxNum, /// Entry already present in the DB. existing: SpentByEntry, /// Entry we tried to add. new: SpentByEntry, }, /// Tried removing a spent-by entry, but it doesn't match what we got from /// the disconnected block. #[error( "Inconsistent DB: Mismatched spend by entry for tx_num = {tx_num}: \ Expected {expected:?} to be present, but got {actual:?}" )] MismatchedSpentByEntry { /// Tried removing a spent-by entry of an output of this tx_num. tx_num: TxNum, /// Entry we expected based on the disconnected block. expected: SpentByEntry, /// Entry actually found in the DB. actual: SpentByEntry, }, /// Tried removing a spent-by entry, but it doesn't exist. #[error( "Inconsistent DB: Missing spend by entry for tx_num = {tx_num}: \ Expected {expected:?} to be present, but none found" )] MissingSpentByEntry { /// Tried removing a spent-by entry of an output of this tx_num. tx_num: TxNum, /// Entry we expected to be present based on the disconnected block. expected: SpentByEntry, }, } use self::SpentByError::*; fn ser_tx_num(tx_num: TxNum) -> Result> { db_serialize(&tx_num) } fn deser_tx_num(bytes: &[u8]) -> Result { db_deserialize(bytes) } fn init_merge_spent_by( _key: &[u8], existing_value: Option<&[u8]>, _operands: &rocksdb::MergeOperands, ) -> Result> { match existing_value { Some(bytes) => db_deserialize_vec::(bytes), None => Ok(vec![]), } } fn apply_merge_spent_by( key: &[u8], entries: &mut Vec, operand: &[u8], ) -> Result<()> { let extra_entries = db_deserialize_vec::(operand)?; entries.reserve(extra_entries.len()); for spent_by in extra_entries { let search_idx = entries .binary_search_by_key(&spent_by.out_idx, |entry| entry.out_idx); match search_idx { Ok(idx) => { let input_tx_num = deser_tx_num(key)?; // Output already spent by another tx -> corrupted DB? return Err(DuplicateSpentByEntry { tx_num: input_tx_num, existing: entries[idx].clone(), new: spent_by, } .into()); } Err(insert_idx) => { // No entry found -> insert it entries.insert(insert_idx, spent_by); } } } Ok(()) } fn ser_merge_spent_by( _key: &[u8], entries: Vec, ) -> Result> { db_serialize_vec::(entries) } impl<'a> SpentByColumn<'a> { fn new(db: &'a Db) -> Result { let cf = db.cf(CF_SPENT_BY)?; Ok(SpentByColumn { db, cf }) } } impl<'a> SpentByWriter<'a> { /// Create a new [`SpentByWriter`]. pub fn new(db: &'a Db) -> Result { let col = SpentByColumn::new(db)?; Ok(SpentByWriter { col }) } /// Add spent-by entries to txs spent in the txs. /// For each tx output spent in `txs`, add which tx spent it. pub fn insert( &self, batch: &mut WriteBatch, txs: &[IndexTx<'_>], mem_data: &mut SpentByMemData, ) -> Result<()> { let stats = &mut mem_data.stats; let t_start = Instant::now(); stats.n_total += txs.len(); let mut spent_by_map = HashMap::>::new(); for tx in txs { if tx.is_coinbase { // a coinbase doesn't spend anything continue; } for (input_idx, (input, &input_tx_num)) in tx.tx.inputs.iter().zip(tx.input_nums.iter()).enumerate() { let spent_by = SpentByEntry { out_idx: input.prev_out.out_idx, tx_num: tx.tx_num, input_idx: input_idx as u32, }; - spent_by_map - .entry(input_tx_num) - .or_insert(vec![]) - .push(spent_by); + spent_by_map.entry(input_tx_num).or_default().push(spent_by); } } for (tx_num, entries) in spent_by_map { batch.merge_cf( self.col.cf, ser_tx_num(tx_num)?, db_serialize_vec::(entries)?, ); } stats.t_total += t_start.elapsed().as_secs_f64(); Ok(()) } /// Remove spent-by entries from txs spent in the txs. /// For each tx output spent in `txs`, remove which tx spent it. pub fn delete( &self, batch: &mut WriteBatch, txs: &[IndexTx<'_>], mem_data: &mut SpentByMemData, ) -> Result<()> { let stats = &mut mem_data.stats; let t_start = Instant::now(); stats.n_total += txs.len(); let mut spent_by_map = HashMap::>::new(); for tx in txs { if tx.is_coinbase { // a coinbase doesn't spend anything continue; } for (input_idx, (input, &input_tx_num)) in tx.tx.inputs.iter().zip(tx.input_nums.iter()).enumerate() { let spent_by = SpentByEntry { out_idx: input.prev_out.out_idx, tx_num: tx.tx_num, input_idx: input_idx as u32, }; let t_fetch = Instant::now(); let spent_by_entries = self.get_or_fetch(&mut spent_by_map, input_tx_num)?; stats.t_fetch += t_fetch.elapsed().as_secs_f64(); let search_idx = spent_by_entries .binary_search_by_key(&spent_by.out_idx, |entry| { entry.out_idx }); match search_idx { Ok(idx) => { // Found the spent-by entry -> remove it. if spent_by_entries[idx] != spent_by { // Existing entry doesn't match what's in the DB. return Err(MismatchedSpentByEntry { tx_num: input_tx_num, expected: spent_by, actual: spent_by_entries[idx].clone(), } .into()); } spent_by_entries.remove(idx); } Err(_) => { // Spent-by entry not found, but should be there. return Err(MissingSpentByEntry { tx_num: input_tx_num, expected: spent_by, } .into()); } } } } for (tx_num, entries) in spent_by_map { let ser_num = ser_tx_num(tx_num)?; if entries.is_empty() { batch.delete_cf(self.col.cf, ser_num); } else { batch.put_cf(self.col.cf, ser_num, db_serialize_vec(entries)?); } } stats.t_total += t_start.elapsed().as_secs_f64(); Ok(()) } fn get_or_fetch<'b>( &self, spent_by_map: &'b mut HashMap>, tx_num: TxNum, ) -> Result<&'b mut Vec> { match spent_by_map.entry(tx_num) { Entry::Occupied(entry) => Ok(entry.into_mut()), Entry::Vacant(entry) => { let db_entries = match self.col.db.get(self.col.cf, ser_tx_num(tx_num)?)? { Some(data) => { db_deserialize_vec::(&data)? } None => vec![], }; Ok(entry.insert(db_entries)) } } } pub(crate) fn add_cfs(columns: &mut Vec) { let mut options = Options::default(); options.set_merge_operator( "spent_by::merge_op", catch_merge_errors( init_merge_spent_by, apply_merge_spent_by, ser_merge_spent_by, ), |_, _, _| None, ); columns.push(ColumnFamilyDescriptor::new(CF_SPENT_BY, options)); } } impl<'a> SpentByReader<'a> { /// Create a new [`SpentByReader`]. pub fn new(db: &'a Db) -> Result { let col = SpentByColumn::new(db)?; Ok(SpentByReader { col }) } /// Query the spent-by entries by [`TxNum`]. pub fn by_tx_num( &self, tx_num: TxNum, ) -> Result>> { match self.col.db.get(self.col.cf, ser_tx_num(tx_num)?)? { Some(data) => Ok(Some(db_deserialize_vec::(&data)?)), None => Ok(None), } } } impl std::fmt::Debug for SpentByColumn<'_> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("SpentByColumn") .field("db", &self.db) .field("cf", &"..") .finish() } } #[cfg(test)] mod tests { use std::cell::RefCell; use abc_rust_error::Result; use bitcoinsuite_core::tx::Tx; use crate::{ db::{Db, WriteBatch}, index_tx::prepare_indexed_txs, io::{ merge::{check_for_errors, MERGE_ERROR_LOCK}, BlockTxs, SpentByEntry, SpentByError, SpentByMemData, SpentByReader, SpentByWriter, TxEntry, TxWriter, TxsMemData, }, test::make_inputs_tx, }; #[test] fn test_spent_by() -> Result<()> { let _guard = MERGE_ERROR_LOCK.lock().unwrap(); abc_rust_error::install(); let tempdir = tempdir::TempDir::new("chronik-db--spent_by")?; let mut cfs = Vec::new(); TxWriter::add_cfs(&mut cfs); SpentByWriter::add_cfs(&mut cfs); let db = Db::open_with_cfs(tempdir.path(), cfs)?; let tx_writer = TxWriter::new(&db)?; let spent_by_writer = SpentByWriter::new(&db)?; let spent_by_reader = SpentByReader::new(&db)?; let mem_data = RefCell::new(SpentByMemData::default()); let txs_mem_data = RefCell::new(TxsMemData::default()); let block_height = RefCell::new(-1); let txs_batch = |txs: &[Tx]| BlockTxs { txs: txs .iter() .map(|tx| TxEntry { txid: tx.txid(), ..Default::default() }) .collect(), block_height: *block_height.borrow(), }; let connect_block = |txs: &[Tx]| -> Result<()> { let mut batch = WriteBatch::default(); *block_height.borrow_mut() += 1; let first_tx_num = tx_writer.insert( &mut batch, &txs_batch(txs), &mut txs_mem_data.borrow_mut(), )?; let index_txs = prepare_indexed_txs(&db, first_tx_num, txs)?; spent_by_writer.insert( &mut batch, &index_txs, &mut mem_data.borrow_mut(), )?; db.write_batch(batch)?; for tx in &index_txs { for &input_tx_num in &tx.input_nums { spent_by_reader.by_tx_num(input_tx_num)?; check_for_errors()?; } } Ok(()) }; let disconnect_block = |txs: &[Tx]| -> Result<()> { let mut batch = WriteBatch::default(); let first_tx_num = tx_writer.delete( &mut batch, &txs_batch(txs), &mut txs_mem_data.borrow_mut(), )?; let index_txs = prepare_indexed_txs(&db, first_tx_num, txs)?; spent_by_writer.delete( &mut batch, &index_txs, &mut mem_data.borrow_mut(), )?; db.write_batch(batch)?; *block_height.borrow_mut() -= 1; Ok(()) }; macro_rules! spent_by { (out_idx=$out_idx:literal -> tx_num=$tx_num:literal, input_idx=$input_idx:literal) => { SpentByEntry { out_idx: $out_idx, tx_num: $tx_num, input_idx: $input_idx, } }; } let block0 = vec![make_inputs_tx(0, [(0x00, u32::MAX, -1)], [-1, -1, -1, -1])]; connect_block(&block0)?; let block1 = vec![ make_inputs_tx(1, [(0x00, u32::MAX, -1)], [-1, -1]), // spend 3rd output of tx_num=0 make_inputs_tx(2, [(0, 3, -1)], [-1, -1, -1]), ]; connect_block(&block1)?; assert_eq!( spent_by_reader.by_tx_num(0)?, Some(vec![spent_by!(out_idx=3 -> tx_num=2, input_idx=0)]), ); // Remove block1 disconnect_block(&block1)?; assert_eq!(spent_by_reader.by_tx_num(0)?, None); // Add block1 again connect_block(&block1)?; let block2 = vec![ make_inputs_tx(3, [(0x00, u32::MAX, -1)], [-1]), // spend 2nd output of tx_num=0, and 0th output of tx_num=1 make_inputs_tx(4, [(0, 2, -1), (1, 0, -1)], [-1, -1, -1]), // spend 1st output of tx_num=1, and 1st output of tx_num=0 make_inputs_tx(5, [(1, 1, -1), (0, 1, -1)], [-1, -1]), ]; connect_block(&block2)?; assert_eq!( spent_by_reader.by_tx_num(0)?, Some(vec![ spent_by!(out_idx=1 -> tx_num=5, input_idx=1), spent_by!(out_idx=2 -> tx_num=4, input_idx=0), spent_by!(out_idx=3 -> tx_num=2, input_idx=0), ]), ); assert_eq!( spent_by_reader.by_tx_num(1)?, Some(vec![ spent_by!(out_idx=0 -> tx_num=4, input_idx=1), spent_by!(out_idx=1 -> tx_num=5, input_idx=0), ]), ); // More complex block let block3 = vec![ make_inputs_tx(6, [(0x00, u32::MAX, -1)], [-1]), make_inputs_tx(7, [(4, 0, -1), (2, 0, -1)], [-1, -1]), make_inputs_tx(8, [(9, 0, -1), (5, 1, -1)], [-1, -1]), make_inputs_tx(9, [(7, 1, -1), (8, 1, -1)], [-1]), ]; connect_block(&block3)?; assert_eq!( spent_by_reader.by_tx_num(2)?, Some(vec![spent_by!(out_idx=0 -> tx_num=7, input_idx=1)]), ); assert_eq!( spent_by_reader.by_tx_num(4)?, Some(vec![spent_by!(out_idx=0 -> tx_num=7, input_idx=0)]), ); assert_eq!( spent_by_reader.by_tx_num(5)?, Some(vec![spent_by!(out_idx=1 -> tx_num=8, input_idx=1)]), ); assert_eq!(spent_by_reader.by_tx_num(6)?, None); assert_eq!( spent_by_reader.by_tx_num(7)?, Some(vec![spent_by!(out_idx=1 -> tx_num=9, input_idx=0)]), ); assert_eq!( spent_by_reader.by_tx_num(8)?, Some(vec![spent_by!(out_idx=1 -> tx_num=9, input_idx=1)]), ); assert_eq!( spent_by_reader.by_tx_num(9)?, Some(vec![spent_by!(out_idx=0 -> tx_num=8, input_idx=0)]), ); disconnect_block(&block3)?; assert_eq!( spent_by_reader.by_tx_num(0)?, Some(vec![ spent_by!(out_idx=1 -> tx_num=5, input_idx=1), spent_by!(out_idx=2 -> tx_num=4, input_idx=0), spent_by!(out_idx=3 -> tx_num=2, input_idx=0), ]), ); assert_eq!( spent_by_reader.by_tx_num(1)?, Some(vec![ spent_by!(out_idx=0 -> tx_num=4, input_idx=1), spent_by!(out_idx=1 -> tx_num=5, input_idx=0), ]), ); assert_eq!(spent_by_reader.by_tx_num(2)?, None); assert_eq!(spent_by_reader.by_tx_num(4)?, None); assert_eq!(spent_by_reader.by_tx_num(5)?, None); assert_eq!(spent_by_reader.by_tx_num(6)?, None); assert_eq!(spent_by_reader.by_tx_num(7)?, None); assert_eq!(spent_by_reader.by_tx_num(8)?, None); assert_eq!(spent_by_reader.by_tx_num(9)?, None); // failed connect: duplicate entry let block_duplicate_spend = vec![ make_inputs_tx(10, [(0x00, u32::MAX, -1)], []), make_inputs_tx(11, [(0, 1, -1)], []), ]; assert_eq!( connect_block(&block_duplicate_spend) .unwrap_err() .downcast::()?, SpentByError::DuplicateSpentByEntry { tx_num: 0, existing: spent_by!(out_idx=1 -> tx_num=5, input_idx=1), new: spent_by!(out_idx=1 -> tx_num=7, input_idx=0), }, ); // Ensure failed connect didn't have any side-effects on spent-by data assert_eq!( spent_by_reader.by_tx_num(0)?, Some(vec![ spent_by!(out_idx=1 -> tx_num=5, input_idx=1), spent_by!(out_idx=2 -> tx_num=4, input_idx=0), spent_by!(out_idx=3 -> tx_num=2, input_idx=0), ]), ); assert_eq!( spent_by_reader.by_tx_num(1)?, Some(vec![ spent_by!(out_idx=0 -> tx_num=4, input_idx=1), spent_by!(out_idx=1 -> tx_num=5, input_idx=0), ]), ); // Undo side effects introduced by failed connect disconnect_block(&[ make_inputs_tx(10, [(0x00, u32::MAX, -1)], []), // Note the missing input values make_inputs_tx(11, [], []), ])?; // failed disconnect: mismatched entry let block_mismatched_spend = vec![ make_inputs_tx(3, [(0x00, u32::MAX, -1)], []), make_inputs_tx(4, [(0, 1, -1)], []), ]; assert_eq!( disconnect_block(&block_mismatched_spend) .unwrap_err() .downcast::()?, SpentByError::MismatchedSpentByEntry { tx_num: 0, expected: spent_by!(out_idx=1 -> tx_num=4, input_idx=0), actual: spent_by!(out_idx=1 -> tx_num=5, input_idx=1), }, ); // failed disconnect: missing entry let block_missing_spend = vec![ make_inputs_tx(3, [(0x00, u32::MAX, -1)], []), make_inputs_tx(4, [(3, 1, -1)], []), ]; assert_eq!( disconnect_block(&block_missing_spend) .unwrap_err() .downcast::()?, SpentByError::MissingSpentByEntry { tx_num: 3, expected: spent_by!(out_idx=1 -> tx_num=4, input_idx=0), }, ); // disconnect blocks disconnect_block(&block2)?; assert_eq!( spent_by_reader.by_tx_num(0)?, Some(vec![spent_by!(out_idx=3 -> tx_num=2, input_idx=0)]), ); assert_eq!(spent_by_reader.by_tx_num(1)?, None); assert_eq!(spent_by_reader.by_tx_num(2)?, None); assert_eq!(spent_by_reader.by_tx_num(3)?, None); assert_eq!(spent_by_reader.by_tx_num(4)?, None); assert_eq!(spent_by_reader.by_tx_num(5)?, None); disconnect_block(&block1)?; assert_eq!(spent_by_reader.by_tx_num(0)?, None); assert_eq!(spent_by_reader.by_tx_num(1)?, None); assert_eq!(spent_by_reader.by_tx_num(2)?, None); assert_eq!(spent_by_reader.by_tx_num(3)?, None); disconnect_block(&block0)?; assert_eq!(spent_by_reader.by_tx_num(0)?, None); assert_eq!(spent_by_reader.by_tx_num(1)?, None); drop(db); rocksdb::DB::destroy(&rocksdb::Options::default(), tempdir.path())?; let _ = check_for_errors(); Ok(()) } } diff --git a/chronik/chronik-db/src/io/token/batch.rs b/chronik/chronik-db/src/io/token/batch.rs index e914c19bb..b6910fbed 100644 --- a/chronik/chronik-db/src/io/token/batch.rs +++ b/chronik/chronik-db/src/io/token/batch.rs @@ -1,463 +1,463 @@ // Copyright (c) 2024 The Bitcoin developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. use std::collections::{BTreeMap, HashMap, HashSet}; use abc_rust_error::Result; use bimap::BiMap; use bitcoinsuite_core::{script::Script, tx::OutPoint}; use bitcoinsuite_slp::{ color::ColoredTx, structs::{GenesisInfo, Token, TokenMeta, TokenVariant, TxType}, token_tx::TokenTx, token_type::TokenType, verify::{SpentToken, VerifyContext}, }; use itertools::{Either, Itertools}; use thiserror::Error; use topo_sort::TopoSort; use crate::{ index_tx::IndexTx, io::{ token::{BatchError::*, DbToken, DbTokenTx, FLAGS_HAS_MINT_VAULT}, TxNum, }, }; /// Tx that has token data encoded in it and is ready for token validation #[derive(Debug)] pub struct PreparedTx<'tx> { /// Tx with index data (tx_num etc.) pub tx: &'tx IndexTx<'tx>, /// Parsed & colored tx. Note that in this context we only store txs with a /// non-empty list of sections. pub colored: ColoredTx, } /// Struct bundling all the data necessary to process a batch (i.e. a block) of /// token txs. #[derive(Debug)] pub struct BatchProcessor<'tx> { /// Tx that have any token info attached pub prepared_txs: HashMap>, /// Non-token txs may still have token inputs pub non_token_txs: Vec<&'tx IndexTx<'tx>>, /// Whether the batch has any GENESIS txs; if the token index is empty and /// we have no GENESIS, we can safely ignore the batch. pub has_any_genesis: bool, } /// DB data required to process a batch of txs #[derive(Debug)] pub struct BatchDbData { /// Token tx data coming from the DB pub token_txs: BTreeMap, /// token_nums assigned for each [`TokenMeta`]. This is a bi-map so we can /// lookup either direction. pub token_metas: BiMap, /// [`GenesisInfo`] from the database; required for SLP V2 Mint Vault txs pub genesis_infos: HashMap, } #[derive(Debug, Default)] /// Result of the batch verification before inserting pub struct ProcessedTokenTxBatch { /// New tokens to be added to the DB pub new_tokens: Vec<(TxNum, TokenMeta, GenesisInfo)>, /// New DB data for txs to be added to the DB. pub db_token_txs: HashMap, /// Validated token txs in the batch. pub valid_txs: HashMap, /// Validated spent tokens in the batch, can have entries not it valid_txs pub spent_tokens: HashMap>>, /// True if validation of txs was performed, or false if validation was /// safely skipped because no tokens are in the DB and the batch contained /// no token txs. pub did_validation: bool, } /// Error when batch-processing txs, usually implies a critical failure #[derive(Debug, Error, PartialEq)] pub enum BatchError { /// Transactions couldn't be ordered topologically because of a cycle /// dependency. Note: This is cryptographically impossible in practice, /// because of the irreversability of SHA256. #[error("Cycle in SLP txs")] Cycle, /// A token_num that should be in the DB was not found #[error("Inconsistent BatchDbData: Missing TokenId for token tx num {0}")] MissingTokenTxNum(TxNum), /// GenesisInfo that should be in the DB was not found #[error("Inconsistent Tx: Missing coin for TxInput {0:?}")] MissingTxInputCoin(OutPoint), } impl<'tx> BatchProcessor<'tx> { /// Prepare the given indexed txs as token txs. pub fn prepare(txs: &'tx [IndexTx<'tx>]) -> Self { let (prepared_txs, non_token_txs): (HashMap<_, _>, Vec<_>) = txs .iter() .partition_map(|tx| match ColoredTx::color_tx(tx.tx) { Some(colored) if !colored.sections.is_empty() => { Either::Left((tx.tx_num, PreparedTx { tx, colored })) } _ => Either::Right(tx), }); // Coloring step ensures all GENESIS are at the first section let has_any_genesis = prepared_txs .values() .any(|tx| tx.colored.sections[0].tx_type == TxType::GENESIS); BatchProcessor { prepared_txs, non_token_txs, has_any_genesis, } } /// Collect all [`TokenMeta`]s of the SLP V2 Mint Vault txs in the batch pub fn collect_mint_vault_metas(&self) -> HashSet { self.prepared_txs .values() .filter(|tx| tx.colored.sections[0].is_mint_vault_mint()) .map(|tx| tx.colored.sections[0].meta) .collect() } /// Verify the entire batch of txs. It updates the DB data with some token /// data during validation, but that can be discarded. The result of the /// verification is returned as [`ProcessedTokenTxBatch`]. pub fn verify( mut self, mut db_data: BatchDbData, ) -> Result { // Build a DAG of tx nums so we can sort topologically let mut topo_sort = TopoSort::with_capacity(self.prepared_txs.len()); for (&tx_num, batch_tx) in &self.prepared_txs { topo_sort.insert_from_slice(tx_num, &batch_tx.tx.input_nums); } // Iterate txs in topological order let mut processed_batch = ProcessedTokenTxBatch { did_validation: true, ..Default::default() }; for tx_num in topo_sort.into_nodes() { let tx_num = tx_num.map_err(|_| Cycle)?; let prepared_tx = self.prepared_txs.remove(&tx_num).unwrap(); self.verify_token_tx( prepared_tx, &mut db_data, &mut processed_batch, )?; } // Non-token txs can still contain token inputs, add those to the index // too for non_token_tx in &self.non_token_txs { self.process_non_token_tx( non_token_tx, &mut db_data, &mut processed_batch, )?; } Ok(processed_batch) } fn verify_token_tx( &self, prepared_tx: PreparedTx<'_>, db_data: &mut BatchDbData, processed_batch: &mut ProcessedTokenTxBatch, ) -> Result<()> { let tx_num = prepared_tx.tx.tx_num; let spent_tokens = self.tx_token_inputs( prepared_tx.tx, db_data, &processed_batch.valid_txs, )?; let first_section = &prepared_tx.colored.sections[0]; let is_genesis = first_section.genesis_info.is_some(); let is_mint_vault_mint = first_section.is_mint_vault_mint(); // MINT txs on SLP V2 tokens need spent scripts and genesis data let mut spent_scripts = None; let mut genesis_info = None; if is_mint_vault_mint { spent_scripts = Some(Self::tx_spent_scripts(prepared_tx.tx)?); genesis_info = db_data.genesis_infos.get(&first_section.meta); } let context = VerifyContext { spent_tokens: &spent_tokens, spent_scripts: spent_scripts.as_deref(), genesis_info, override_has_mint_vault: None, }; let valid_tx = context.verify(prepared_tx.colored); let has_any_inputs = spent_tokens.iter().any(|input| input.is_some()); let has_any_outputs = valid_tx.outputs.iter().any(|token| token.is_some()); // Don't store txs that have no actual token inputs or outputs if !has_any_outputs && !has_any_inputs && !is_genesis { return Ok(()); } // Add new tokens from GENESIS txs - if let Some(entry) = valid_tx.entries.get(0) { + if let Some(entry) = valid_tx.entries.first() { // Skip invalid GENESIS txs if !entry.is_invalid { if let Some(info) = &entry.genesis_info { db_data.token_metas.insert(tx_num, entry.meta); processed_batch.new_tokens.push(( tx_num, entry.meta, info.clone(), )); // Note: Don't update db_data.genesis_info, because SLP V2 // GENESIS txs require a confirmation before they take // effect. } } } let mut token_tx_nums = Vec::new(); let mut token_metas = Vec::new(); let mut group_token_metas = BTreeMap::new(); for entry in &valid_tx.entries { let Some(&token_tx_num) = db_data.token_metas.get_by_right(&entry.meta) else { continue; }; if !token_metas.iter().contains(&entry.meta) { token_tx_nums.push(token_tx_num); token_metas.push(entry.meta); } entry.group_token_meta.and_then(|group_meta| { let &tx_num = db_data.token_metas.get_by_right(&group_meta)?; if !token_metas.iter().contains(&group_meta) { token_tx_nums.push(tx_num); token_metas.push(group_meta); group_token_metas.insert(entry.meta, group_meta); } Some(()) }); } let mut flags = 0; if is_mint_vault_mint { - let first_entry = valid_tx.entries.get(0); + let first_entry = valid_tx.entries.first(); let has_mint_vault = first_entry.map_or(false, |entry| !entry.is_invalid); if has_mint_vault { flags |= FLAGS_HAS_MINT_VAULT; } } let db_token_tx = DbTokenTx { token_tx_nums, group_token_indices: group_token_metas .iter() .map(|(meta, group_meta)| { ( meta_idx(meta, &token_metas), meta_idx(group_meta, &token_metas), ) }) .collect(), inputs: spent_tokens .iter() .map(|input| { to_db_token( input.as_ref().map(|input| &input.token), &token_metas, ) }) .collect::>(), outputs: valid_tx .outputs .iter() .map(|output| { to_db_token( output .as_ref() .map(|output| valid_tx.token(output)) .as_ref(), &token_metas, ) }) .collect::>(), flags, }; processed_batch.db_token_txs.insert(tx_num, db_token_tx); processed_batch.valid_txs.insert(tx_num, valid_tx); processed_batch.spent_tokens.insert(tx_num, spent_tokens); Ok(()) } fn process_non_token_tx( &self, tx: &IndexTx<'_>, db_data: &mut BatchDbData, processed_batch: &mut ProcessedTokenTxBatch, ) -> Result<()> { let mut db_token_tx_nums = Vec::new(); let mut db_inputs = Vec::with_capacity(tx.input_nums.len()); let mut db_group_token_indices = BTreeMap::new(); for (&input_tx_num, input) in tx.input_nums.iter().zip(&tx.tx.inputs) { let out_idx = input.prev_out.out_idx as usize; let db_token_tx = processed_batch .db_token_txs .get(&input_tx_num) .or_else(|| db_data.token_txs.get(&input_tx_num)); let Some(db_token_tx) = db_token_tx else { continue; }; let db_token = &db_token_tx.outputs[out_idx]; let Some(token_tx_num) = db_token_tx.token_tx_num(db_token) else { db_inputs.push(*db_token); continue; }; let token_num_idx = db_token_tx_nums .iter() .position(|&tx_num| tx_num == token_tx_num) .unwrap_or_else(|| { db_token_tx_nums.push(token_tx_num); db_token_tx_nums.len() - 1 }); if let Some(group_token_tx_num) = db_token_tx.group_token_tx_num(db_token) { let group_token_num_idx = db_token_tx_nums .iter() .position(|&tx_num| tx_num == group_token_tx_num) .unwrap_or_else(|| { db_token_tx_nums.push(group_token_tx_num); db_token_tx_nums.len() - 1 }); db_group_token_indices .insert(token_num_idx as u32, group_token_num_idx as u32); } db_inputs.push(db_token.with_idx(token_num_idx as u32)); } // Skip non-token tx if we don't have any token inputs if db_inputs.iter().any(|&input| input != DbToken::NoToken) { processed_batch.db_token_txs.insert( tx.tx_num, DbTokenTx { token_tx_nums: db_token_tx_nums, group_token_indices: db_group_token_indices, inputs: db_inputs, outputs: vec![DbToken::NoToken; tx.tx.outputs.len()], flags: 0, }, ); processed_batch.spent_tokens.insert( tx.tx_num, self.tx_token_inputs(tx, db_data, &processed_batch.valid_txs)?, ); } Ok(()) } fn tx_spent_scripts(tx: &IndexTx<'_>) -> Result> { let mut spent_scripts = Vec::with_capacity(tx.tx.inputs.len()); for tx_input in &tx.tx.inputs { let coin = tx_input .coin .as_ref() .ok_or(MissingTxInputCoin(tx_input.prev_out))?; spent_scripts.push(coin.output.script.clone()); } Ok(spent_scripts) } fn tx_token_inputs( &self, tx: &IndexTx<'_>, db_data: &BatchDbData, valid_txs: &HashMap, ) -> Result>> { if tx.is_coinbase { Ok(vec![]) } else { let mut inputs = Vec::with_capacity(tx.input_nums.len()); for (&input_num, input) in tx.input_nums.iter().zip(&tx.tx.inputs) { inputs.push(self.token_output( input_num, input.prev_out.out_idx as usize, db_data, valid_txs, )?); } Ok(inputs) } } fn token_output( &self, tx_num: TxNum, out_idx: usize, db_data: &BatchDbData, valid_txs: &HashMap, ) -> Result> { // Output is from this batch if let Some(token_tx) = valid_txs.get(&tx_num) { let Some(Some(token_output)) = token_tx.outputs.get(out_idx) else { return Ok(None); }; return Ok(Some(token_tx.spent_token(token_output))); } // Output is from the DB let Some(db_token_tx) = db_data.token_txs.get(&tx_num) else { return Ok(None); }; Ok(db_token_tx.spent_token( &db_token_tx.outputs[out_idx], |tx_num| { db_data .token_metas .get_by_left(&tx_num) .cloned() .ok_or(MissingTokenTxNum(tx_num)) }, )?) } } fn meta_idx(needle_meta: &TokenMeta, metas: &[TokenMeta]) -> u32 { metas .iter() .position(|meta| meta == needle_meta) .expect("TokenMeta should be in the list") as u32 } fn to_db_token(token: Option<&Token>, metas: &[TokenMeta]) -> DbToken { let Some(token) = token else { return DbToken::NoToken; }; match token.variant { TokenVariant::Amount(amount) => { DbToken::Amount(meta_idx(&token.meta, metas), amount) } TokenVariant::MintBaton => { DbToken::MintBaton(meta_idx(&token.meta, metas)) } TokenVariant::Unknown(token_type) => match token.meta.token_type { TokenType::Slp(_) => DbToken::UnknownSlp(token_type), TokenType::Alp(_) => DbToken::UnknownAlp(token_type), }, } } diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml index 133fb4759..1ba2fb332 100644 --- a/contrib/gitian-descriptors/gitian-linux.yml +++ b/contrib/gitian-descriptors/gitian-linux.yml @@ -1,245 +1,245 @@ --- name: "bitcoin-abc-linux" enable_cache: true distro: "debian" suites: - "bullseye" architectures: - "amd64" packages: - "autoconf" - "automake" - "binutils-aarch64-linux-gnu" - "binutils-arm-linux-gnueabihf" - "binutils-gold" - "bison" - "bsdmainutils" - "ca-certificates" - "clang" - "cmake" - "curl" - "faketime" # Use gcc/g++ 9 to avoid introducing the new pthread_cond_clockwait from glibc # 2.30, which would make our release binary incompatible with systems using an # older glibc version. - "g++-9" - "g++-9-aarch64-linux-gnu" - "g++-9-arm-linux-gnueabihf" - "gcc-9" - "gcc-9-aarch64-linux-gnu" - "gcc-9-arm-linux-gnueabihf" - "git" - "gperf" # Needed for Rocksdb - "libc6-dev-i386" - "libtool" - "ninja-build" - "pkg-config" - "protobuf-compiler" - "python3" - "python3-pip" remotes: - "url": "https://github.com/Bitcoin-ABC/bitcoin-abc.git" "dir": "bitcoin" files: [] script: | WRAP_DIR=$HOME/wrapped HOSTS=( x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu ) # CMake toolchain file name differ from host name declare -A CMAKE_TOOLCHAIN_FILE CMAKE_TOOLCHAIN_FILE[x86_64-linux-gnu]=Linux64.cmake CMAKE_TOOLCHAIN_FILE[arm-linux-gnueabihf]=LinuxARM.cmake CMAKE_TOOLCHAIN_FILE[aarch64-linux-gnu]=LinuxAArch64.cmake # Allow extra cmake option to be specified for each host declare -A CMAKE_EXTRA_OPTIONS # ARM assembly is supported but experimental, disable it for the release CMAKE_EXTRA_OPTIONS[arm-linux-gnueabihf]="-DSECP256K1_USE_ASM=OFF" FAKETIME_HOST_PROGS="" FAKETIME_PROGS="date ar ranlib nm" HOST_CFLAGS="-O2 -g" HOST_CXXFLAGS="-O2 -g" HOST_LDFLAGS=-static-libstdc++ export TZ="UTC" export BUILD_DIR=`pwd` mkdir -p ${WRAP_DIR} if test -n "$GBUILD_CACHE_ENABLED"; then export SOURCES_PATH=${GBUILD_COMMON_CACHE} export BASE_CACHE=${GBUILD_PACKAGE_CACHE} mkdir -p ${BASE_CACHE} ${SOURCES_PATH} fi function create_global_faketime_wrappers { for prog in ${FAKETIME_PROGS}; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} echo "REAL=\`which -a ${prog} | grep -v ${WRAP_DIR}/${prog} | head -1\`" >> ${WRAP_DIR}/${prog} echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${prog} echo "\$REAL \$@" >> $WRAP_DIR/${prog} chmod +x ${WRAP_DIR}/${prog} done } function create_per-host_faketime_wrappers { for i in ${HOSTS[@]}; do for prog in ${FAKETIME_HOST_PROGS}; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} echo "REAL=\`which -a ${i}-${prog} | grep -v ${WRAP_DIR}/${i}-${prog} | head -1\`" >> ${WRAP_DIR}/${i}-${prog} echo 'export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1' >> ${WRAP_DIR}/${i}-${prog} echo "export FAKETIME=\"$1\"" >> ${WRAP_DIR}/${i}-${prog} echo "\$REAL \"\$@\"" >> $WRAP_DIR/${i}-${prog} chmod +x ${WRAP_DIR}/${i}-${prog} done done } function create_per-host_compiler_wrapper { for i in ${HOSTS[@]}; do for prog in gcc g++; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${i}-${prog} echo "REAL=\`which -a ${i}-${prog}-9 | head -1\`" >> ${WRAP_DIR}/${i}-${prog} echo "\$REAL \"\$@\"" >> $WRAP_DIR/${i}-${prog} chmod +x ${WRAP_DIR}/${i}-${prog} done done } function create_native_compiler_wrapper { for prog in gcc g++; do echo '#!/usr/bin/env bash' > ${WRAP_DIR}/${prog} echo "REAL=\`which -a x86_64-linux-gnu-${prog}-9 | head -1\`" >> ${WRAP_DIR}/${prog} echo "\$REAL \"\$@\"" >> $WRAP_DIR/${prog} chmod +x ${WRAP_DIR}/${prog} done } pip3 install lief==0.13.2 # Faketime for depends so intermediate results are comparable export PATH_orig=${PATH} create_global_faketime_wrappers "2000-01-01 12:00:00" create_per-host_faketime_wrappers "2000-01-01 12:00:00" # Wrap the compiler -gcc-9 and -g++-9 into -gcc and # -g++ create_per-host_compiler_wrapper # For the current host platform also wrap into regular gcc and g++, assume # x86_64 create_native_compiler_wrapper export PATH=${WRAP_DIR}:${PATH} cd bitcoin SOURCEDIR=`pwd` BASEPREFIX=`pwd`/depends # Build dependencies for each host for i in ${HOSTS[@]}; do make ${MAKEOPTS} -C ${BASEPREFIX} HOST="${i}" done # Faketime for binaries export PATH=${PATH_orig} create_global_faketime_wrappers "${REFERENCE_DATETIME}" create_per-host_faketime_wrappers "${REFERENCE_DATETIME}" export PATH=${WRAP_DIR}:${PATH} mkdir -p source_package pushd source_package # Any toolchain file will work for building the source package, just pick the # first one cmake -GNinja .. \ -DCMAKE_TOOLCHAIN_FILE=${SOURCEDIR}/cmake/platforms/${CMAKE_TOOLCHAIN_FILE[${HOSTS[0]}]} ninja package_source SOURCEDIST=`echo bitcoin-abc-*.tar.gz` mv ${SOURCEDIST} .. popd DISTNAME=`echo ${SOURCEDIST} | sed 's/.tar.*//'` # Correct tar file order mkdir -p temp pushd temp tar -xf ../$SOURCEDIST find bitcoin-abc-* | sort | tar --mtime="${REFERENCE_DATETIME}" --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ../$SOURCEDIST popd ORIGPATH="$PATH" # Install chronik dependencies # Rust curl -sSf https://static.rust-lang.org/rustup/archive/1.26.0/x86_64-unknown-linux-gnu/rustup-init -o rustup-init echo "0b2f6c8f85a3d02fde2efc0ced4657869d73fccfce59defb4e8d29233116e6db rustup-init" | sha256sum -c chmod +x rustup-init - ./rustup-init -y --default-toolchain=1.72.0 + ./rustup-init -y --default-toolchain=1.76.0 # Rust target name differs from our host name, let's map declare -A RUST_TARGET RUST_TARGET[x86_64-linux-gnu]=x86_64-unknown-linux-gnu RUST_TARGET[arm-linux-gnueabihf]=arm-unknown-linux-gnueabihf RUST_TARGET[aarch64-linux-gnu]=aarch64-unknown-linux-gnu for i in ${HOSTS[@]}; do $HOME/.cargo/bin/rustup target add ${RUST_TARGET[${i}]} done # Cleanup rm -f rustup-init # Extend the hosts to include an experimental chronik build. # Despite not being a new host per se, it makes it easy to reuse the same code # and prevent errors. # TODO Remove after chronik is made part of the main release. for i in ${HOSTS[@]}; do HOSTS+=($i-chronik-experimental) CMAKE_TOOLCHAIN_FILE[$i-chronik-experimental]=${CMAKE_TOOLCHAIN_FILE[$i]} CMAKE_EXTRA_OPTIONS[$i-chronik-experimental]="${CMAKE_EXTRA_OPTIONS[$i]} -DBUILD_BITCOIN_CHRONIK=ON" done # Extract the release tarball into a dir for each host and build for i in ${HOSTS[@]}; do export PATH=${BASEPREFIX}/${i}/native/bin:${ORIGPATH} mkdir -p distsrc-${i} cd distsrc-${i} INSTALLPATH=`pwd`/installed/${DISTNAME} mkdir -p ${INSTALLPATH} cmake -GNinja .. \ -DCMAKE_TOOLCHAIN_FILE=${SOURCEDIR}/cmake/platforms/${CMAKE_TOOLCHAIN_FILE[${i}]} \ -DCLIENT_VERSION_IS_RELEASE=ON \ -DENABLE_CLANG_TIDY=OFF \ -DENABLE_REDUCE_EXPORTS=ON \ -DENABLE_STATIC_LIBSTDCXX=ON \ -DENABLE_GLIBC_BACK_COMPAT=ON \ -DCMAKE_INSTALL_PREFIX=${INSTALLPATH} \ -DCCACHE=OFF \ -DUSE_LINKER= \ ${CMAKE_EXTRA_OPTIONS[${i}]} ninja ninja security-check # TODO Rust pulls several symbols from GLIBC 2.30, this needs to be fixed. # Since it is still in an experimental state, ignore for now. if [[ "${i}" != *"chronik-experimental" ]]; then ninja symbol-check else # Install the chronik protobuf files ninja install-chronik-proto fi ninja install-debug cd installed find ${DISTNAME} -not -name "*.dbg" | sort | tar --mtime="${REFERENCE_DATETIME}" --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}.tar.gz find ${DISTNAME} -name "*.dbg" | sort | tar --mtime="${REFERENCE_DATETIME}" --no-recursion --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 -c -T - | gzip -9n > ${OUTDIR}/${DISTNAME}-${i}-debug.tar.gz cd ../../ rm -rf distsrc-${i} done mkdir -p $OUTDIR/src mv $SOURCEDIST $OUTDIR/src diff --git a/contrib/utils/install-dependencies-bullseye.sh b/contrib/utils/install-dependencies-bullseye.sh index 2be4f5d6a..e6c2ed07b 100755 --- a/contrib/utils/install-dependencies-bullseye.sh +++ b/contrib/utils/install-dependencies-bullseye.sh @@ -1,212 +1,212 @@ #!/usr/bin/env bash export LC_ALL=C.UTF-8 set -euxo pipefail dpkg --add-architecture i386 PACKAGES=( arcanist automake autotools-dev binutils bison bsdmainutils build-essential ccache cmake curl default-jdk devscripts doxygen dput g++-9 g++-9-aarch64-linux-gnu g++-9-arm-linux-gnueabihf g++-9-multilib g++-mingw-w64 gcc-9 gcc-9-aarch64-linux-gnu gcc-9-arm-linux-gnueabihf gcc-9-multilib gettext-base git golang gnupg graphviz gperf help2man jq lcov less lib32stdc++-10-dev libboost-dev libbz2-dev libc6-dev:i386 libcap-dev libdb++-dev libdb-dev libevent-dev libjemalloc-dev libminiupnpc-dev libnatpmp-dev libprotobuf-dev libpcsclite-dev libqrencode-dev libqt5core5a libqt5dbus5 libqt5gui5 libsqlite3-dev libssl-dev libtinfo5 libtool libzmq3-dev lld make ninja-build nsis pandoc php-codesniffer pkg-config protobuf-compiler python3 python3-pip python3-setuptools python3-yaml python3-zmq qttools5-dev qttools5-dev-tools shellcheck software-properties-common swig tar wget xorriso xvfb yamllint ) function join_by() { local IFS="$1" shift echo "$*" } apt-get update DEBIAN_FRONTEND=noninteractive apt-get install -y $(join_by ' ' "${PACKAGES[@]}") BACKPORTS=( git-filter-repo qemu-user-static ) echo "deb http://deb.debian.org/debian bullseye-backports main" | tee -a /etc/apt/sources.list apt-get update DEBIAN_FRONTEND=noninteractive apt-get -t bullseye-backports install -y $(join_by ' ' "${BACKPORTS[@]}") # Install llvm and clang apt-key add "$(dirname "$0")"/llvm.pub add-apt-repository "deb https://apt.llvm.org/bullseye/ llvm-toolchain-bullseye-12 main" apt-get update LLVM_PACKAGES=( clang-12 clang-format-12 clang-tidy-12 clang-tools-12 ) DEBIAN_FRONTEND=noninteractive apt-get install -y $(join_by ' ' "${LLVM_PACKAGES[@]}") # Make sure our specific llvm and clang versions have highest priority update-alternatives --install /usr/bin/clang clang "$(command -v clang-12)" 100 update-alternatives --install /usr/bin/clang++ clang++ "$(command -v clang++-12)" 100 update-alternatives --install /usr/bin/llvm-symbolizer llvm-symbolizer "$(command -v llvm-symbolizer-12)" 100 # Use gcc-9/g++-9 by default so it uses libstdc++-9. This prevents from pulling # the new pthread_cond_clockwait symbol from GLIBC_30 and ensure we are testing # under the same condition our release it built. update-alternatives --install /usr/bin/gcc gcc "$(command -v gcc-9)" 100 update-alternatives --install /usr/bin/g++ g++ "$(command -v g++-9)" 100 update-alternatives --install /usr/bin/aarch64-linux-gnu-gcc aarch64-linux-gnu-gcc "$(command -v aarch64-linux-gnu-gcc-9)" 100 update-alternatives --install /usr/bin/aarch64-linux-gnu-g++ aarch64-linux-gnu-g++ "$(command -v aarch64-linux-gnu-g++-9)" 100 update-alternatives --install /usr/bin/arm-linux-gnueabihf-gcc arm-linux-gnueabihf-gcc "$(command -v arm-linux-gnueabihf-gcc-9)" 100 update-alternatives --install /usr/bin/arm-linux-gnueabihf-g++ arm-linux-gnueabihf-g++ "$(command -v arm-linux-gnueabihf-g++-9)" 100 # Use the mingw posix variant update-alternatives --set x86_64-w64-mingw32-g++ $(command -v x86_64-w64-mingw32-g++-posix) update-alternatives --set x86_64-w64-mingw32-gcc $(command -v x86_64-w64-mingw32-gcc-posix) # Python library for merging nested structures pip3 install deepmerge # For running Python test suites pip3 install pytest # For en/-decoding protobuf messages # This version is compatible with Debian's "protobuf-compiler" package pip3 install "protobuf<=3.20" # For security-check.py and symbol-check.py pip3 install "lief==0.13.2" # For Chronik WebSocket endpoint pip3 install websocket-client # Required python linters pip3 install black==23.3.0 isort==5.6.4 mypy==0.910 flynt==0.78 flake8==6.0.0 echo "export PATH=\"$(python3 -m site --user-base)/bin:\$PATH\"" >> ~/.bashrc # shellcheck source=/dev/null source ~/.bashrc # Install npm v10.x and nodejs v20.x wget https://deb.nodesource.com/setup_20.x -O nodesetup.sh echo "f8fb478685fb916cc70858200595a4f087304bcde1e69aa713bf2eb41695afc1 nodesetup.sh" | sha256sum -c chmod +x nodesetup.sh ./nodesetup.sh apt-get install -y nodejs # Install nyc for mocha unit test reporting npm i -g nyc -# Install Rust stable 1.72.0 and nightly from the 2023-12-29 +# Install Rust stable 1.76.0 and nightly from the 2023-12-29 curl -sSf https://static.rust-lang.org/rustup/archive/1.26.0/x86_64-unknown-linux-gnu/rustup-init -o rustup-init echo "0b2f6c8f85a3d02fde2efc0ced4657869d73fccfce59defb4e8d29233116e6db rustup-init" | sha256sum -c chmod +x rustup-init -./rustup-init -y --default-toolchain=1.72.0 +./rustup-init -y --default-toolchain=1.76.0 RUST_HOME="${HOME}/.cargo/bin" RUST_NIGHTLY_DATE=2023-12-29 "${RUST_HOME}/rustup" install nightly-${RUST_NIGHTLY_DATE} "${RUST_HOME}/rustup" component add rustfmt --toolchain nightly-${RUST_NIGHTLY_DATE} # Name the nightly toolchain "abc-nightly" "${RUST_HOME}/rustup" toolchain link abc-nightly "$(${RUST_HOME}/rustc +nightly-${RUST_NIGHTLY_DATE} --print sysroot)" # Install required compile platform targets on stable "${RUST_HOME}/rustup" target add "i686-unknown-linux-gnu" \ "x86_64-unknown-linux-gnu" \ "aarch64-unknown-linux-gnu" \ "arm-unknown-linux-gnueabihf" \ "x86_64-apple-darwin" \ "x86_64-pc-windows-gnu" # Install Electrum ABC test dependencies here=$(dirname -- "$(readlink -f -- "${BASH_SOURCE[0]}")") pip3 install -r "${here}/../../electrum/contrib/requirements/requirements.txt" pip3 install -r "${here}/../../electrum/contrib/requirements/requirements-regtest.txt" pip3 install -r "${here}/../../electrum/contrib/requirements/requirements-hw.txt" # Install the winehq-staging version of wine that doesn't suffer from the memory # limitations of the previous versions. Installation instructions are from # https://wiki.winehq.org/Debian mkdir -p /etc/apt/keyrings wget -O /etc/apt/keyrings/winehq-archive.key https://dl.winehq.org/wine-builds/winehq.key wget -NP /etc/apt/sources.list.d/ https://dl.winehq.org/wine-builds/debian/dists/bullseye/winehq-bullseye.sources apt-get update WINE_VERSION=8.19~bullseye-1 # We need all the packages and dependencies to use a pinpointed vesion WINE_PACKAGES=( winehq-staging wine-staging wine-staging-amd64 wine-staging-i386 ) # Pinpoint the version so we get consistent results on CI DEBIAN_FRONTEND=noninteractive apt-get install -y $(join_by ' ' "${WINE_PACKAGES[@]/%/=${WINE_VERSION}}")