diff --git a/contrib/linearize/README.md b/contrib/linearize/README.md index 0971e7816..f2a2ab276 100644 --- a/contrib/linearize/README.md +++ b/contrib/linearize/README.md @@ -1,54 +1,55 @@ # Linearize Construct a linear, no-fork, best version of the Bitcoin blockchain. The scripts run using Python 3 but are compatible with Python 2. ## Step 1: Download hash list $ ./linearize-hashes.py linearize.cfg > hashlist.txt Required configuration file settings for linearize-hashes: -* RPC: `rpcuser`, `rpcpassword` +* RPC: `datadir` (Required if `rpcuser` and `rpcpassword` are not specified) +* RPC: `rpcuser`, `rpcpassword` (Required if `datadir` is not specified) Optional config file setting for linearize-hashes: * RPC: `host` (Default: `127.0.0.1`) * RPC: `port` (Default: `8332`) * Blockchain: `min_height`, `max_height` * `rev_hash_bytes`: If true, the written block hash list will be byte-reversed. (In other words, the hash returned by getblockhash will have its bytes reversed.) False by default. Intended for generation of standalone hash lists but safe to use with linearize-data.py, which will output the same data no matter which byte format is chosen. The `linearize-hashes` script requires a connection, local or remote, to a JSON-RPC server. Running `bitcoind` or `bitcoin-qt -server` will be sufficient. ## Step 2: Copy local block data $ ./linearize-data.py linearize.cfg Required configuration file settings: * `output_file`: The file that will contain the final blockchain. or * `output`: Output directory for linearized `blocks/blkNNNNN.dat` output. Optional config file setting for linearize-data: * `debug_output`: Some printouts may not always be desired. If true, such output will be printed. * `file_timestamp`: Set each file's last-accessed and last-modified times, respectively, to the current time and to the timestamp of the most recent block written to the script's blockchain. * `genesis`: The hash of the genesis block in the blockchain. * `input`: bitcoind blocks/ directory containing blkNNNNN.dat * `hashlist`: text file containing list of block hashes created by linearize-hashes.py. * `max_out_sz`: Maximum size for files created by the `output_file` option. (Default: `1000*1000*1000 bytes`) * `netmagic`: Network magic number. * `out_of_order_cache_sz`: If out-of-order blocks are being read, the block can be written to a cache so that the blockchain doesn't have to be seeked again. This option specifies the cache size. (Default: `100*1000*1000 bytes`) * `rev_hash_bytes`: If true, the block hash list written by linearize-hashes.py will be byte-reversed when read by linearize-data.py. See the linearize-hashes entry for more information. * `split_timestamp`: Split blockchain files when a new month is first seen, in addition to reaching a maximum file size (`max_out_sz`). diff --git a/contrib/linearize/example-linearize.cfg b/contrib/linearize/example-linearize.cfg index 2cc910edf..d019b06b6 100644 --- a/contrib/linearize/example-linearize.cfg +++ b/contrib/linearize/example-linearize.cfg @@ -1,42 +1,43 @@ # bitcoind RPC settings (linearize-hashes) rpcuser=someuser rpcpassword=somepassword +#datadir=~/.bitcoin host=127.0.0.1 port=8332 #port=18332 # bootstrap.dat hashlist settings (linearize-hashes) max_height=313000 # bootstrap.dat input/output settings (linearize-data) # mainnet netmagic=f9beb4d9 genesis=000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f input=/home/example/.bitcoin/blocks # testnet #netmagic=0b110907 #genesis=000000000933ea01ad0ee984209779baaec3ced90fa3f408719526f8d77f4943 #input=/home/example/.bitcoin/testnet3/blocks # "output" option causes blockchain files to be written to the given location, # with "output_file" ignored. If not used, "output_file" is used instead. # output=/home/example/blockchain_directory output_file=/home/example/Downloads/bootstrap.dat hashlist=hashlist.txt # Maximum size in bytes of out-of-order blocks cache in memory out_of_order_cache_sz = 100000000 # Do we want the reverse the hash bytes coming from getblockhash? rev_hash_bytes = False # On a new month, do we want to set the access and modify times of the new # blockchain file? file_timestamp = 0 # Do we want to split the blockchain files given a new month or specific height? split_timestamp = 0 # Do we want debug printouts? debug_output = False diff --git a/contrib/linearize/linearize-hashes.py b/contrib/linearize/linearize-hashes.py index 555ff2926..facfea9a1 100755 --- a/contrib/linearize/linearize-hashes.py +++ b/contrib/linearize/linearize-hashes.py @@ -1,143 +1,165 @@ #!/usr/bin/env python3 # # linearize-hashes.py: List blocks in a linear, no-fork version of the chain. # # Copyright (c) 2013-2016 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. # from __future__ import print_function try: # Python 3 import http.client as httplib except ImportError: # Python 2 import httplib import json import re import base64 import sys +import os +import os.path settings = {} ##### Switch endian-ness ##### def hex_switchEndian(s): """ Switches the endianness of a hex string (in pairs of hex chars) """ pairList = [s[i:i + 2].encode() for i in range(0, len(s), 2)] return b''.join(pairList[::-1]).decode() class BitcoinRPC: def __init__(self, host, port, username, password): authpair = "%s:%s" % (username, password) authpair = authpair.encode('utf-8') self.authhdr = b"Basic " + base64.b64encode(authpair) self.conn = httplib.HTTPConnection(host, port=port, timeout=30) def execute(self, obj): try: self.conn.request('POST', '/', json.dumps(obj), {'Authorization': self.authhdr, 'Content-type': 'application/json'}) except ConnectionRefusedError: print('RPC connection refused. Check RPC settings and the server status.', file=sys.stderr) return None resp = self.conn.getresponse() if resp is None: print("JSON-RPC: no response", file=sys.stderr) return None body = resp.read().decode('utf-8') resp_obj = json.loads(body) return resp_obj @staticmethod def build_request(idx, method, params): obj = {'version': '1.1', 'method': method, 'id': idx} if params is None: obj['params'] = [] else: obj['params'] = params return obj @staticmethod def response_is_error(resp_obj): return 'error' in resp_obj and resp_obj['error'] is not None def get_block_hashes(settings, max_blocks_per_call=10000): rpc = BitcoinRPC(settings['host'], settings['port'], settings['rpcuser'], settings['rpcpassword']) height = settings['min_height'] while height < settings['max_height'] + 1: num_blocks = min(settings['max_height'] + 1 - height, max_blocks_per_call) batch = [] for x in range(num_blocks): batch.append(rpc.build_request(x, 'getblockhash', [height + x])) reply = rpc.execute(batch) if reply is None: print('Cannot continue. Program will halt.') return None for x, resp_obj in enumerate(reply): if rpc.response_is_error(resp_obj): print('JSON-RPC: error at height', height + x, ': ', resp_obj['error'], file=sys.stderr) exit(1) assert(resp_obj['id'] == x) # assume replies are in-sequence if settings['rev_hash_bytes'] == 'true': resp_obj['result'] = hex_switchEndian(resp_obj['result']) print(resp_obj['result']) height += num_blocks +def get_rpc_cookie(): + # Open the cookie file + with open(os.path.join(os.path.expanduser(settings['datadir']), '.cookie'), 'r') as f: + combined = f.readline() + combined_split = combined.split(":") + settings['rpcuser'] = combined_split[0] + settings['rpcpassword'] = combined_split[1] + + if __name__ == '__main__': if len(sys.argv) != 2: print("Usage: linearize-hashes.py CONFIG-FILE") sys.exit(1) f = open(sys.argv[1]) for line in f: # skip comment lines m = re.search('^\s*#', line) if m: continue # parse key=value lines m = re.search('^(\w+)\s*=\s*(\S.*)$', line) if m is None: continue settings[m.group(1)] = m.group(2) f.close() if 'host' not in settings: settings['host'] = '127.0.0.1' if 'port' not in settings: settings['port'] = 8332 if 'min_height' not in settings: settings['min_height'] = 0 if 'max_height' not in settings: settings['max_height'] = 313000 if 'rev_hash_bytes' not in settings: settings['rev_hash_bytes'] = 'false' + + use_userpass = True + use_datadir = False if 'rpcuser' not in settings or 'rpcpassword' not in settings: - print("Missing username and/or password in cfg file", file=sys.stderr) + use_userpass = False + if 'datadir' in settings and not use_userpass: + use_datadir = True + if not use_userpass and not use_datadir: + print("Missing datadir or username and/or password in cfg file", file=sys.stderr) sys.exit(1) settings['port'] = int(settings['port']) settings['min_height'] = int(settings['min_height']) settings['max_height'] = int(settings['max_height']) # Force hash byte format setting to be lowercase to make comparisons easier. settings['rev_hash_bytes'] = settings['rev_hash_bytes'].lower() + # Get the rpc user and pass from the cookie if the datadir is set + if use_datadir: + get_rpc_cookie() + get_block_hashes(settings)