Changeset View
Changeset View
Standalone View
Standalone View
contrib/linearize/linearize-data.py
| Show First 20 Lines • Show All 139 Lines • ▼ Show 20 Lines | def __init__(self, settings, blkindex, blkmap): | ||||
| self.timestampSplit = True | self.timestampSplit = True | ||||
| # Extents and cache for out-of-order blocks | # Extents and cache for out-of-order blocks | ||||
| self.blockExtents = {} | self.blockExtents = {} | ||||
| self.outOfOrderData = {} | self.outOfOrderData = {} | ||||
| self.outOfOrderSize = 0 # running total size for items in outOfOrderData | self.outOfOrderSize = 0 # running total size for items in outOfOrderData | ||||
| def writeBlock(self, inhdr, blk_hdr, rawblock): | def writeBlock(self, inhdr, blk_hdr, rawblock): | ||||
| blockSizeOnDisk = len(inhdr) + len(blk_hdr) + len(rawblock) | blockSizeOnDisk = len(inhdr) + len(blk_hdr) + len(rawblock) | ||||
| if not self.fileOutput and ((self.outsz + blockSizeOnDisk) > self.maxOutSz): | if not self.fileOutput and ( | ||||
| (self.outsz + blockSizeOnDisk) > self.maxOutSz): | |||||
| self.outF.close() | self.outF.close() | ||||
| if self.setFileTime: | if self.setFileTime: | ||||
| os.utime(self.outFname, (int(time.time()), self.highTS)) | os.utime(self.outFname, (int(time.time()), self.highTS)) | ||||
| self.outF = None | self.outF = None | ||||
| self.outFname = None | self.outFname = None | ||||
| self.outFn = self.outFn + 1 | self.outFn = self.outFn + 1 | ||||
| self.outsz = 0 | self.outsz = 0 | ||||
| ▲ Show 20 Lines • Show All 41 Lines • ▼ Show 20 Lines | def fetchBlock(self, extent): | ||||
| with open(self.inFileName(extent.fn), "rb") as f: | with open(self.inFileName(extent.fn), "rb") as f: | ||||
| f.seek(extent.offset) | f.seek(extent.offset) | ||||
| return f.read(extent.size) | return f.read(extent.size) | ||||
| def copyOneBlock(self): | def copyOneBlock(self): | ||||
| '''Find the next block to be written in the input, and copy it to the output.''' | '''Find the next block to be written in the input, and copy it to the output.''' | ||||
| extent = self.blockExtents.pop(self.blkCountOut) | extent = self.blockExtents.pop(self.blkCountOut) | ||||
| if self.blkCountOut in self.outOfOrderData: | if self.blkCountOut in self.outOfOrderData: | ||||
| # If the data is cached, use it from memory and remove from the cache | # If the data is cached, use it from memory and remove from the | ||||
| # cache | |||||
| rawblock = self.outOfOrderData.pop(self.blkCountOut) | rawblock = self.outOfOrderData.pop(self.blkCountOut) | ||||
| self.outOfOrderSize -= len(rawblock) | self.outOfOrderSize -= len(rawblock) | ||||
| else: # Otherwise look up data on disk | else: # Otherwise look up data on disk | ||||
| rawblock = self.fetchBlock(extent) | rawblock = self.fetchBlock(extent) | ||||
| self.writeBlock(extent.inhdr, extent.blkhdr, rawblock) | self.writeBlock(extent.inhdr, extent.blkhdr, rawblock) | ||||
| def run(self): | def run(self): | ||||
| Show All 23 Lines | def run(self): | ||||
| inLen = su[0] - 80 # length without header | inLen = su[0] - 80 # length without header | ||||
| blk_hdr = self.inF.read(80) | blk_hdr = self.inF.read(80) | ||||
| inExtent = BlockExtent( | inExtent = BlockExtent( | ||||
| self.inFn, self.inF.tell(), inhdr, blk_hdr, inLen) | self.inFn, self.inF.tell(), inhdr, blk_hdr, inLen) | ||||
| self.hash_str = calc_hash_str(blk_hdr) | self.hash_str = calc_hash_str(blk_hdr) | ||||
| if self.hash_str not in blkmap: | if self.hash_str not in blkmap: | ||||
| # Because blocks can be written to files out-of-order as of 0.10, the script | # Because blocks can be written to files out-of-order as of 0.10, the script | ||||
| # may encounter blocks it doesn't know about. Treat as debug output. | # may encounter blocks it doesn't know about. Treat as debug | ||||
| # output. | |||||
| if settings['debug_output'] == 'true': | if settings['debug_output'] == 'true': | ||||
| print("Skipping unknown block " + self.hash_str) | print("Skipping unknown block " + self.hash_str) | ||||
| self.inF.seek(inLen, os.SEEK_CUR) | self.inF.seek(inLen, os.SEEK_CUR) | ||||
| continue | continue | ||||
| blkHeight = self.blkmap[self.hash_str] | blkHeight = self.blkmap[self.hash_str] | ||||
| self.blkCountIn += 1 | self.blkCountIn += 1 | ||||
| ▲ Show 20 Lines • Show All 86 Lines • Show Last 20 Lines | |||||