Changeset View
Changeset View
Standalone View
Standalone View
contrib/linearize/linearize-data.py
| Show First 20 Lines • Show All 168 Lines • ▼ Show 20 Lines | def writeBlock(self, inhdr, blk_hdr, rawblock): | ||||
| self.outFn = self.outFn + 1 | self.outFn = self.outFn + 1 | ||||
| self.outsz = 0 | self.outsz = 0 | ||||
| if not self.outF: | if not self.outF: | ||||
| if self.fileOutput: | if self.fileOutput: | ||||
| self.outFname = self.settings['output_file'] | self.outFname = self.settings['output_file'] | ||||
| else: | else: | ||||
| self.outFname = os.path.join( | self.outFname = os.path.join( | ||||
| self.settings['output'], "blk%05d.dat" % self.outFn) | self.settings['output'], "blk{:05d}.dat".format(self.outFn)) | ||||
| print("Output file " + self.outFname) | print("Output file " + self.outFname) | ||||
| self.outF = open(self.outFname, "wb") | self.outF = open(self.outFname, "wb") | ||||
| self.outF.write(inhdr) | self.outF.write(inhdr) | ||||
| self.outF.write(blk_hdr) | self.outF.write(blk_hdr) | ||||
| self.outF.write(rawblock) | self.outF.write(rawblock) | ||||
| self.outsz = self.outsz + len(inhdr) + len(blk_hdr) + len(rawblock) | self.outsz = self.outsz + len(inhdr) + len(blk_hdr) + len(rawblock) | ||||
| self.blkCountOut = self.blkCountOut + 1 | self.blkCountOut = self.blkCountOut + 1 | ||||
| if blkTS > self.highTS: | if blkTS > self.highTS: | ||||
| self.highTS = blkTS | self.highTS = blkTS | ||||
| if (self.blkCountOut % 1000) == 0: | if (self.blkCountOut % 1000) == 0: | ||||
| print('%i blocks scanned, %i blocks written (of %i, %.1f%% complete)' % | print('{} blocks scanned, {} blocks written (of {}, {:.1f}% complete)'.format( | ||||
| (self.blkCountIn, self.blkCountOut, len(self.blkindex), 100.0 * self.blkCountOut / len(self.blkindex))) | self.blkCountIn, self.blkCountOut, len(self.blkindex), 100.0 * self.blkCountOut / len(self.blkindex))) | ||||
| def inFileName(self, fn): | def inFileName(self, fn): | ||||
| return os.path.join(self.settings['input'], "blk%05d.dat" % fn) | return os.path.join(self.settings['input'], "blk{:05d}.dat".format(fn)) | ||||
| def fetchBlock(self, extent): | def fetchBlock(self, extent): | ||||
| '''Fetch block contents from disk given extents''' | '''Fetch block contents from disk given extents''' | ||||
| with open(self.inFileName(extent.fn), "rb") as f: | with open(self.inFileName(extent.fn), "rb") as f: | ||||
| f.seek(extent.offset) | f.seek(extent.offset) | ||||
| return f.read(extent.size) | return f.read(extent.size) | ||||
| def copyOneBlock(self): | def copyOneBlock(self): | ||||
| ▲ Show 20 Lines • Show All 64 Lines • ▼ Show 20 Lines | def run(self): | ||||
| # If there is space in the cache, read the data | # If there is space in the cache, read the data | ||||
| # Reading the data in file sequence instead of seeking and fetching it later is preferred, | # Reading the data in file sequence instead of seeking and fetching it later is preferred, | ||||
| # but we don't want to fill up memory | # but we don't want to fill up memory | ||||
| self.outOfOrderData[blkHeight] = self.inF.read(inLen) | self.outOfOrderData[blkHeight] = self.inF.read(inLen) | ||||
| self.outOfOrderSize += inLen | self.outOfOrderSize += inLen | ||||
| else: # If no space in cache, seek forward | else: # If no space in cache, seek forward | ||||
| self.inF.seek(inLen, os.SEEK_CUR) | self.inF.seek(inLen, os.SEEK_CUR) | ||||
| print("Done (%i blocks written)" % (self.blkCountOut)) | print("Done ({} blocks written)".format((self.blkCountOut))) | ||||
| if __name__ == '__main__': | if __name__ == '__main__': | ||||
| if len(sys.argv) != 2: | if len(sys.argv) != 2: | ||||
| print("Usage: linearize-data.py CONFIG-FILE") | print("Usage: linearize-data.py CONFIG-FILE") | ||||
| sys.exit(1) | sys.exit(1) | ||||
| f = open(sys.argv[1]) | f = open(sys.argv[1]) | ||||
| ▲ Show 20 Lines • Show All 57 Lines • Show Last 20 Lines | |||||