diff --git a/bloscpack/append.py b/bloscpack/append.py index 5ffac81d..0cc6ceb2 100644 --- a/bloscpack/append.py +++ b/bloscpack/append.py @@ -112,11 +112,9 @@ def append_fp(original_fp, new_content_fp, new_size, blosc_args=None): (METADATA_HEADER_LENGTH + metadata_header['max_meta_size'] + CHECKSUMS_LOOKUP[metadata_header['meta_checksum']].size if metadata is not None else 0)) - # seek to the final offset - original_fp.seek(offsets[-1], 0) # decompress the last chunk compressed, blosc_header, digest = _read_compressed_chunk_fp(original_fp, - checksum_impl) + checksum_impl, offsets[-1]) # TODO check digest decompressed = blosc.decompress(compressed) # figure out how many bytes we need to read to rebuild the last chunk diff --git a/bloscpack/cli.py b/bloscpack/cli.py index fea7548d..fad02d48 100644 --- a/bloscpack/cli.py +++ b/bloscpack/cli.py @@ -510,8 +510,9 @@ def main(): _read_beginning(fp) checksum_impl = bloscpack_header.checksum_impl # get the header of the first chunk + # pass -1 to avoid seeking _, blosc_header, _ = _read_compressed_chunk_fp( - fp, checksum_impl) + fp, checksum_impl, -1) except ValueError as ve: log.error(str(ve) + "\n" + "This might not be a bloscpack compressed file.") diff --git a/bloscpack/file_io.py b/bloscpack/file_io.py index 2e802dad..b9205e61 100644 --- a/bloscpack/file_io.py +++ b/bloscpack/file_io.py @@ -298,7 +298,7 @@ def _write_offsets(output_fp, offsets): output_fp.write(encoded_offsets) -def _read_compressed_chunk_fp(input_fp, checksum_impl): +def _read_compressed_chunk_fp(input_fp, checksum_impl, offset): """ Read a compressed chunk from a file pointer. Parameters @@ -307,6 +307,8 @@ def _read_compressed_chunk_fp(input_fp, checksum_impl): the file pointer to read the chunk from checksum_impl : Checksum the checksum that has been used + offset: int + the offset for the file pointer Returns ------- @@ -318,6 +320,8 @@ def _read_compressed_chunk_fp(input_fp, checksum_impl): the checksum of the chunk """ # read blosc header + if offset >= 0: + input_fp.seek(offset) blosc_header_raw = input_fp.read(BLOSC_HEADER_LENGTH) blosc_header = decode_blosc_header(blosc_header_raw) if log.LEVEL == log.DEBUG: @@ -365,7 +369,7 @@ def __init__(self, input_fp): def __iter__(self): for i in xrange(self.nchunks): - compressed, header, digest = _read_compressed_chunk_fp(self.input_fp, self.checksum_impl) + compressed, header, digest = _read_compressed_chunk_fp(self.input_fp, self.checksum_impl, self.offsets[i]) yield compressed, digest diff --git a/test/test_append.py b/test/test_append.py index c2413338..296388b3 100644 --- a/test/test_append.py +++ b/test/test_append.py @@ -366,14 +366,12 @@ def test_append_mix_shuffle(): # now get the first and the last chunk and check that the shuffle doesn't # match bloscpack_header, offsets = reset_read_beginning(orig)[0:4:3] - orig.seek(offsets[0]) checksum_impl = CHECKSUMS_LOOKUP[bloscpack_header['checksum']] compressed_zero, blosc_header_zero, digest = \ - _read_compressed_chunk_fp(orig, checksum_impl) + _read_compressed_chunk_fp(orig, checksum_impl, offsets[0]) decompressed_zero = blosc.decompress(compressed_zero) - orig.seek(offsets[-1]) compressed_last, blosc_header_last, digest = \ - _read_compressed_chunk_fp(orig, checksum_impl) + _read_compressed_chunk_fp(orig, checksum_impl, offsets[-1]) decompressed_last = blosc.decompress(compressed_last) # first chunk has shuffle active nt.assert_equal(blosc_header_zero['flags'], 1)