pax_global_header00006660000000000000000000000064131725722740014524gustar00rootroot0000000000000052 comment=ebce570fc88f71e46cfc7eb88ff85f1fce7df082 indexed-gzip-0.6.1/000077500000000000000000000000001317257227400141175ustar00rootroot00000000000000indexed-gzip-0.6.1/.ci/000077500000000000000000000000001317257227400145705ustar00rootroot00000000000000indexed-gzip-0.6.1/.ci/run_32bit_test.sh000077500000000000000000000014771317257227400200060ustar00rootroot00000000000000#!/bin/bash # # This script is called via .travis.yml. It is not intended # to be called in any other manner. # PYV=$PYTHON_VERSION apt-get update apt-get install -y libssl-dev openssl wget build-essential cd / wget https://www.python.org/ftp/python/$PYV/Python-$PYV.tar.xz tar xf Python-$PYV.tar.xz cd Python-$PYV ./configure --prefix=/mypython make && make install cd /mypython/bin if [ ! -e python ]; then ln -s python3 python fi if [ ! -e pip ]; then if [ -e pip3 ]; then ln -s pip3 pip else wget https://bootstrap.pypa.io/get-pip.py ./python get-pip.py fi fi export PATH=/mypython/bin:$PATH pip install numpy pytest cython cd /indexed_gzip python setup.py develop python setup.py test --addopts "-v -s --niters 500"; python setup.py test --addopts "-v -s --niters 500 --concat"; indexed-gzip-0.6.1/.ci/run_tests.sh000077500000000000000000000015521317257227400171600ustar00rootroot00000000000000#!/bin/bash # # This script is called via .travis.yml. It is not intended # to be called in any other manner. # # Get the path to this script script_dir=`dirname $0` pushd $script_dir > /dev/null script_dir=`pwd` pushd .. > /dev/null igzip_dir=`pwd` popd > /dev/null popd > /dev/null # 32 bit platform test has to be run in a docker container if [ "$TEST_SUITE" == "32bittest" ]; then PYTHON_VERSION=$(python --version 2>&1) PYTHON_VERSION=${PYTHON_VERSION#* } docker run --rm \ -e PYTHON_VERSION="$PYTHON_VERSION" \ -v $igzip_dir:/indexed_gzip \ 32bit/ubuntu:16.04 \ /indexed_gzip/.ci/run_32bit_test.sh # Run standard test suite else python setup.py develop; python setup.py test --addopts "-v -s -m \"$TEST_SUITE\" -k \"$TEST_PATTERN\" --nelems \"$NELEMS\" --niters \"$NITERS\" $EXTRA_ARGS"; fi indexed-gzip-0.6.1/.travis.yml000066400000000000000000000205031317257227400162300ustar00rootroot00000000000000branches: only: - master sudo: required language: python python: - "2.7" - "3.5" services: - docker env: - TEST_SUITE=zran_test NITERS=5000 NELEMS=50000 TEST_PATTERN="not test_readbuf_spacing_sizes and not test_seek_then_read_block" - TEST_SUITE=zran_test EXTRA_ARGS="--concat" NITERS=5000 NELEMS=50000 TEST_PATTERN="not test_readbuf_spacing_sizes and not test_seek_then_read_block" - TEST_SUITE=zran_test NITERS=1000 NELEMS=50000 TEST_PATTERN="test_readbuf_spacing_sizes" - TEST_SUITE=zran_test EXTRA_ARGS="--concat" NITERS=1000 NELEMS=50000 TEST_PATTERN="test_readbuf_spacing_sizes" - TEST_SUITE=zran_test NITERS=1000 NELEMS=50000 TEST_PATTERN="test_seek_then_read_block" - TEST_SUITE=zran_test EXTRA_ARGS="--concat" NITERS=1000 NELEMS=50000 TEST_PATTERN="test_seek_then_read_block" - TEST_SUITE=indexed_gzip_test NITERS=5000 NELEMS=50000 - TEST_SUITE=indexed_gzip_test EXTRA_ARGS="--concat" NITERS=5000 NELEMS=50000 - TEST_SUITE=zran_test NITERS=5000 NELEMS=131072 TEST_PATTERN="not test_readbuf_spacing_sizes and not test_seek_then_read_block" - TEST_SUITE=zran_test EXTRA_ARGS="--concat" NITERS=5000 NELEMS=131072 TEST_PATTERN="not test_readbuf_spacing_sizes and not test_seek_then_read_block" - TEST_SUITE=zran_test NITERS=1000 NELEMS=131072 TEST_PATTERN="test_readbuf_spacing_sizes" - TEST_SUITE=zran_test EXTRA_ARGS="--concat" NITERS=1000 NELEMS=131072 TEST_PATTERN="test_readbuf_spacing_sizes" - TEST_SUITE=zran_test NITERS=1000 NELEMS=131072 TEST_PATTERN="test_seek_then_read_block" - TEST_SUITE=zran_test EXTRA_ARGS="--concat" NITERS=1000 NELEMS=131072 TEST_PATTERN="test_seek_then_read_block" - TEST_SUITE=indexed_gzip_test NITERS=5000 NELEMS=131072 - TEST_SUITE=indexed_gzip_test EXTRA_ARGS="--concat" NITERS=5000 NELEMS=131072 - TEST_SUITE=zran_test NITERS=5000 NELEMS=1000000 TEST_PATTERN="not test_readbuf_spacing_sizes and not test_seek_then_read_block" - TEST_SUITE=zran_test EXTRA_ARGS="--concat" NITERS=5000 NELEMS=1000000 TEST_PATTERN="not test_readbuf_spacing_sizes and not test_seek_then_read_block" - TEST_SUITE=zran_test NITERS=1000 NELEMS=1000000 TEST_PATTERN="test_readbuf_spacing_sizes" - TEST_SUITE=zran_test EXTRA_ARGS="--concat" NITERS=1000 NELEMS=1000000 TEST_PATTERN="test_readbuf_spacing_sizes" - TEST_SUITE=zran_test NITERS=1000 NELEMS=1000000 TEST_PATTERN="test_seek_then_read_block" - TEST_SUITE=zran_test EXTRA_ARGS="--concat" NITERS=1000 NELEMS=1000000 TEST_PATTERN="test_seek_then_read_block" - TEST_SUITE=indexed_gzip_test NITERS=5000 NELEMS=1000000 - TEST_SUITE=indexed_gzip_test EXTRA_ARGS="--concat" NITERS=5000 NELEMS=1000000 - TEST_SUITE=zran_test NITERS=5000 NELEMS=1048576 TEST_PATTERN="not test_readbuf_spacing_sizes and not test_seek_then_read_block" - TEST_SUITE=zran_test EXTRA_ARGS="--concat" NITERS=5000 NELEMS=1048576 TEST_PATTERN="not test_readbuf_spacing_sizes and not test_seek_then_read_block" - TEST_SUITE=zran_test NITERS=1000 NELEMS=1048576 TEST_PATTERN="test_readbuf_spacing_sizes" - TEST_SUITE=zran_test EXTRA_ARGS="--concat" NITERS=1000 NELEMS=1048576 TEST_PATTERN="test_readbuf_spacing_sizes" - TEST_SUITE=zran_test NITERS=1000 NELEMS=1048576 TEST_PATTERN="test_seek_then_read_block" - TEST_SUITE=zran_test EXTRA_ARGS="--concat" NITERS=1000 NELEMS=1048576 TEST_PATTERN="test_seek_then_read_block" - TEST_SUITE=indexed_gzip_test NITERS=5000 NELEMS=1048576 - TEST_SUITE=indexed_gzip_test EXTRA_ARGS="--concat" NITERS=5000 NELEMS=1048576 # NELEMS=something arbitrary that is not divisible # by any of the default buffer/spacing sizes - TEST_SUITE=zran_test NITERS=5000 NELEMS=2000000 TEST_PATTERN="not test_readbuf_spacing_sizes and not test_seek_then_read_block" - TEST_SUITE=zran_test EXTRA_ARGS="--concat" NITERS=5000 NELEMS=2000000 TEST_PATTERN="not test_readbuf_spacing_sizes and not test_seek_then_read_block" - TEST_SUITE=zran_test NITERS=500 NELEMS=2000000 TEST_PATTERN="test_readbuf_spacing_sizes" - TEST_SUITE=zran_test EXTRA_ARGS="--concat" NITERS=500 NELEMS=2000000 TEST_PATTERN="test_readbuf_spacing_sizes" - TEST_SUITE=zran_test NITERS=500 NELEMS=2000000 TEST_PATTERN="test_seek_then_read_block" - TEST_SUITE=zran_test EXTRA_ARGS="--concat" NITERS=500 NELEMS=2000000 TEST_PATTERN="test_seek_then_read_block" - TEST_SUITE=indexed_gzip_test NITERS=5000 NELEMS=2000000 - TEST_SUITE=indexed_gzip_test EXTRA_ARGS="--concat" NITERS=5000 NELEMS=2000000 # NELEMS=16777216 ~ 128MB - TEST_SUITE=zran_test NITERS=5000 NELEMS=16777216 TEST_PATTERN="not test_readbuf_spacing_sizes and not test_seek_then_read_block" - TEST_SUITE=zran_test EXTRA_ARGS="--concat" NITERS=5000 NELEMS=16777216 TEST_PATTERN="not test_readbuf_spacing_sizes and not test_seek_then_read_block" - TEST_SUITE=zran_test NITERS=500 NELEMS=16777216 TEST_PATTERN="test_readbuf_spacing_sizes" - TEST_SUITE=zran_test EXTRA_ARGS="--concat" NITERS=500 NELEMS=16777216 TEST_PATTERN="test_readbuf_spacing_sizes" - TEST_SUITE=zran_test NITERS=500 NELEMS=16777216 TEST_PATTERN="test_seek_then_read_block" - TEST_SUITE=zran_test EXTRA_ARGS="--concat" NITERS=500 NELEMS=16777216 TEST_PATTERN="test_seek_then_read_block" - TEST_SUITE=indexed_gzip_test NITERS=5000 NELEMS=16777216 - TEST_SUITE=indexed_gzip_test EXTRA_ARGS="--concat" NITERS=5000 NELEMS=16777216 # NELEMS=268435456 ~ 2GB - TEST_SUITE=zran_test EXTRA_ARGS="--use_mmap" NITERS=5000 NELEMS=268435456 TEST_PATTERN="not test_readbuf_spacing_sizes and not test_seek_then_read_block" - TEST_SUITE=zran_test EXTRA_ARGS="--use_mmap --concat" NITERS=5000 NELEMS=268435456 TEST_PATTERN="not test_readbuf_spacing_sizes and not test_seek_then_read_block" - TEST_SUITE=zran_test EXTRA_ARGS="--use_mmap" NITERS=500 NELEMS=268435456 TEST_PATTERN="test_readbuf_spacing_sizes" - TEST_SUITE=zran_test EXTRA_ARGS="--use_mmap --concat" NITERS=500 NELEMS=268435456 TEST_PATTERN="test_readbuf_spacing_sizes" - TEST_SUITE=zran_test EXTRA_ARGS="--use_mmap" NITERS=250 NELEMS=268435456 TEST_PATTERN="test_seek_then_read_block" - TEST_SUITE=zran_test EXTRA_ARGS="--use_mmap --concat" NITERS=250 NELEMS=268435456 TEST_PATTERN="test_seek_then_read_block" - TEST_SUITE=indexed_gzip_test EXTRA_ARGS="--use_mmap" NITERS=5000 NELEMS=268435456 - TEST_SUITE=indexed_gzip_test EXTRA_ARGS="--use_mmap --concat" NITERS=5000 NELEMS=268435456 # NELEMS=805306368 ~ 6GB (for testing 64 bit support) - TEST_SUITE=zran_test EXTRA_ARGS="--use_mmap" NITERS=1000 NELEMS=805306368 TEST_PATTERN="not test_readbuf_spacing_sizes and not test_seek_then_read_block" - TEST_SUITE=zran_test EXTRA_ARGS="--use_mmap --concat" NITERS=1000 NELEMS=805306368 TEST_PATTERN="not test_readbuf_spacing_sizes and not test_seek_then_read_block" - TEST_SUITE=zran_test EXTRA_ARGS="--use_mmap" NITERS=50 NELEMS=805306368 TEST_PATTERN="test_readbuf_spacing_sizes" - TEST_SUITE=zran_test EXTRA_ARGS="--use_mmap --concat" NITERS=50 NELEMS=805306368 TEST_PATTERN="test_readbuf_spacing_sizes" - TEST_SUITE=zran_test EXTRA_ARGS="--use_mmap" NITERS=25 NELEMS=805306368 TEST_PATTERN="test_seek_then_read_block" - TEST_SUITE=zran_test EXTRA_ARGS="--use_mmap --concat" NITERS=25 NELEMS=805306368 TEST_PATTERN="test_seek_then_read_block" - TEST_SUITE=indexed_gzip_test EXTRA_ARGS="--use_mmap" NITERS=1000 NELEMS=805306368 - TEST_SUITE=indexed_gzip_test EXTRA_ARGS="--use_mmap --concat" NITERS=1000 NELEMS=805306368 - TEST_SUITE=32bittest install: "pip install cython numpy pytest" script: - .ci/run_tests.sh indexed-gzip-0.6.1/LICENSE000066400000000000000000000016511317257227400151270ustar00rootroot00000000000000indexed_gzip: Copyright (C) 2016-2017 Paul McCarthy zlib: Copyright (C) 1995-2013 Jean-loup Gailly and Mark Adler This software is provided 'as-is', without any express or implied warranty. In no event will the authors be held liable for any damages arising from the use of this software. Permission is granted to anyone to use this software for any purpose, including commercial applications, and to alter it and redistribute it freely, subject to the following restrictions: 1. The origin of this software must not be misrepresented; you must not claim that you wrote the original software. If you use this software in a product, an acknowledgment in the product documentation would be appreciated but is not required. 2. Altered source versions must be plainly marked as such, and must not be misrepresented as being the original software. 3. This notice may not be removed or altered from any source distribution. indexed-gzip-0.6.1/MANIFEST.in000066400000000000000000000004321317257227400156540ustar00rootroot00000000000000recursive-include indexed_gzip *.py recursive-include indexed_gzip *.pyx recursive-include indexed_gzip *.pxd recursive-include indexed_gzip *.c recursive-include indexed_gzip *.h exclude bench_plot.py exclude benchmark_indexed_gzip.py indexed-gzip-0.6.1/README.md000066400000000000000000000131771317257227400154070ustar00rootroot00000000000000# indexed_gzip [![Build Status](https://travis-ci.org/pauldmccarthy/indexed_gzip.svg?branch=master)](https://travis-ci.org/pauldmccarthy/indexed_gzip) *Fast random access of gzip files in Python* * [Overview](#overview) * [Installation](#installation) * [Usage](#usage) * [Performance](#performance) * [Acknowledgements](#acknowledgements) * [License](#license) ## Overview The `indexed_gzip` project is a Python extension which aims to provide a drop-in replacement for the built-in Python `gzip.GzipFile` class, the `IndexedGzipFile`. `indexed_gzip` was written to allow fast random access of compressed [NIFTI](http://nifti.nimh.nih.gov/) image files (for which GZIP is the de-facto compression standard), but will work with any GZIP file. `indexed_gzip` is easy to use with `nibabel` (http://nipy.org/nibabel/). The standard `gzip.GzipFile` class exposes a random access-like interface (via its `seek` and `read` methods), but every time you seek to a new point in the uncompressed data stream, the `GzipFile` instance has to start decompressing from the beginning of the file, until it reaches the requested location. An `IndexedGzipFile` instance gets around this performance limitation by building an index, which contains *seek points*, mappings between corresponding locations in the compressed and uncompressed data streams. Each seek point is accompanied by a chunk (32KB) of uncompressed data which is used to initialise the decompression algorithm, allowing us to start reading from any seek point. If the index is built with a seek point spacing of 1MB, we only have to decompress (on average) 512KB of data to read from any location in the file. ## Installation `indexed_gzip` is available on [PyPi](https://pypi.python.org/pypi) - to install, simply type: ```sh pip install indexed_gzip ``` To compile `indexed_gzip`, make sure you have [cython](http://cython.org/) installed, and then run: ```sh python setup.py build_ext --inplace ``` To run the tests, type the following; you will need `numpy` and `pytest` installed: ```sh python setup.py test ``` ## Usage You can use the `indexed_gzip` module directly: ```python import indexed_gzip as igzip # You can create an IndexedGzipFile instance # by specifying a file name, or an open file # handle. For the latter use, the file handle # must be opened in read-only binary mode. # Write support is currently non-existent. myfile = igzip.IndexedGzipFile(filename='big_file.gz') some_offset_into_uncompressed_data = 234195 # The index will be automatically # built on-demand when seeking or # reading. myfile.seek(some_offset_into_uncompressed_data) data = myfile.read(1048576) ``` Or you can use `indexed_gzip` with `nibabel`: ```python import nibabel as nib import indexed_gzip as igzip # Here we are usin 4MB spacing between # seek points, and using a larger read # buffer (than the default size of 16KB). fobj = igzip.IndexedGzipFile( filename='big_image.nii.gz', spacing=4194304, readbuf_size=131072) # Create a nibabel image using # the existing file handle. fmap = nib.Nifti1Image.make_file_map() fmap['image'].fileobj = fobj image = nib.Nifti1Image.from_file_map(fmap) # Use the image ArrayProxy to access the # data - the index will automatically be # built as data is accessed. vol3 = image.dataobj[:, :, :, 3] ``` `indexed_gzip` does not currently have any support for writing. Currently if you wish to write to a file, you will need to save the file by alternate means (e.g. via `gzip` or `nibabel`), and then re-create a new `IndexedGzipFile` instance. Building on the `nibabel` example above: ```python # Load the entire image into memory data = image.get_data() # Make changes to the data data[:, :, :, 5] *= 100 # Save the image using nibabel nib.save(data, 'big_image.nii.gz') # Re-create an IndexedGzipFile and # Nifti1Image instance as above fobj = igzip.IndexedGzipFile(...) fmap = nib.Nifti1Image.make_file_map() fmap['image'].fileobj = fobj image = nib.Nifti1Image.from_file_map(fmap) ``` ## Performance A small [test script](benchmark_indexed_gzip.py) is included with `indexed_gzip`; this script compares the performance of the `IndexedGzipFile` class with the `gzip.GzipFile` class. This script does the following: 1. Generates a specified number of seek locations, uniformly spaced throughout the input file. 2. Randomly shuffles these locations 3. Seeks to each location, and reads a chunk of data from the file. This plot shows the results of this test for a few compresed files of varying sizes, with 1000 seeks: ![Indexed gzip performance](./performance.png) ## Acknowledgements The `indexed_gzip` project is based upon the `zran.c` example (written by Mark Alder) which ships with the [zlib](http://www.zlib.net/) source code. `indexed_gzip` was originally inspired by Zalan Rajna's (@zrajna) [zindex](https://github.com/zrajna/zindex) project: Z. Rajna, A. Keskinarkaus, V. Kiviniemi and T. Seppanen "Speeding up the file access of large compressed NIfTI neuroimaging data" Engineering in Medicine and Biology Society (EMBC), 2015 37th Annual International Conference of the IEEE, Milan, 2015, pp. 654-657. https://sourceforge.net/projects/libznzwithzindex/ Initial work on `indexed_gzip` took place at [Brainhack](http://www.brainhack.org/) Paris, at the Institut Pasteur, 24th-26th February 2016, with the support of the [FMRIB Centre](https://www.ndcn.ox.ac.uk/divisions/fmrib/), at the University of Oxford, UK. Many thanks to Martin Craig (@mcraig-ibme) for help in porting `indexed_gzip` to Windows. ## License `indexed_gzip` inherits the [zlib](http://www.zlib.net) license, available for perusal in the [LICENSE](LICENSE) file. indexed-gzip-0.6.1/benchmark_indexed_gzip.py000077500000000000000000000107721317257227400211660ustar00rootroot00000000000000#!/usr/bin/env python # # benchmark_indexed_gzip.py - # # Author: Paul McCarthy # from __future__ import print_function import os.path as op import sys import time import hashlib import random import gzip import itertools as it import indexed_gzip as igzip def create_gz(fid): return gzip.GzipFile(fileobj=fid) def create_igz(fid): return igzip.IndexedGzipFile(fid=fid) def md5(data): hashobj = hashlib.md5() hashobj.update(data) return str(hashobj.hexdigest()) def testfile(name, fid, createfunc, length, offsets, loud): title = 'Testing {}'.format(name) print('\n\n{}\n{}\n\n'.format(title, '=' * len(title))) fid.seek(0) if loud: print(' creating file object...') createstart = time.time() fileobj = createfunc(fid) createend = time.time() seektimes = [] readtimes = [] hashtimes = [] hashes = [] for i, offset in enumerate(offsets): if loud: print('{: 3d}: seeking to {}...'.format(i, offset)) else: print('\r{: 3d}: reading {:0.2f}MB from location {}...'.format( i, length / 1048576, offset), end='') seekstart = time.time() fileobj.seek(offset) seekend = time.time() if loud: print(' reading {:0.2f}MB...'.format(length / 1048576), end='') readstart = time.time() data = fileobj.read(length) readend = time.time() hashstart = time.time() datahash = md5(data) hashend = time.time() readtimes.append(readend - readstart) seektimes.append(seekend - seekstart) hashtimes.append(hashend - hashstart) hashes .append(md5(data)) if loud: print('data hash: {}\n'.format(hashes[-1])) totalhashstart = time.time() totalhash = ''.join(it.chain(*hashes)).encode('ascii') datahash = md5(totalhash) totalhashend = time.time() createtime = createend - createstart seektime = sum(seektimes) readtime = sum(readtimes) hashtime = sum(hashtimes) + (totalhashend - totalhashstart) totaltime = createtime + seektime + readtime + hashtime print(' done!\n\n') subtitle = '{} summary'.format(name) print('{}\n{}\n'.format(subtitle, '-' * len(subtitle))) print('Total time: {:0.2f} sec' .format(totaltime)) print('Create time: {:0.2f} sec' .format(createtime)) print('Seek time: {:0.2f} sec' .format(seektime)) print('Read time: {:0.2f} sec' .format(readtime)) print('MD5 time: {:0.2f} sec' .format(hashtime)) print('Data MD5: {}' .format(datahash)) print() print('Average times over {} seeks/reads:'.format(numseeks)) print(' seek time: {:0.2f} sec' .format(seektime / numseeks)) print(' read time: {:0.2f} sec' .format(readtime / numseeks)) return totaltime, datahash if len(sys.argv) not in (2, 3, 4, 5): print('usage: test_indexed_gzip.py filename [numseeks [loud [seed]]]') sys.exit(1) infile = sys.argv[1] numseeks = 10 loud = False seed = random.randint(1, 2**32) if len(sys.argv) >= 3: numseeks = int( sys.argv[2]) if len(sys.argv) >= 4: loud = bool(sys.argv[3]) if len(sys.argv) >= 5: seed = int( sys.argv[4]) print('Random seed: {}'.format(seed)) random.seed(seed) with open(infile, 'rb') as fid: fid.seek(-1, 2) cmp_size = fid.tell() maxoff = int(cmp_size * 1.5) offstep = int(maxoff / numseeks) offsets = list(range(0, maxoff, offstep)) length = int(0.75 * cmp_size / len(offsets)) numseeks = len(offsets) random.shuffle(offsets) try: gztime, gzhash = testfile("gzip.GzipFile", fid, create_gz, length, offsets, loud) igztime, igzhash = testfile("igzip.IndexedGzipFile", fid, create_igz, length, offsets, loud) if gzhash == igzhash: result = 'PASS' else: result = 'FAIL' print('\n{:5d} {:<20s} {:15d} [gz: {:0.2f} seconds] [igz: {:0.2f} seconds] {}'.format( numseeks, op.basename(infile), seed, gztime, igztime, result)) except: print('\n{:5d} {:<20s} {:15d} CRASH'.format( numseeks, op.basename(infile), seed)) raise indexed-gzip-0.6.1/compat/000077500000000000000000000000001317257227400154025ustar00rootroot00000000000000indexed-gzip-0.6.1/compat/stdint.h000066400000000000000000000170631317257227400170670ustar00rootroot00000000000000// ISO C9x compliant stdint.h for Microsoft Visual Studio // Based on ISO/IEC 9899:TC2 Committee draft (May 6, 2005) WG14/N1124 // // Copyright (c) 2006-2008 Alexander Chemeris // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are met: // // 1. Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimer. // // 2. Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // // 3. The name of the author may be used to endorse or promote products // derived from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED // WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF // MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO // EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; // OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, // WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR // OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF // ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // /////////////////////////////////////////////////////////////////////////////// #ifndef _MSC_VER // [ #error "Use this header only with Microsoft Visual C++ compilers!" #endif // _MSC_VER ] #ifndef _MSC_STDINT_H_ // [ #define _MSC_STDINT_H_ #if _MSC_VER > 1000 #pragma once #endif #include // For Visual Studio 6 in C++ mode and for many Visual Studio versions when // compiling for ARM we should wrap include with 'extern "C++" {}' // or compiler give many errors like this: // error C2733: second C linkage of overloaded function 'wmemchr' not allowed #ifdef __cplusplus extern "C" { #endif # include #ifdef __cplusplus } #endif // Define _W64 macros to mark types changing their size, like intptr_t. #ifndef _W64 # if !defined(__midl) && (defined(_X86_) || defined(_M_IX86)) && _MSC_VER >= 1300 # define _W64 __w64 # else # define _W64 # endif #endif // 7.18.1 Integer types // 7.18.1.1 Exact-width integer types // Visual Studio 6 and Embedded Visual C++ 4 doesn't // realize that, e.g. char has the same size as __int8 // so we give up on __intX for them. #if (_MSC_VER < 1300) typedef signed char int8_t; typedef signed short int16_t; typedef signed int int32_t; typedef unsigned char uint8_t; typedef unsigned short uint16_t; typedef unsigned int uint32_t; #else typedef signed __int8 int8_t; typedef signed __int16 int16_t; typedef signed __int32 int32_t; typedef unsigned __int8 uint8_t; typedef unsigned __int16 uint16_t; typedef unsigned __int32 uint32_t; #endif typedef signed __int64 int64_t; typedef unsigned __int64 uint64_t; // 7.18.1.2 Minimum-width integer types typedef int8_t int_least8_t; typedef int16_t int_least16_t; typedef int32_t int_least32_t; typedef int64_t int_least64_t; typedef uint8_t uint_least8_t; typedef uint16_t uint_least16_t; typedef uint32_t uint_least32_t; typedef uint64_t uint_least64_t; // 7.18.1.3 Fastest minimum-width integer types typedef int8_t int_fast8_t; typedef int16_t int_fast16_t; typedef int32_t int_fast32_t; typedef int64_t int_fast64_t; typedef uint8_t uint_fast8_t; typedef uint16_t uint_fast16_t; typedef uint32_t uint_fast32_t; typedef uint64_t uint_fast64_t; // 7.18.1.4 Integer types capable of holding object pointers #ifdef _WIN64 // [ typedef signed __int64 intptr_t; typedef unsigned __int64 uintptr_t; #else // _WIN64 ][ typedef _W64 signed int intptr_t; typedef _W64 unsigned int uintptr_t; #endif // _WIN64 ] // 7.18.1.5 Greatest-width integer types typedef int64_t intmax_t; typedef uint64_t uintmax_t; // 7.18.2 Limits of specified-width integer types #if !defined(__cplusplus) || defined(__STDC_LIMIT_MACROS) // [ See footnote 220 at page 257 and footnote 221 at page 259 // 7.18.2.1 Limits of exact-width integer types #define INT8_MIN ((int8_t)_I8_MIN) #define INT8_MAX _I8_MAX #define INT16_MIN ((int16_t)_I16_MIN) #define INT16_MAX _I16_MAX #define INT32_MIN ((int32_t)_I32_MIN) #define INT32_MAX _I32_MAX #define INT64_MIN ((int64_t)_I64_MIN) #define INT64_MAX _I64_MAX #define UINT8_MAX _UI8_MAX #define UINT16_MAX _UI16_MAX #define UINT32_MAX _UI32_MAX #define UINT64_MAX _UI64_MAX // 7.18.2.2 Limits of minimum-width integer types #define INT_LEAST8_MIN INT8_MIN #define INT_LEAST8_MAX INT8_MAX #define INT_LEAST16_MIN INT16_MIN #define INT_LEAST16_MAX INT16_MAX #define INT_LEAST32_MIN INT32_MIN #define INT_LEAST32_MAX INT32_MAX #define INT_LEAST64_MIN INT64_MIN #define INT_LEAST64_MAX INT64_MAX #define UINT_LEAST8_MAX UINT8_MAX #define UINT_LEAST16_MAX UINT16_MAX #define UINT_LEAST32_MAX UINT32_MAX #define UINT_LEAST64_MAX UINT64_MAX // 7.18.2.3 Limits of fastest minimum-width integer types #define INT_FAST8_MIN INT8_MIN #define INT_FAST8_MAX INT8_MAX #define INT_FAST16_MIN INT16_MIN #define INT_FAST16_MAX INT16_MAX #define INT_FAST32_MIN INT32_MIN #define INT_FAST32_MAX INT32_MAX #define INT_FAST64_MIN INT64_MIN #define INT_FAST64_MAX INT64_MAX #define UINT_FAST8_MAX UINT8_MAX #define UINT_FAST16_MAX UINT16_MAX #define UINT_FAST32_MAX UINT32_MAX #define UINT_FAST64_MAX UINT64_MAX // 7.18.2.4 Limits of integer types capable of holding object pointers #ifdef _WIN64 // [ # define INTPTR_MIN INT64_MIN # define INTPTR_MAX INT64_MAX # define UINTPTR_MAX UINT64_MAX #else // _WIN64 ][ # define INTPTR_MIN INT32_MIN # define INTPTR_MAX INT32_MAX # define UINTPTR_MAX UINT32_MAX #endif // _WIN64 ] // 7.18.2.5 Limits of greatest-width integer types #define INTMAX_MIN INT64_MIN #define INTMAX_MAX INT64_MAX #define UINTMAX_MAX UINT64_MAX // 7.18.3 Limits of other integer types #ifdef _WIN64 // [ # define PTRDIFF_MIN _I64_MIN # define PTRDIFF_MAX _I64_MAX #else // _WIN64 ][ # define PTRDIFF_MIN _I32_MIN # define PTRDIFF_MAX _I32_MAX #endif // _WIN64 ] #define SIG_ATOMIC_MIN INT_MIN #define SIG_ATOMIC_MAX INT_MAX #ifndef SIZE_MAX // [ # ifdef _WIN64 // [ # define SIZE_MAX _UI64_MAX # else // _WIN64 ][ # define SIZE_MAX _UI32_MAX # endif // _WIN64 ] #endif // SIZE_MAX ] // WCHAR_MIN and WCHAR_MAX are also defined in #ifndef WCHAR_MIN // [ # define WCHAR_MIN 0 #endif // WCHAR_MIN ] #ifndef WCHAR_MAX // [ # define WCHAR_MAX _UI16_MAX #endif // WCHAR_MAX ] #define WINT_MIN 0 #define WINT_MAX _UI16_MAX #endif // __STDC_LIMIT_MACROS ] // 7.18.4 Limits of other integer types #if !defined(__cplusplus) || defined(__STDC_CONSTANT_MACROS) // [ See footnote 224 at page 260 // 7.18.4.1 Macros for minimum-width integer constants #define INT8_C(val) val##i8 #define INT16_C(val) val##i16 #define INT32_C(val) val##i32 #define INT64_C(val) val##i64 #define UINT8_C(val) val##ui8 #define UINT16_C(val) val##ui16 #define UINT32_C(val) val##ui32 #define UINT64_C(val) val##ui64 // 7.18.4.2 Macros for greatest-width integer constants #define INTMAX_C INT64_C #define UINTMAX_C UINT64_C #endif // __STDC_CONSTANT_MACROS ] #endif // _MSC_STDINT_H_ ] indexed-gzip-0.6.1/conftest.py000066400000000000000000000044071317257227400163230ustar00rootroot00000000000000#!/usr/bin/env python # # conftest.py - # # Author: Paul McCarthy # import os import os.path as op import numpy as np import pytest from indexed_gzip.tests import gen_test_data def pytest_addoption(parser): parser.addoption('--nelems', type=int, action='store', default=2**24 + 1, help='Number of uint64 elements for test data') parser.addoption('--concat', action='store_true', help='Generate test data made of ' 'concatenated GZIP streams') parser.addoption('--use_mmap', action='store_true', help='Use mmap for read buffer instead of main memory') parser.addoption('--seed', type=int, help='Seed for random number generator') parser.addoption('--testfile', action='store', help='Name of test file') parser.addoption('--niters', type=int, action='store', default=5000, help='Number of inputs for tests which ' 'use a random set of inputs') @pytest.fixture def nelems(request): return request.config.getoption('--nelems') @pytest.fixture def niters(request): return request.config.getoption('--niters') @pytest.fixture def concat(request): return request.config.getoption('--concat') @pytest.fixture def use_mmap(request): return request.config.getoption('--use_mmap') @pytest.fixture def seed(request): seed = request.config.getoption('--seed') if seed is None: seed = np.random.randint(2 ** 30) np.random.seed(seed) print('Seed for random number generator: {}'.format(seed)) return seed @pytest.fixture def testfile(request): filename = request.config.getoption('--testfile') _nelems = nelems(request) _concat = concat(request) if filename is None: filename = op.join(os.getcwd(), 'ctest_zran_{}_{}.gz'.format(_nelems, _concat)) if not op.exists(filename): gen_test_data(filename, _nelems, _concat) return filename indexed-gzip-0.6.1/debian/000077500000000000000000000000001317257227400153415ustar00rootroot00000000000000indexed-gzip-0.6.1/debian/changelog000066400000000000000000000002321317257227400172100ustar00rootroot00000000000000indexed-gzip (0.3.1-1) unstable; urgency=low * Initial release (Closes: #835493). -- Michael Hanke Wed, 05 Apr 2017 13:09:20 +0200 indexed-gzip-0.6.1/debian/compat000066400000000000000000000000021317257227400165370ustar00rootroot000000000000009 indexed-gzip-0.6.1/debian/control000066400000000000000000000054151317257227400167510ustar00rootroot00000000000000Source: indexed-gzip Maintainer: NeuroDebian Team Uploaders: Michael Hanke , Alex Waite Section: python Testsuite: autopkgtest Priority: optional Build-Depends: debhelper (>= 9), zlib1g-dev, cython, cython3, dh-python, python-all-dev, python3-all-dev, python-pytest, python3-pytest, python-pytest-runner, python3-pytest-runner, python-setuptools, python3-setuptools, python-numpy, python3-numpy, Standards-Version: 3.9.8 Homepage: https://github.com/pauldmccarthy/indexed_gzip X-Python-Version: >= 2.7 Package: python-indexed-gzip Architecture: any Section: python Depends: cython, python-numpy, ${misc:Depends}, ${shlibs:Depends}, ${python:Depends}, Provides: ${python:Provides} Description: fast random access of gzip files in Python Drop-in replacement `IndexedGzipFile` for the built-in Python `gzip.GzipFile` class that does not need to start decompressing from the beginning of the file when for every `seek()`. It gets around this performance limitation by building an index, which contains *seek points*, mappings between corresponding locations in the compressed and uncompressed data streams. Each seek point is accompanied by a chunk (32KB) of uncompressed data which is used to initialise the decompression algorithm, allowing to start reading from any seek point. If the index is built with a seek point spacing of 1MB, only 512KB (on average) of data have to be decompressed to read from any location in the file. . This package provides the Python 2 module. Package: python3-indexed-gzip Architecture: any Section: python Depends: cython3, python3-numpy, ${misc:Depends}, ${shlibs:Depends}, ${python3:Depends}, Provides: ${python3:Provides} Description: fast random access of gzip files in Python Drop-in replacement `IndexedGzipFile` for the built-in Python `gzip.GzipFile` class that does not need to start decompressing from the beginning of the file when for every `seek()`. It gets around this performance limitation by building an index, which contains *seek points*, mappings between corresponding locations in the compressed and uncompressed data streams. Each seek point is accompanied by a chunk (32KB) of uncompressed data which is used to initialise the decompression algorithm, allowing to start reading from any seek point. If the index is built with a seek point spacing of 1MB, only 512KB (on average) of data have to be decompressed to read from any location in the file. . This package provides the Python 3 module. indexed-gzip-0.6.1/debian/copyright000066400000000000000000000024141317257227400172750ustar00rootroot00000000000000Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ Upstream-Name: indexed_gzip Source: https://github.com/pauldmccarthy/indexed_gzip Files: * Copyright: 2016 Paul McCarthy , 1995-2013 Jean-loup Gailly and Mark Adler License: zLIB Files: debian/* Copyright: 2016-2017 Michael Hanke , Alex Waite License: zLIB License: zLIB This software is provided 'as-is', without any express or implied warranty. In no event will the authors be held liable for any damages arising from the use of this software. . Permission is granted to anyone to use this software for any purpose, including commercial applications, and to alter it and redistribute it freely, subject to the following restrictions: . 1. The origin of this software must not be misrepresented; you must not claim that you wrote the original software. If you use this software in a product, an acknowledgment in the product documentation would be appreciated but is not required. . 2. Altered source versions must be plainly marked as such, and must not be misrepresented as being the original software. . 3. This notice may not be removed or altered from any source distribution. indexed-gzip-0.6.1/debian/patches/000077500000000000000000000000001317257227400167705ustar00rootroot00000000000000indexed-gzip-0.6.1/debian/patches/jessie-dsc-patch000077700000000000000000000000001317257227400251032xenial-dsc-patchustar00rootroot00000000000000indexed-gzip-0.6.1/debian/patches/no_pytest000066400000000000000000000010011317257227400207270ustar00rootroot00000000000000diff --git a/setup.cfg b/setup.cfg index 9af7e6f..50a6c9c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,2 @@ [aliases] -test=pytest \ No newline at end of file +#test=pytest diff --git a/setup.py b/setup.py index 9adae76..fed4e27 100644 --- a/setup.py +++ b/setup.py @@ -127,7 +127,7 @@ setup( ext_modules=extensions, - setup_requires=['pytest-runner'], - tests_require=['pytest', 'numpy'], + #setup_requires=['pytest-runner'], + tests_require=['numpy'], test_suite='tests', ) indexed-gzip-0.6.1/debian/patches/series-jessie000077700000000000000000000000001317257227400241532series-xenialustar00rootroot00000000000000indexed-gzip-0.6.1/debian/patches/series-xenial000066400000000000000000000000121317257227400214540ustar00rootroot00000000000000no_pytest indexed-gzip-0.6.1/debian/patches/xenial-dsc-patch000066400000000000000000000026301317257227400220400ustar00rootroot00000000000000diff --git a/debian/control b/debian/control index 71b50d1..536afbf 100644 --- a/debian/control +++ b/debian/control @@ -12,10 +12,6 @@ Build-Depends: debhelper (>= 9), dh-python, python-all-dev, python3-all-dev, - python-pytest, - python3-pytest, - python-pytest-runner, - python3-pytest-runner, python-setuptools, python3-setuptools, python-numpy, diff --git a/debian/rules b/debian/rules index 9d31ed3..cd3adc1 100755 --- a/debian/rules +++ b/debian/rules @@ -12,7 +12,7 @@ gitver = $(shell [ -x /usr/bin/git ] && git describe --tags --match 'v[0-9].*' $ export DH_VERBOSE = 1 export PYBUILD_NAME = indexed-gzip - +export PYBUILD_DISABLE = test/python2 test/python3 export PYBUILD_BEFORE_TEST=cp -vr {dir}/tests {build_dir} # one ring to rule them all ... diff --git a/setup.cfg b/setup.cfg index 9af7e6f..50a6c9c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,2 @@ [aliases] -test=pytest \ No newline at end of file +#test=pytest diff --git a/setup.py b/setup.py index 9adae76..fed4e27 100644 --- a/setup.py +++ b/setup.py @@ -127,7 +127,7 @@ setup( ext_modules=extensions, - setup_requires=['pytest-runner'], - tests_require=['pytest', 'numpy'], + #setup_requires=['pytest-runner'], + tests_require=['numpy'], test_suite='tests', ) indexed-gzip-0.6.1/debian/rules000077500000000000000000000030571317257227400164260ustar00rootroot00000000000000#!/usr/bin/make -f srcpkg = $(shell LC_ALL=C dpkg-parsechangelog | grep '^Source:' | cut -d ' ' -f 2,2) debver = $(shell LC_ALL=C dpkg-parsechangelog | grep '^Version:' | cut -d ' ' -f 2,2 ) upstreamver = $(shell echo $(debver) | cut -d '-' -f 1,1 ) # this figures out the last merge point from 'master' into the Debian branch and # then described this commit relative to the last release tag (V...) # If this should make any sense the local upstream branch must track upstream's # master or whatever other source branch. gitver = $(shell [ -x /usr/bin/git ] && git describe --tags --match 'v[0-9].*' $$(git merge-base -a HEAD upstream) | sed -e 's/^v//' -e 's/-/+git/') export DH_VERBOSE = 1 export PYBUILD_NAME = indexed-gzip export PYBUILD_BEFORE_TEST=cp -vr {dir}/tests {build_dir} export PYBUILD_TEST_ARGS = --niter 1 # one ring to rule them all ... %: dh $@ --with python2,python3 --buildsystem=pybuild clean:: dh_clean # hand clean generated C code -rm indexed_gzip.c tests/*.c # Ahhhhhhh! rm -rf .pybuild build -rm -rf indexed_gzip.egg-info/ override_dh_install: dh_install # tests leave a bit of trash behind -find debian/ -name 'ctest_zran*.gz' -delete # make orig tarball from repository content get-orig-source: # orig tarball, turn directory into something nicer git archive --format=tar --prefix=$(srcpkg)-$(gitver)/ HEAD | \ gzip -9 > $(srcpkg)_$(gitver).orig.tar.gz # check that DSC patches still apply maint-check-dsc-patches: @for p in debian/patches/*-dsc-patch; \ do echo "check $$p"; \ patch -p1 --dry-run < $$p || exit 1 ; \ done indexed-gzip-0.6.1/debian/source/000077500000000000000000000000001317257227400166415ustar00rootroot00000000000000indexed-gzip-0.6.1/debian/source/format000066400000000000000000000000141317257227400200470ustar00rootroot000000000000003.0 (quilt) indexed-gzip-0.6.1/debian/tests/000077500000000000000000000000001317257227400165035ustar00rootroot00000000000000indexed-gzip-0.6.1/debian/tests/control000066400000000000000000000001701317257227400201040ustar00rootroot00000000000000Test-Command: python setup.py test Depends: @, @builddeps@ Test-Command: python3 setup.py test Depends: @, @builddeps@ indexed-gzip-0.6.1/debian/watch000066400000000000000000000002441317257227400163720ustar00rootroot00000000000000version=3 opts="filenamemangle=s/.*\/v(\d)\.(\d).*/indexed_gzip-$1.$2\.tar\.gz/" \ http://github.com/pauldmccarthy/indexed_gzip/tags .*archive/v(\d[\d.]+).tar.gz indexed-gzip-0.6.1/indexed_gzip/000077500000000000000000000000001317257227400165705ustar00rootroot00000000000000indexed-gzip-0.6.1/indexed_gzip/__init__.py000066400000000000000000000005451317257227400207050ustar00rootroot00000000000000#!/usr/bin/env python # # __init__.py - The indexed_gzip namespace. # # Author: Paul McCarthy # """The indexed_gzip namespace. """ from .indexed_gzip import (IndexedGzipFile, SafeIndexedGzipFile, NotCoveredError, ZranError) __version__ = '0.6.1' indexed-gzip-0.6.1/indexed_gzip/indexed_gzip.pyx000066400000000000000000000452611317257227400220130ustar00rootroot00000000000000# # The IndexedGzipFile class. # # Author: Paul McCarthy # """This module provides the :class:`IndexedGzipFile` class, a drop-in replacement for the built-in ``gzip.GzipFile`` class, for faster read-only random access to gzip files. """ from libc.stdio cimport (SEEK_SET, SEEK_CUR, FILE, fdopen, fclose) from libc.stdint cimport (uint8_t, uint64_t, int64_t) from cpython.mem cimport (PyMem_Malloc, PyMem_Realloc, PyMem_Free) from cpython.buffer cimport (PyObject_GetBuffer, PyBuffer_Release, PyBUF_ANY_CONTIGUOUS, PyBUF_SIMPLE) cimport indexed_gzip.zran as zran import io import threading import logging log = logging.getLogger(__name__) log.setLevel(logging.WARNING) class NotCoveredError(Exception): """Exception raised by the :class:`IndexedGzipFile` when an attempt is made to seek to/read from a location that is not covered by the index. This exception will never be raised if the ``IndexedGzipFile`` was created with ``auto_build=True``. """ pass class ZranError(Exception): """Exception raised by the :class:`IndexedGzipFile` when the ``zran`` library signals an error. """ pass cdef class IndexedGzipFile: """The ``IndexedGzipFile`` class allows for fast random access of a gzip file by using the ``zran`` library to build and maintain an index of seek points into the file. .. note:: The :meth:`seek` and :meth:`read` methods release the GIL while calling ``zran`` functions, but the ``IndexedGzipFile`` is *not* thread-safe. Use the ``SafeIndexedGzipFile`` class if you need thread-safety. """ cdef zran.zran_index_t index """A reference to the ``zran_index`` struct. """ cdef bint auto_build """Flag which is set to ``True`` if the file index is built automatically on seeks/reads. """ cdef unsigned int readall_buf_size """Used by meth:`read` as the read buffer size.""" cdef bint own_file """Flag which is set to ``True`` if the user specified a file name instead of an open file handle. In this case, the IndexedGzipFile is responsible for closing the file handle when it is destroyed. """ cdef FILE *cfid """A reference to the C file handle. """ cdef object pyfid """A reference to the python file handle. """ def __cinit__(self, filename=None, fid=None, mode=None, auto_build=True, spacing=4194304, window_size=32768, readbuf_size=1048576, readall_buf_size=16777216): """Create an ``IndexedGzipFile``. The file may be specified either with an open file handle, or with a filename. If the former, the file must have been opened in ``'rb'`` mode. :arg filename: File name. :arg mode: Opening mode. Must be either ``'r'`` or ``'rb``. :arg fid: Open file handle. :arg auto_build: If ``True`` (the default), the index is automatically built on seeks/reads. :arg spacing: Number of bytes between index seek points. :arg window_size: Number of bytes of uncompressed data stored with each seek point. :arg readbuf_size: Size of buffer in bytes for storing compressed data read in from the file. :arg readall_buf_size: Size of buffer in bytes used by :meth:`read` when reading until EOF. """ if fid is None and filename is None: raise ValueError('One of fid or filename must be specified') if fid is not None and fid.mode != 'rb': raise ValueError('The gzip file must be opened in ' 'read-only binary ("rb") mode') if (fid is None) and (mode not in (None, 'r', 'rb')): raise ValueError('Invalid mode ({}), must be ' '\'r\' or \'rb\''.format(mode)) mode = 'rb' self.own_file = fid is None self.auto_build = auto_build self.readall_buf_size = readall_buf_size if self.own_file: self.pyfid = open(filename, 'rb') else: self.pyfid = fid self.cfid = fdopen(self.pyfid.fileno(), 'rb') if self.auto_build: flags = zran.ZRAN_AUTO_BUILD else: flags = 0 if zran.zran_init(index=&self.index, fd=self.cfid, spacing=spacing, window_size=window_size, readbuf_size=readbuf_size, flags=flags): raise ZranError('zran_init returned error') log.debug('{}.__init__({}, {}, {}, {}, {}, {})'.format( type(self).__name__, fid, filename, auto_build, spacing, window_size, readbuf_size)) def __init__(self, *args, **kwargs): """This method does nothing. It is here to make sub-classing ``IndexedGzipFile`` easier. """ pass def fileno(self): """Calls ``fileno`` on the underlying file object. """ return self.pyfid.fileno() def fileobj(self): """Returns a reference to the python file object. """ return self.pyfid @property def mode(self): """Returns the mode that this file was opened in. Currently always returns ``'rb'``. """ return 'rb' def close(self): """Closes this ``IndexedGzipFile``. """ if self.closed: raise IOError('IndexedGzipFile is already closed') zran.zran_free(&self.index) if self.own_file: self.pyfid.close() self.cfid = NULL self.pyfid = None log.debug('{}.close()'.format(type(self).__name__)) @property def closed(self): """Returns ``True`` if this ``IndexedGzipFile`` is closed, ``False`` otherwise. """ return self.pyfid is None def readable(self): """Returns ``True`` if this ``IndexedGzipFile`` is readable, ``False`` otherwise. """ return not self.closed def writable(self): """Currently always returns ``False`` - the ``IndexedGzipFile`` does not support writing yet. """ return False def seekable(self): """Returns ``True`` if this ``IndexedGzipFile`` supports seeking, ``False`` otherwise. """ return not self.closed def tell(self): """Returns the current seek offset into the uncompressed data stream. """ return zran.zran_tell(&self.index) def __enter__(self): """Returns this ``IndexedGzipFile``. """ return self def __exit__(self, *args): """Calls close on this ``IndexedGzipFile``. """ if not self.closed: self.close() def __dealloc__(self): """Frees the memory used by this ``IndexedGzipFile``. If a file name was passed to :meth:`__cinit__`, the file handle is closed. """ if not self.closed: self.close() def build_full_index(self): """Re-builds the full file index. """ if zran.zran_build_index(&self.index, 0, 0) != 0: raise ZranError('zran_build_index returned error') log.debug('{}.build_fuill_index()'.format(type(self).__name__)) def seek(self, offset, whence=SEEK_SET): """Seeks to the specified position in the uncompressed data stream. If this ``IndexedGzipFile`` was created with ``auto_build=False``, and the requested offset is not covered by the index, a :exc:`NotCoveredError` is raised. :arg offset: Desired seek offset into the uncompressed data :arg whence: Either ``SEEK_SET`` or ``SEEK_CUR``. If not one of these, a :exc:`ValueError` is raised. :returns: The final seek location into the uncompressed stream. .. note:: This method releases the GIL while ``zran_seek`` is running. """ cdef int ret cdef uint64_t off = offset cdef uint8_t c_whence = whence cdef zran.zran_index_t *index = &self.index if whence not in (SEEK_SET, SEEK_CUR): raise ValueError('Seek from end not supported') with nogil: ret = zran.zran_seek(index, off, c_whence, NULL) if ret < 0: raise ZranError('zran_seek returned error: {}'.format(ret)) elif ret == zran.ZRAN_SEEK_NOT_COVERED: raise NotCoveredError('Index does not cover ' 'offset {}'.format(offset)) elif ret not in (zran.ZRAN_SEEK_OK, zran.ZRAN_SEEK_EOF): raise ZranError('zran_seek returned unknown code: {}'.format(ret)) offset = self.tell() log.debug('{}.seek({})'.format(type(self).__name__, offset)) return offset def read(self, nbytes=-1): """Reads up to ``nbytes`` bytes from the uncompressed data stream. If ``nbytes < 0`` the stream is read until EOF. If the stream is already at EOF, ``b''`` is returned. .. note:: This method releases the GIL while ``zran_read`` is running. """ if nbytes == 0: return bytes() elif nbytes < 0: buf = ReadBuffer(self.readall_buf_size) else: buf = ReadBuffer(nbytes) cdef zran.zran_index_t *index = &self.index cdef size_t nread = 0 cdef uint64_t bufsz = buf.size cdef size_t offset = 0 cdef void *buffer cdef int64_t ret # Read until EOF or enough # bytes have been read while True: buffer = buf.buffer + offset # read some bytes with nogil: ret = zran.zran_read(index, buffer, bufsz) # see how the read went if ret == zran.ZRAN_READ_FAIL: raise ZranError('zran_read returned error ({})'.format(ret)) # This will happen if the current # seek point is not covered by the # index, and auto-build is disabled elif ret == zran.ZRAN_READ_NOT_COVERED: raise NotCoveredError('Index does not cover current offset') # No bytes were read, and there are # no more bytes to read. This will # happen when the seek point was at # or beyond EOF when zran_read was # called elif ret == zran.ZRAN_READ_EOF: break nread += ret offset += ret # If we requested a specific number of # bytes, zran_read will have returned # them all (or all until EOF), so we're # finished if nbytes > 0: break # Otherwise if reading until EOF, check # and increase the buffer size if necessary if (nread + self.readall_buf_size) > buf.size: buf.resize(buf.size + self.readall_buf_size) offset = nread buf.resize(nread) pybuf = (buf.buffer)[:nread] log.debug('{}.read({})'.format(type(self).__name__, len(pybuf))) return pybuf def readinto(self, buf): """Reads up to ``len(buf)`` bytes directly into ``buf``, which is assumed to be a mutable ``bytes``-like object (e.g. a ``memoryview`` or ``bytearray``. """ cdef zran.zran_index_t *index = &self.index cdef uint64_t bufsz = len(buf) cdef Py_buffer pbuf cdef void *vbuf cdef int64_t ret # Create a Py_Buffer which allows # us to access the memory managed # by the provided buf PyObject_GetBuffer(buf, &pbuf, PyBUF_SIMPLE | PyBUF_ANY_CONTIGUOUS) # read soem bytes try: vbuf = pbuf.buf with nogil: ret = zran.zran_read(index, vbuf, bufsz) # release the py_buffer finally: PyBuffer_Release(&pbuf) # see how the read went if ret == zran.ZRAN_READ_FAIL: raise ZranError('zran_read returned error ({})'.format(ret)) # This will happen if the current # seek point is not covered by the # index, and auto-build is disabled elif ret == zran.ZRAN_READ_NOT_COVERED: raise NotCoveredError('Index does not cover current offset') # No bytes were read, and there are # no more bytes to read. This will # happen when the seek point was at # or beyond EOF when zran_read was # called elif ret == zran.ZRAN_READ_EOF: return 0 # Return the number of bytes that # were read else: return ret def pread(self, nbytes, offset): """Seeks to the specified ``offset``, then reads and returns ``nbytes``. See :meth:`seek` and :meth:`read`. """ IndexedGzipFile.seek(self, offset) return IndexedGzipFile.read(self, nbytes) def readline(self, size=-1): """Read and return up to the next ``'\n'`` character (up to at most ``size`` bytes, if ``size >= 0``) from the uncompressed data stream. If the end of the stream has been reached, ``b''`` is returned. """ if size == 0: return bytes() linebuf = b'' startpos = self.tell() bufsz = 1024 # Read in chunks of [bufsz] bytes at a time while True: buf = self.read(bufsz) lineidx = buf.find(b'\n') haveline = lineidx >= 0 eof = len(buf) == 0 # Are we at EOF? Nothing more to do if eof: break # Have we found a line? Discard # everything that comes after it if haveline: linebuf = linebuf + buf[:lineidx + 1] # If we've found a line, and are # not size-limiting, we're done if haveline and size < 0: break # If we're size limiting, and have # read in enough bytes, we're done if size >= 0 and len(linebuf) > size: linebuf = linebuf[:size] break # Rewind the seek location # to the finishing point self.seek(startpos + len(linebuf)) return linebuf def readlines(self, hint=-1): """Reads and returns a list of lines from the uncompressed data. If ``hint`` is provided, lines will be read until the total size of all lines exceeds ``hint`` in bytes. """ totalsize = 0 lines = [] while True: line = self.readline() if line == b'': break lines.append(line) totalsize += len(line) if hint >= 0 and totalsize > hint: break return lines def __iter__(self): """Returns this ``IndexedGzipFile`` which can be iterated over to return lines (separated by ``'\n'``) in the uncompressed stream. """ return self def __next__(self): """Returns the next line from the uncompressed stream. Raises :exc:`StopIteration` when there are no lines left. """ line = self.readline() if line == b'': raise StopIteration() else: return line def write(self, *args, **kwargs): """Currently raises a :exc:`NotImplementedError`.""" raise NotImplementedError('IndexedGzipFile does not support writing') def flush(self): """Currently does nothing. """ pass cdef class ReadBuffer: """Wrapper around a chunk of memory. .. see:: http://docs.cython.org/src/tutorial/memory_allocation.html """ cdef void *buffer """A raw chunk of bytes. """ cdef size_t size; """Size of the buffer. """ def __cinit__(self, size_t size): """Allocate ``size`` bytes of memory. """ self.size = size self.buffer = PyMem_Malloc(size) if not self.buffer: raise MemoryError('PyMem_Malloc fail') log.debug('ReadBuffer.__cinit__({})'.format(size)) def resize(self, size_t size): """Re-allocate the memory to the given ``size``. """ if size == self.size: return buf = PyMem_Realloc(self.buffer, size) if not buf: raise MemoryError('PyMem_Realloc fail') log.debug('ReadBuffer.resize({})'.format(size)) self.size = size self.buffer = buf def __dealloc__(self): """Free the mwmory. """ PyMem_Free(self.buffer) log.debug('ReadBuffer.__dealloc__()') class SafeIndexedGzipFile(io.BufferedReader): """The ``SafeIndexedGzipFile`` is an ``io.BufferedReader`` which wraps an :class:`IndexedGzipFile` instance. By accessing the ``IndexedGzipFile`` instance through an ``io.BufferedReader``, read performance is improved through buffering, and access to the I/O methods is made thread-safe. A :meth:`pread` method is also implemented, as it is not implemented by the ``io.BufferedReader``. """ def __init__(self, *args, **kwargs): """Opens an ``IndexedGzipFile``, and then calls ``io.BufferedReader.__init__``. :arg buffer_size: Optional, must be passed as a keyword argument. Passed through to ``io.BufferedReader.__init__``. If not provided, a default value of 1048576 is used. All other arguments are passed through to ``IndezedGzipFile.__init__``. """ buffer_size = kwargs.pop('buffer_size', 1048576) fobj = IndexedGzipFile(*args, **kwargs) self.__fileLock = threading.RLock() io.BufferedReader.__init__(self, fobj, buffer_size) def pread(self, nbytes, offset): """Seeks to ``offset``, then reads and returns up to ``nbytes``. The calls to seek and read are protected by a ``threading.RLock``. """ with self.__fileLock: self.seek(offset) return self.read(nbytes) indexed-gzip-0.6.1/indexed_gzip/tests/000077500000000000000000000000001317257227400177325ustar00rootroot00000000000000indexed-gzip-0.6.1/indexed_gzip/tests/__init__.py000066400000000000000000000121301317257227400220400ustar00rootroot00000000000000#!/usr/bin/env python # # __init__.py - # # Author: Paul McCarthy # import os import sys import time import gzip import shutil import tempfile import threading import subprocess as sp import multiprocessing as mp import numpy as np def testdir(): """Returnsa context manager which creates and returns a temporary directory, and then deletes it on exit. """ class ctx(object): def __enter__(self): self.testdir = tempfile.mkdtemp() return self.testdir def __exit__(self, *a, **kwa): shutil.rmtree(self.testdir) return ctx() def poll(until): """Waits until ``until`` returns ``True``, printing out a message every minute. """ start = time.time() while not until(): time.sleep(0.5) cur = time.time() elapsed = cur - start if int(elapsed) % 60 == 0: print('Waiting ({:0.2f} minutes)'.format(elapsed / 60.0)) def compress(infile, outfile, buflen=-1): """Use gzip to compress the data in infile, saving it to outfile. If buflen == -1, we compress all of the data at once. Otherwise we compress chunks, creating a concatenated gzip stream. """ def compress_with_gzip_module(): print('Compressing data using python gzip module ...', outfile) with open(infile, 'rb') as inf: while True: with gzip.open(outfile, 'a') as outf: data = inf.read(buflen) if len(data) == 0: break outf.write(data) def compress_with_gzip_command(): with open(infile, 'rb') as inf, open(outfile, 'wb') as outf: # If buflen == -1, do a single call if buflen == -1: print('Compressing data with a single ' 'call to gzip ...', outfile) sp.Popen(['gzip', '-c'], stdin=inf, stdout=outf).wait() # Otherwise chunk the call else: print('Compressing data with multiple ' 'calls to gzip ...', outfile) nbytes = 0 chunk = inf.read(buflen) while len(chunk) != 0: proc = sp.Popen(['gzip', '-c'], stdin=sp.PIPE, stdout=outf) proc.communicate(chunk) nbytes += len(chunk) if (nbytes / buflen) % 10 == 0: print('Compressed to {}...'.format(nbytes)) chunk = inf.read(buflen) # Use python gzip module on windows, # can't assume gzip exists if sys.platform.startswith("win"): target = compress_with_gzip_module # If not windows, assume that gzip command # exists, and use it, because the python # gzip module is super-slow. else: target = compress_with_gzip_command cmpThread = threading.Thread(target=target) cmpThread.start() poll(lambda : not cmpThread.is_alive()) def gen_test_data(filename, nelems, concat): """Make some data to test with. """ start = time.time() # The file just contains a sequentially # increasing list of numbers print('Generating test data ({} elems, {} bytes -> {})'.format( nelems, nelems * 8, filename)) # Generate the data as a numpy memmap array. # Allocate at most 128MB at a time toWrite = nelems offset = 0 writeBlockSize = min(16777216, nelems) datafile = '{}_temp'.format(filename) open(datafile, 'wb+').close() data = np.memmap(datafile, dtype=np.uint64, shape=nelems) idx = 0 while toWrite > 0: if idx % 10 == 0: print('Generated to {}...'.format(offset)) thisWrite = min(writeBlockSize, toWrite) vals = np.arange(offset, offset + thisWrite, dtype=np.uint64) data[offset:offset + thisWrite] = vals toWrite -= thisWrite offset += thisWrite idx += 1 data.flush() if not concat: maxBufSize = -1 else: maxBufSize = 8 * min(16777216, nelems // 50) compress(datafile, filename, maxBufSize) end = time.time() del data os.remove(datafile) print('Done in {:0.2f} seconds'.format(end - start)) def _check_chunk(args): s, e, test_data = args valid = np.arange(s, e, dtype=np.uint64) return np.all(test_data == valid) def check_data_valid(data, startval, endval=None): if endval is None: endval = len(data) chunksize = 10000000 startval = int(startval) endval = int(endval) offsets = np.arange(0, len(data), chunksize) args = [] result = True for offset in offsets: s = startval + offset e = min(s + chunksize, endval) nelems = e - s test_chunk = data[offset:offset + nelems] args.append((s, e, test_chunk)) pool = mp.Pool() result = all(pool.map(_check_chunk, args)) pool.terminate() return result indexed-gzip-0.6.1/indexed_gzip/tests/ctest_indexed_gzip.pyx000066400000000000000000000267101317257227400243550ustar00rootroot00000000000000# # Tests for the indexed_gzip module. # # Author: Paul McCarthy # from __future__ import print_function import os import os.path as op import itertools as it import subprocess as sp import sys import time import gzip import random import struct import hashlib import textwrap import numpy as np import pytest import indexed_gzip as igzip from . import gen_test_data from . import check_data_valid from . import testdir def read_element(gzf, element, seek=True): if seek: gzf.seek(element * 8) bytes = gzf.read(8) val = np.ndarray(1, np.uint64, buffer=bytes) return val[0] def write_text_to_gzip_file(fname, lines): with gzip.open(fname, mode='wb') as f: for line in lines: f.write('{}\n'.format(line).encode()) def test_open_close(testfile, nelems, seed): f = igzip.IndexedGzipFile(filename=testfile) try: element = np.random.randint(0, nelems, 1) readval = read_element(f, element) assert readval == element finally: f.close() assert f.closed def test_open_close_ctxmanager(testfile, nelems, seed): with igzip.IndexedGzipFile(filename=testfile) as f: element = np.random.randint(0, nelems, 1) readval = read_element(f, element) assert readval == element assert f.closed def test_atts(testfile): modes = [None, 'rb', 'r'] for m in modes: with igzip.IndexedGzipFile(filename=testfile, mode=m) as f: assert not f.closed assert f.readable() assert f.seekable() assert not f.writable() assert f.mode == 'rb' assert f.tell() == 0 assert f.fileno() == f.fileobj().fileno() def test_init_failure_cases(concat): with testdir() as td: testfile = op.join(td, 'test.gz') gen_test_data(testfile, 65536, concat) # No writing with pytest.raises(ValueError): gf = igzip.IndexedGzipFile(filename=testfile, mode='w') with pytest.raises(ValueError): gf = igzip.IndexedGzipFile(filename=testfile, mode='wb') # No writing f = open(testfile, mode='wb') with pytest.raises(ValueError): gf = igzip.IndexedGzipFile(fid=f) f.close() # No writing f = open(testfile, mode='w') with pytest.raises(ValueError): gf = igzip.IndexedGzipFile(fid=f) f.close() # Need a filename or fid with pytest.raises(ValueError): f = igzip.IndexedGzipFile() def test_init_success_cases(concat): with testdir() as td: testfile = op.join(td, 'test.gz') gen_test_data(testfile, 65536, concat) gf1 = igzip.IndexedGzipFile(filename=testfile) gf2 = igzip.IndexedGzipFile(filename=testfile, mode='r') gf3 = igzip.IndexedGzipFile(filename=testfile, mode='rb') gf1.close() gf2.close() gf3.close() def test_create_from_open_handle(testfile, nelems, seed): f = open(testfile, 'rb') gzf = igzip.IndexedGzipFile(fid=f) element = np.random.randint(0, nelems, 1) readval = read_element(gzf, element) gzf.close() try: assert readval == element assert gzf.closed assert not f.closed finally: f.close() def test_read_all(testfile, nelems, use_mmap): if use_mmap: pytest.skip('skipping test_read_all test as ' 'it will require too much memory') with igzip.IndexedGzipFile(filename=testfile) as f: data = f.read(nelems * 8) data = np.ndarray(shape=nelems, dtype=np.uint64, buffer=data) # Check that every value is valid assert check_data_valid(data, 0) def test_read_beyond_end(concat): with testdir() as tdir: nelems = 65536 testfile = op.join(tdir, 'test.gz') gen_test_data(testfile, nelems, concat) with igzip.IndexedGzipFile(filename=testfile, readall_buf_size=1024) as f: # Try with a specific number of bytes data1 = f.read(nelems * 8 + 10) # And also with unspecified numbytes f.seek(0) data2 = f.read() data1 = np.ndarray(shape=nelems, dtype=np.uint64, buffer=data1) data2 = np.ndarray(shape=nelems, dtype=np.uint64, buffer=data2) assert check_data_valid(data1, 0) assert check_data_valid(data2, 0) def test_seek_and_read(testfile, nelems, niters, seed): with igzip.IndexedGzipFile(filename=testfile) as f: # Pick some random elements and make # sure their values are all right seekelems = np.random.randint(0, nelems, niters) for i, testval in enumerate(seekelems): readval = read_element(f, testval) ft = f.tell() assert ft == (testval + 1) * 8 assert readval == testval def test_seek_and_tell(testfile, nelems, niters, seed): filesize = nelems * 8 with igzip.IndexedGzipFile(filename=testfile) as f: # Pick some random seek positions # and make sure that seek and tell # return their location correctly seeklocs = np.random.randint(0, filesize, niters) for seekloc in seeklocs: st = f.seek(seekloc) ft = f.tell() assert ft == seekloc assert st == seekloc # Also test that seeking beyond # EOF is clamped to EOF eofseeks = [filesize, filesize + 1, filesize + 2, filesize + 3, filesize + 4, filesize + 1000, filesize * 1000] for es in eofseeks: assert f.seek(es) == filesize assert f.tell() == filesize def test_readinto(): lines = textwrap.dedent(""" line 1 line 2 this is line 3 line the fourth here is the fifth line """).strip().split('\n') def line_offset(idx): return sum([len(l) for l in lines[:idx]]) + idx with testdir() as td: testfile = op.join(td, 'test.gz') write_text_to_gzip_file(testfile, lines) with igzip.IndexedGzipFile(filename=testfile) as f: # read first line into a byte array buf = bytearray(len(lines[0])) f.seek(0) assert f.readinto(buf) == len(lines[0]) assert buf.decode() == lines[0] # read first line into memoryvew buf = memoryview(bytearray(len(lines[0]))) f.seek(0) assert f.readinto(buf) == len(lines[0]) assert buf.tobytes().decode() == lines[0] # read an arbitrary line offset = line_offset(2) buf = bytearray(len(lines[2])) f.seek(offset) assert f.readinto(buf) == len(lines[2]) assert buf.decode() == lines[2] # read the end line, sans-newline offset = line_offset(len(lines) - 1) buf = bytearray(len(lines[-1])) f.seek(offset) assert f.readinto(buf) == len(lines[-1]) assert buf.decode() == lines[-1] # read the end line, with newline buf = bytearray(len(lines[-1]) + 1) f.seek(offset) assert f.readinto(buf) == len(lines[-1]) + 1 assert buf.decode() == lines[-1] + '\n' # read the end line with a bigger buffer buf = bytearray(len(lines[-1]) + 10) f.seek(offset) assert f.readinto(buf) == len(lines[-1]) + 1 assert buf.decode() == lines[-1] + '\n' + (b'\0' * 9).decode() # start at EOF, and try to read something filelen = sum([len(l) for l in lines]) + len(lines) f.seek(filelen) buf = bytearray([99 for i in range(len(buf))]) assert f.readinto(buf) == 0 assert all([b == chr(99) for b in buf.decode()]) def test_readline(): lines = textwrap.dedent(""" this is some text split across several lines how creative """).strip().split('\n') with testdir() as td: fname = op.join(td, 'test.gz') write_text_to_gzip_file(fname, lines) with igzip.IndexedGzipFile(fname) as f: seekpos = 0 for line in lines: assert f.readline() == (line + '\n').encode() seekpos += len(line) + 1 assert f.tell() == seekpos # Should return empty string after EOF assert f.readline() == b'' def test_readline_sizelimit(): lines = ['line one', 'line two'] with testdir() as td: fname = op.join(td, 'test.gz') write_text_to_gzip_file(fname, lines) with igzip.IndexedGzipFile(fname) as f: # limit to one character before the end of the first line l = f.readline(len(lines[0]) - 1) assert l == (lines[0][:-1]).encode() # limit to the last character of the first line f.seek(0) l = f.readline(len(lines[0]) - 1) assert l == (lines[0][:-1]).encode() # limit to the newline at the end of the first line f.seek(0) l = f.readline(len(lines[0]) + 1) assert l == (lines[0] + '\n').encode() # limit to the first character after the first line f.seek(0) l = f.readline(len(lines[0]) + 2) assert l == (lines[0] + '\n').encode() def test_readlines(): lines = textwrap.dedent(""" this is some more text split across several lines super imaginative test data """).strip().split('\n') with testdir() as td: fname = op.join(td, 'test.gz') write_text_to_gzip_file(fname, lines) with igzip.IndexedGzipFile(fname) as f: gotlines = f.readlines() assert len(lines) == len(gotlines) for expl, gotl in zip(lines, gotlines): assert (expl + '\n').encode() == gotl assert f.read() == b'' def test_readlines_sizelimit(): lines = ['line one', 'line two'] data = '\n'.join(lines) + '\n' with testdir() as td: fname = op.join(td, 'test.gz') write_text_to_gzip_file(fname, lines) limits = range(len(data) + 2) with igzip.IndexedGzipFile(fname) as f: for lim in limits: f.seek(0) gotlines = f.readlines(lim) # Expect the first line if lim < len(lines[0]) + 1: assert len(gotlines) == 1 assert gotlines[0] == (lines[0] + '\n').encode() # Expect both lines else: assert len(gotlines) == 2 assert gotlines[0] == (lines[0] + '\n').encode() assert gotlines[1] == (lines[1] + '\n').encode() def test_iter(): lines = textwrap.dedent(""" this is even more text that is split across several lines the creativity involved in generating this test data is unparalleled """).strip().split('\n') with testdir() as td: fname = op.join(td, 'test.gz') write_text_to_gzip_file(fname, lines) with igzip.IndexedGzipFile(fname) as f: for i, gotline in enumerate(f): assert (lines[i] + '\n').encode() == gotline with pytest.raises(StopIteration): next(f) indexed-gzip-0.6.1/indexed_gzip/tests/ctest_zran.pyx000066400000000000000000000542601317257227400226570ustar00rootroot00000000000000# # Tests for the zran module. # # Author: Paul McCarthy # from __future__ import print_function from __future__ import division import os import os.path as op import itertools as it import subprocess as sp import sys import time import random import hashlib import tempfile import threading import numpy as np cimport numpy as np from posix.types cimport off_t from libc.stdio cimport (SEEK_SET, SEEK_CUR, FILE, fdopen, fwrite) from libc.string cimport memset from cpython.mem cimport (PyMem_Malloc, PyMem_Realloc, PyMem_Free) from posix.mman cimport (mmap, munmap, PROT_READ, PROT_WRITE, MAP_ANON, MAP_SHARED) from . import poll, check_data_valid cdef extern from "sys/mman.h": cdef enum: MAP_FAILED cimport indexed_gzip.zran as zran np.import_array() cdef read_element(zran.zran_index_t *index, element, nelems, seek=True): cdef void *buffer buf = ReadBuffer(8) buffer = buf.buffer if element >= nelems: expseek = zran.ZRAN_SEEK_EOF else: expseek = zran.ZRAN_SEEK_OK if element >= nelems: exptell = (nelems * 8) else: exptell = element * 8 if seek: gotseek = zran.zran_seek(index, element * 8, SEEK_SET, NULL) gottell = zran.zran_tell(index) try: assert gotseek == expseek assert gottell == exptell except: print('expseek: {}'.format(expseek)) print('exptell: {}'.format(exptell)) print('gotseek: {}'.format(gotseek)) print('gottell: {}'.format(gottell)) raise if element >= nelems: expread = zran.ZRAN_READ_EOF else: expread = 8 if element >= nelems: exptell = (nelems * 8) else: exptell = (element + 1) * 8 gotread = zran.zran_read(index, buffer, 8) gottell = zran.zran_tell(index) try: assert gotread == expread assert gottell == exptell except: print('nelems: {}'.format(nelems)) print('element: {}'.format(element)) print('expread: {}'.format(expread)) print('exptell: {}'.format(exptell)) print('gotread: {}'.format(gotread)) print('gottell: {}'.format(gottell)) raise if element < nelems: pybuf = (buffer)[:8] val = np.ndarray(1, np.uint64, buffer=pybuf) return val[0] else: return None cdef class ReadBuffer: """Wrapper around a chunk of memory. .. see:: http://docs.cython.org/src/tutorial/memory_allocation.html """ cdef void *buffer """A raw chunk of bytes. """ cdef bint use_mmap """ """ cdef size_t size """ """ cdef object mmap_fd cdef object mmap_path def __cinit__(self, size_t size, use_mmap=False): """Allocate ``size`` bytes of memory. """ self.use_mmap = use_mmap self.mmap_fd = None self.mmap_path = None self.size = size self.buffer = NULL if not self.use_mmap: self.buffer = PyMem_Malloc(size) memset(self.buffer, 0, size); else: fd, path = tempfile.mkstemp('readbuf_mmap_{}'.format(id(self))) print('Memory-mapping {:0.2f} GB ({})'.format(size / 1073741824., path)) def initmem(): towrite = size while towrite > 0: zeros = np.zeros(min(towrite, 134217728), dtype=np.uint8) towrite -= len(zeros) os.write(fd, zeros.tostring()) th = threading.Thread(target=initmem) th.start() poll(lambda : not th.is_alive()) self.mmap_fd = fd self.mmap_path = path self.buffer = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0) if self.buffer == MAP_FAILED: raise RuntimeError('mmap fail') if not self.buffer: raise RuntimeError('ReadBuffer init fail') def resize(self, size_t size): """Re-allocate the memory to the given ``size``. """ if self.use_mmap: raise NotImplementedError('Cannot resize a memmapped array!') buf = PyMem_Realloc(self.buffer, size) if not buf: raise MemoryError('PyMem_Realloc fail') self.buffer = buf self.size = size def __dealloc__(self): """Free the mwmory. """ if not self.use_mmap: PyMem_Free(self.buffer) else: munmap(self.buffer, self.size) os.close( self.mmap_fd) os.remove(self.mmap_path) def test_init(testfile): """Tests a bunch of permutations of the parameters to zran_init. """ spacings = [0, 16384, 32768, 65536, 524288, 1048576, 2097152, 4194304] window_sizes = [0, 8192, 16384, 32768, 65536, 131072] readbuf_sizes = [0, 8192, 16384, 24576, 32768, 65536, 131072] flags = [0, zran.ZRAN_AUTO_BUILD] cdef zran.zran_index_t index cdef FILE *cfid with open(testfile, 'rb') as pyfid: cfid = fdopen(pyfid.fileno(), 'rb') for s, w, r, f in it.product(spacings, window_sizes, readbuf_sizes, flags): result = not zran.zran_init(&index, cfid, s, w, r, f) expected = True # zran_init should fail if the point spacing # is less than the window size if w == 0: w = 32768 if s == 0: s = 1048576 if r == 0: r = 16384 expected = (w >= 32768) and (s > w) assert result == expected zran.zran_free(&index) def test_init_file_modes(testfile): modes = ['r', 'r+', 'w', 'w+', 'a', 'a+'] files = [testfile, testfile, 'dummy.gz', 'dummy.gz', 'dummy.gz', 'dummy.gz'] cdef zran.zran_index_t index cdef FILE *cfid cdef bytes bmode cdef char *cmode for filename, mode in zip(files, modes): with open(filename, mode) as pyfid: bmode = mode.encode() cmode = bmode cfid = fdopen(pyfid.fileno(), cmode) expected = mode == 'r' result = not zran.zran_init(&index, cfid, 0, 0, 0, 0) assert result == expected zran.zran_free(&index) if filename == 'dummy.gz' and op.exists(filename): os.remove(filename) def test_no_auto_build(testfile, nelems): cdef zran.zran_index_t index cdef void *buffer filesize = nelems * 8 indexSpacing = max(1048576, filesize // 1500) bufSize = 1048576 buf = ReadBuffer(bufSize) buffer = buf.buffer with open(testfile, 'rb') as pyfid: cfid = fdopen(pyfid.fileno(), 'rb') assert not zran.zran_init(&index, cfid, indexSpacing, 32768, 131072, 0) assert zran.zran_seek(&index, 0, SEEK_SET, NULL) == zran.ZRAN_SEEK_OK assert zran.zran_tell(&index) == 0 assert zran.zran_seek(&index, 1, SEEK_SET, NULL) == zran.ZRAN_SEEK_NOT_COVERED assert zran.zran_tell(&index) == 0 gotread = zran.zran_read(&index, buffer, bufSize) gottell = zran.zran_tell(&index) if bufSize > filesize: expread = filesize else: expread = bufSize if bufSize > filesize: exptell = filesize else: exptell = bufSize try: assert gotread == expread assert gottell == exptell except: print("expread: {}".format(expread)) print("gotread: {}".format(gotread)) print("exptell: {}".format(exptell)) print("gottell: {}".format(gottell)) raise pybuf = (buffer)[:gotread] data = np.ndarray(gotread // 8, np.uint64, pybuf) assert check_data_valid(data, 0) if bufSize < filesize: assert zran.zran_read(&index, buffer, bufSize) == zran.ZRAN_READ_NOT_COVERED def test_seek_to_end(testfile, nelems): cdef zran.zran_index_t index filesize = nelems * 8 seek_point = filesize - 1 indexSpacing = max(524288, filesize // 1500) with open(testfile, 'rb') as pyfid: cfid = fdopen(pyfid.fileno(), 'rb') assert not zran.zran_init(&index, cfid, indexSpacing, 32768, 131072, zran.ZRAN_AUTO_BUILD) assert zran.zran_seek(&index, seek_point, SEEK_SET, NULL) == 0 zt = zran.zran_tell(&index) assert zt == seek_point zran.zran_free(&index) def test_seek_cur(testfile, nelems): cdef zran.zran_index_t index filesize = nelems * 8 indexSpacing = max(524288, filesize // 1500) seekstep = max(1, (nelems - 1) // 500) curelem = 0; with open(testfile, 'rb') as pyfid: cfid = fdopen(pyfid.fileno(), 'rb') assert not zran.zran_init(&index, cfid, indexSpacing, 32768, 131072, zran.ZRAN_AUTO_BUILD) while curelem < nelems: if (curelem + seekstep) * 8 < filesize: exp = zran.ZRAN_SEEK_OK else: exp = zran.ZRAN_SEEK_EOF assert zran.zran_seek(&index, seekstep * 8, SEEK_CUR, NULL) == exp if exp == zran.ZRAN_SEEK_EOF: break curelem += seekstep zt = zran.zran_tell(&index) val = read_element(&index, curelem, nelems, False) assert zt == curelem * 8 assert val == curelem assert zran.zran_seek(&index, -8, SEEK_CUR, NULL) == zran.ZRAN_SEEK_OK zran.zran_free(&index) def test_seek_beyond_end(testfile, nelems): cdef zran.zran_index_t index filesize = nelems * 8 indexSpacing = max(524288, filesize // 1500) seekpoints = [filesize - 10, filesize - 2, filesize - 1, filesize, filesize + 1, filesize + 2, filesize + 10] with open(testfile, 'rb') as pyfid: cfid = fdopen(pyfid.fileno(), 'rb') assert not zran.zran_init(&index, cfid, indexSpacing, 32768, 131072, zran.ZRAN_AUTO_BUILD) for sp in seekpoints: zs = zran.zran_seek(&index, sp, SEEK_SET, NULL) if sp >= filesize: expected = zran.ZRAN_SEEK_EOF else: expected = zran.ZRAN_SEEK_OK try: assert zs == expected except: print("{} != {} [sp={}, size={}]".format(zs, expected, sp, filesize)) raise zt = zran.zran_tell(&index) if sp >= filesize: expected = filesize else: expected = sp try: assert zt == expected except: print("{} != {}".format(zt, expected)) raise zran.zran_free(&index) def test_sequential_seek_to_end(testfile, nelems, niters): cdef zran.zran_index_t index filesize = nelems * 8 seek_points = np.random.randint(0, filesize, niters, dtype=np.uint64) seek_points = np.sort(seek_points) indexSpacing = max(524288, filesize // 2000) with open(testfile, 'rb') as pyfid: cfid = fdopen(pyfid.fileno(), 'rb') assert not zran.zran_init(&index, cfid, indexSpacing, 32768, 131072, zran.ZRAN_AUTO_BUILD) for sp in seek_points: if sp >= filesize: expseek = zran.ZRAN_SEEK_EOF exptell = filesize else: expseek = zran.ZRAN_SEEK_OK exptell = sp seek = zran.zran_seek(&index, sp, SEEK_SET, NULL) tell = zran.zran_tell(&index) try: assert seek == expseek assert tell == exptell except: print("expseek: {}".format(expseek)) print("exptell: {}".format(exptell)) print("seek: {}".format(seek)) print("tell: {}".format(tell)) raise zran.zran_free(&index) def test_random_seek(testfile, nelems, niters, seed): cdef zran.zran_index_t index filesize = nelems * 8 seekpoints = [random.randint(0, filesize) for i in range(niters)] indexSpacing = max(524288, filesize // 1000) with open(testfile, 'rb') as pyfid: cfid = fdopen(pyfid.fileno(), 'rb') assert not zran.zran_init(&index, cfid, indexSpacing, 32768, 131072, zran.ZRAN_AUTO_BUILD) for sp in seekpoints: assert zran.zran_seek(&index, sp, SEEK_SET, NULL) == 0 zt = zran.zran_tell(&index) assert zt == sp zran.zran_free(&index) def test_read_all(testfile, nelems, use_mmap): filesize = nelems * 8 indexSpacing = max(524288, filesize // 1000) cdef zran.zran_index_t index cdef void *buffer cdef np.npy_intp nelemsp buf = ReadBuffer(filesize, use_mmap=use_mmap) buffer = buf.buffer with open(testfile, 'rb') as pyfid: cfid = fdopen(pyfid.fileno(), 'rb') assert not zran.zran_init(&index, cfid, indexSpacing, 32768, 131072, zran.ZRAN_AUTO_BUILD) nbytes = zran.zran_read(&index, buffer, filesize) assert nbytes == filesize assert zran.zran_tell(&index) == nbytes zran.zran_free(&index) nelemsp = nbytes / 8. data = np.PyArray_SimpleNewFromData(1, &nelemsp, np.NPY_UINT64, buffer) assert check_data_valid(data, 0) def test_seek_then_read_block(testfile, nelems, niters, seed, use_mmap): filesize = nelems * 8 indexSpacing = max(524288, filesize // 1000) buf = ReadBuffer(filesize, use_mmap=use_mmap) seekelems = np.random.randint(0, nelems - 1, niters, dtype=np.uint64) cdef zran.zran_index_t index cdef void *buffer = buf.buffer cdef np.npy_intp nelemsp with open(testfile, 'rb') as pyfid: cfid = fdopen(pyfid.fileno(), 'rb') assert not zran.zran_init(&index, cfid, indexSpacing, 32768, 131072, zran.ZRAN_AUTO_BUILD) for i, se in enumerate(seekelems): if se == nelems - 1: readelems = 1 else: readelems = np.random.randint(1, nelems - se) start = time.time() print("{} / {}: reading {} elements from {} ... ".format( i, len(seekelems), readelems, se), end='') assert zran.zran_seek(&index, se * 8, SEEK_SET, NULL) == zran.ZRAN_SEEK_OK nbytes = zran.zran_read(&index, buffer, readelems * 8) try: assert nbytes == readelems * 8 assert zran.zran_tell(&index) == (se + readelems) * 8 except: print('seekelem: {}'.format(se)) print('readelems: {}'.format(readelems)) print('nbytes: {}'.format(nbytes)) print(' should be: {}'.format(readelems * 8)) print('ftell: {}'.format(zran.zran_tell(&index))) print(' should be: {}'.format((se + readelems) * 8)) raise nelemsp = nbytes / 8. data = np.PyArray_SimpleNewFromData(1, &nelemsp, np.NPY_UINT64, buffer) assert check_data_valid(data, se, se + readelems) end = time.time() print("{:0.2f} seconds".format(end - start)) zran.zran_free(&index) def test_random_seek_and_read(testfile, nelems, niters, seed): cdef zran.zran_index_t index filesize = nelems * 8 seekelems = np.random.randint(0, nelems, niters) indexSpacing = max(524288, filesize // 1000) with open(testfile, 'rb') as pyfid: cfid = fdopen(pyfid.fileno(), 'rb') assert not zran.zran_init(&index, cfid, indexSpacing, 32768, 131072, zran.ZRAN_AUTO_BUILD) for se in seekelems: # Should never happen if se >= nelems: expval = None else: expval = se val = read_element(&index, se, nelems, True) try: assert val == expval except: print("{} != {}".format(val, se)) raise zran.zran_free(&index) def test_read_all_sequential(testfile, nelems): cdef zran.zran_index_t index filesize = nelems * 8 indexSpacing = max(524288, filesize // 1000) # Takes too long to read all elements seekelems = np.random.randint(0, nelems - 1, 10000, dtype=np.uint64) seekelems = np.sort(seekelems) with open(testfile, 'rb') as pyfid: cfid = fdopen(pyfid.fileno(), 'rb') assert not zran.zran_init(&index, cfid, indexSpacing, 32768, 131072, zran.ZRAN_AUTO_BUILD) for se in seekelems: val = read_element(&index, se, nelems, True) try: assert val == se except: print("{} != {}".format(val, se)) print("{:x} != {:x}".format(val, se)) raise zran.zran_free(&index) def test_build_then_read(testfile, nelems, seed, use_mmap): filesize = nelems * 8 indexSpacing = max(524288, filesize // 1000) buf = ReadBuffer(filesize, use_mmap) seekelems = np.random.randint(0, nelems - 1, 5000, dtype=np.uint64) cdef zran.zran_index_t index cdef void *buffer = buf.buffer with open(testfile, 'rb') as pyfid: cfid = fdopen(pyfid.fileno(), 'rb') assert not zran.zran_init(&index, cfid, indexSpacing, 32768, 131072, zran.ZRAN_AUTO_BUILD) assert not zran.zran_build_index(&index, 0, 0) for se in seekelems: assert zran.zran_seek(&index, se * 8, SEEK_SET, NULL) == 0 if se == nelems - 1: readelems = 1 else: readelems = np.random.randint(1, min(nelems - se, 5000)) nbytes = zran.zran_read(&index, buffer, readelems * 8) assert nbytes == readelems * 8 assert zran.zran_tell(&index) == (se + readelems) * 8 pybuf = (buffer)[:nbytes] data = np.ndarray(nbytes // 8, np.uint64, pybuf) for i, val in enumerate(data, se): assert val == i zran.zran_free(&index) def test_readbuf_spacing_sizes(testfile, nelems, niters, seed): cdef zran.zran_index_t index spacings = [262144, 524288, 1048576, 2097152, 4194304, 8388608] bufsizes = [16384, 65536, 131072, 262144, 524288, 1048575, 1048576, 1048577, 2097152, 4194304, 8388608] seekelems = np.random.randint(0, nelems, niters // 2) seekelems = np.concatenate((spacings, bufsizes, seekelems)) for sbi, (spacing, bufsize) in enumerate(it.product(spacings, bufsizes)): with open(testfile, 'rb') as pyfid: print('{} / {}: spacing={}, bufsize={} ... '.format( sbi, len(spacings) * len(bufsizes), spacing, bufsize), end='') cfid = fdopen(pyfid.fileno(), 'rb') assert not zran.zran_init(&index, cfid, spacing, 32768, bufsize, zran.ZRAN_AUTO_BUILD) for i, se in enumerate(seekelems): # print('{} / {}: {}'.format(i, len(seekelems), se)) if se >= nelems: expval = None else: expval = se val = read_element(&index, se, nelems, seek=True) try: assert val == expval except: print('{} != {}'.format(val, expval)) raise print() zran.zran_free(&index) indexed-gzip-0.6.1/indexed_gzip/tests/test_indexed_gzip.py000066400000000000000000000042401317257227400240140ustar00rootroot00000000000000#!/usr/bin/env python # # test_indexed_gzip.py - Python wrapper around ctest_indexed_gzip.pyx. # # Author: Paul McCarthy # import pytest from . import ctest_indexed_gzip pytestmark = pytest.mark.indexed_gzip_test def test_open_close( testfile, nelems, seed): ctest_indexed_gzip.test_open_close( testfile, nelems, seed) def test_open_close_ctxmanager( testfile, nelems, seed): ctest_indexed_gzip.test_open_close_ctxmanager( testfile, nelems, seed) def test_atts( testfile): ctest_indexed_gzip.test_atts( testfile) def test_init_failure_cases( concat): ctest_indexed_gzip.test_init_failure_cases( concat) def test_init_success_cases( concat): ctest_indexed_gzip.test_init_success_cases( concat) def test_create_from_open_handle(testfile, nelems, seed): ctest_indexed_gzip.test_create_from_open_handle(testfile, nelems, seed) def test_read_all( testfile, nelems, use_mmap): ctest_indexed_gzip.test_read_all( testfile, nelems, use_mmap) def test_read_beyond_end( concat): ctest_indexed_gzip.test_read_beyond_end( concat) def test_seek_and_read( testfile, nelems, niters, seed): ctest_indexed_gzip.test_seek_and_read( testfile, nelems, niters, seed) def test_seek_and_tell( testfile, nelems, niters, seed): ctest_indexed_gzip.test_seek_and_tell( testfile, nelems, niters, seed) def test_readinto(): ctest_indexed_gzip.test_readinto() def test_readline(): ctest_indexed_gzip.test_readline() def test_readline_sizelimit(): ctest_indexed_gzip.test_readline_sizelimit() def test_readlines(): ctest_indexed_gzip.test_readlines() def test_readlines_sizelimit(): ctest_indexed_gzip.test_readlines_sizelimit() def test_iter(): ctest_indexed_gzip.test_iter() indexed-gzip-0.6.1/indexed_gzip/tests/test_safe_indexed_gzip.py000066400000000000000000000030771317257227400250210ustar00rootroot00000000000000#!/usr/bin/env python # # test_safe_indexed_gzip.py - # # Author: Paul McCarthy # from __future__ import print_function from __future__ import division import sys import threading import numpy as np import pytest import indexed_gzip as igzip from . import check_data_valid pytestmark = pytest.mark.indexed_gzip_test def test_SafeIndexedGzipFile_open_close(testfile): f = igzip.SafeIndexedGzipFile(filename=testfile) f.seek(10) f.read(10) f.close() def test_SafeIndexedGzipFile_pread_threaded(testfile, nelems): filesize = nelems * 8 indexSpacing = max(524288, filesize // 2000) with igzip.SafeIndexedGzipFile(filename=testfile, spacing=indexSpacing) as f: readelems = 50 readsize = readelems * 8 nthreads = 100 allreads = [] def do_pread(nbytes, offset): data = f.pread(nbytes, int(offset * 8)) allreads.append((offset, data)) offsets = np.linspace(0, nelems - readelems, nthreads, dtype=np.uint64) threads = [threading.Thread(target=do_pread, args=(readsize, o)) for o in offsets] [t.start() for t in threads] [t.join() for t in threads] assert len(allreads) == nthreads for offset, data in allreads: assert len(data) == readsize data = np.ndarray(shape=readelems, dtype=np.uint64, buffer=data) assert check_data_valid(data, offset, offset + readelems) indexed-gzip-0.6.1/indexed_gzip/tests/test_zran.py000066400000000000000000000045031317257227400223170ustar00rootroot00000000000000#!/usr/bin/env python # # test_zran.py - Python wrapper around ctest_zran.pyx. # # Author: Paul McCarthy # from __future__ import print_function import sys if not sys.platform.startswith("win"): # Run these tests only on POSIX systems import pytest import numpy as np from . import ctest_zran pytestmark = pytest.mark.zran_test def test_init( testfile): ctest_zran.test_init( testfile) def test_init_file_modes( testfile): ctest_zran.test_init_file_modes( testfile) def test_no_auto_build( testfile, nelems): ctest_zran.test_no_auto_build( testfile, nelems) def test_seek_to_end( testfile, nelems): ctest_zran.test_seek_to_end( testfile, nelems) def test_seek_cur( testfile, nelems): ctest_zran.test_seek_cur( testfile, nelems) def test_seek_beyond_end( testfile, nelems): ctest_zran.test_seek_beyond_end( testfile, nelems) def test_sequential_seek_to_end(testfile, nelems, niters): ctest_zran.test_sequential_seek_to_end(testfile, nelems, niters) def test_random_seek( testfile, nelems, niters, seed): ctest_zran.test_random_seek( testfile, nelems, niters, seed) def test_read_all( testfile, nelems, use_mmap): ctest_zran.test_read_all( testfile, nelems, use_mmap) def test_seek_then_read_block( testfile, nelems, niters, seed, use_mmap): ctest_zran.test_seek_then_read_block( testfile, nelems, niters, seed, use_mmap) def test_random_seek_and_read( testfile, nelems, niters, seed): ctest_zran.test_random_seek_and_read( testfile, nelems, niters, seed) def test_read_all_sequential( testfile, nelems): ctest_zran.test_read_all_sequential( testfile, nelems) def test_build_then_read( testfile, nelems, seed, use_mmap): ctest_zran.test_build_then_read( testfile, nelems, seed, use_mmap) def test_readbuf_spacing_sizes( testfile, nelems, niters, seed): ctest_zran.test_readbuf_spacing_sizes( testfile, nelems, niters, seed) indexed-gzip-0.6.1/indexed_gzip/zran.c000066400000000000000000002137601317257227400177170ustar00rootroot00000000000000/* * zran.c - indexed access to gzip files. * * See zran.h for documentation. * * This module was originally based on the zran example, written by Mark * Alder, which ships with the zlib source code. * * Author: Paul McCarthy */ #include #include #include #include #include #include "zlib.h" #ifdef _WIN32 #define FSEEK _fseeki64 #define FTELL _ftelli64 #include "windows.h" #include "io.h" static int is_readonly(FILE *fd) { /* Can't find a way to do this correctly under Windows and the check is not required anyway since the underlying Python module checks it already */ return 1; } #else #include #define FSEEK fseeko #define FTELL ftello /* Check if file is read-only */ static int is_readonly(FILE *fd) { return (fcntl(fileno(fd), F_GETFL) & O_ACCMODE) == O_RDONLY; } #endif #include "zran.h" #ifdef NO_C99 static double round(double val) { return floor(val + 0.5); } #endif /* * Turn this on to make noise. * * #define ZRAN_VERBOSE */ //#define ZRAN_VERBOSE #ifdef ZRAN_VERBOSE #define zran_log(...) fprintf(stderr, __VA_ARGS__) #else #define zran_log(...) #endif /* * Discards all points in the index which come after the specfiied * compressed offset. * * Returns 0 on success, non-0 on failure. */ static int _zran_invalidate_index( zran_index_t *index, /* The index */ uint64_t from /* Offset into the compressed data */ ); /* * Expands the capacity of the memory used to store the index ilst. * * Returns 0 on success, non-0 on failure. */ static int _zran_expand_point_list( zran_index_t *index /* The index */ ); /* * Reduces the capacity of the memory used to store the index list, so that it * is only as big as necessary. * * Returns 0 on success, non-0 on failure. */ static int _zran_free_unused( zran_index_t *index /* The index */ ); /* * Returns the current limit of the index, i.e. how much of the file is covered * by the index. */ static uint64_t _zran_index_limit( zran_index_t *index, /* The index */ uint8_t compressed /* Pass in non-0 to get the compressed stream limit, or 0 for the uncompressed limit. */ ); /* Return codes for _zran_get_point_at */ int ZRAN_GET_POINT_FAIL = -1; int ZRAN_GET_POINT_OK = 0; int ZRAN_GET_POINT_NOT_COVERED = 1; int ZRAN_GET_POINT_EOF = 2; /* * Searches for the zran_point which preceeds the given offset. The offset * may be specified as being relative to the start of the compressed data, * or the uncompressed data. * * Returns: * * - ZRAN_GET_POINT_OK on success. * * - ZRAN_GET_POINT_NOT_COVERED if the index does not yet cover the * specified offset. * - ZRAN_GET_POINT_EOF if the specified offset is at or beyond the end of * the file. */ static int _zran_get_point_at( zran_index_t *index, /* The index */ uint64_t offset, /* The desired offset into the compressed or uncompressed data stream */ uint8_t compressed, /* Pass in 0 or non-0 to indicate that the offset is relative to the uncompressed or compressed data streams, respectively. */ zran_point_t **point /* If an index point corresponding to the specified offset is identified, this pointer will be updated to point to it. */ ); /* * If the index has been created without the ZRAN_AUTO_BUILD flag, this * function is identical to the _zran_get_point_at function. * * If the index has been created with the ZRAN_AUTO_BUILD flag, and the * requested offset is beyond the current range of the index, the index will * be expanded to encompass it. * * The input arguments and return values are identical to the * _zran_get_point_at function, however if the index has been initialised * with the ZRAN_AUTO_BUILD flag, this function will never return * ZRAN_GET_POINT_NOT_COVERED. */ static int _zran_get_point_with_expand( zran_index_t *index, /* The index */ uint64_t offset, /* Desired offset */ uint8_t compressed, /* Compressed or uncompressed offset */ zran_point_t **point /* Place to store the identified point */ ); /* * Estimate an offset in the compressed / uncompressed data stream * corresponding to the given offset, which is specified in the uncompressed / * compressed data stream. If the given offset is specified relative to the * compressed data stream, the returned value is a location in the * uncompressed data stream which approximately corresponds to the given * offset. * * This function is used by the _zran_get_point_with_expand function, if the * index has been created with the ZRAN_AUTO_BUILD flag, to determine how far * the index needs to be expanded to cover a requested offset that is not yet * covered. */ static uint64_t _zran_estimate_offset( zran_index_t *index, /* The index */ uint64_t offset, /* The offset for which a corresponding offset is to be estimated. */ uint8_t compressed /* Pass in 0 or non-0 to indicate that the given offset is specified relative to the uncompressed or compressed stream, respectively. */ ); /* * Used by _zran_expand_index, and _zran_read. Seeks to the specified * compressed data offset, and initialises zlib to start * decompressing/inflating from said offset. * * Returns 0 for success, non-0 on failure. */ static int _zran_init_zlib_inflate( zran_index_t *index, /* The index */ z_stream *stream, /* Pointer to a z_stream struct */ zran_point_t *point /* Pass in NULL to initialise for inflation from the beginning of the stream. Or pass a pointer to the index point corresponding to the location to start from. */ ); /* * Expands the index from its current end-point until the given offset (which * must be specified relative to the compressed data stream). * * The index is expanded so that the last point comes after the given offset. * If the specified offset is past the last point in the index, a call to this * function is guaranteed to create at least one more index point. If there * is already an index point which comes after the offset, this function does * nothing, and return a success code. * * We require at least one point to be created, because we want index points * to be located at compression block boundaries, but in some data there may * be a long distance between block boundaries (longer than the desired index * point spacing). * * Returns 0 on success, non-0 on failure. */ static int _zran_expand_index( zran_index_t *index, /* The index */ uint64_t until /* Expand the index to this point */ ); /* * Adds a new point to the end of the index. */ static int _zran_add_point( zran_index_t *index, /* The index */ uint8_t bits, /* If the compressed and uncompressed offsets are not byte-aligned, this is the number of bits in the compressed data, before the cmp_offset, where the point is located. */ uint64_t cmp_offset, /* Offset into the compressed data. */ uint64_t uncmp_offset, /* Offset into the uncompressed data. */ uint32_t data_offset, /* Offset into the data pointer specifying the point at which the uncompressed data associated with this point begins - see _zran_expand_index. It is assumed that the uncompressed data wraps around this offset. */ uint32_t data_size, /* Number of bytes in data */ uint8_t *data /* Pointer to data_size bytes of uncompressed data preceeding this index point. */ ); /* _zran_find_next_stream return codes */ int ZRAN_FIND_STREAM_ERROR = -2; int ZRAN_FIND_STREAM_NOT_FOUND = -1; /* * This function is used to search for concatenated compressed streams. It * searches through the compressed data (pointed to by stream->next_in) to * find the location of the next compressed stream. * * If a new stream was found, the z_stream struct is re-initialised to * decompress data from the new stream. In this case, this function returns * the number of bytes in the compressed data that were read before the stream * was found. * * Otherwise (if a compressed stream was not found), this function returns * ZRAN_FIND_STREAM_NOT_FOUND. * * If an error occurs, ZRAN_FIND_STREAM_ERROR is returned. */ static int _zran_find_next_stream( zran_index_t *index, /* The index */ z_stream *stream /* The z_stream struct */ ); /* _zran_inflate return codes */ int ZRAN_INFLATE_ERROR = -5; int ZRAN_INFLATE_NOT_COVERED = -4; int ZRAN_INFLATE_OUTPUT_FULL = -3; int ZRAN_INFLATE_BLOCK_BOUNDARY = -2; int ZRAN_INFLATE_EOF = -1; int ZRAN_INFLATE_OK = 0; /* * _zran_inflate input flags. * Bit position, as a power of 2 */ uint32_t ZRAN_INFLATE_INIT_Z_STREAM = 1; uint32_t ZRAN_INFLATE_FREE_Z_STREAM = 2; uint32_t ZRAN_INFLATE_INIT_READBUF = 4; uint32_t ZRAN_INFLATE_FREE_READBUF = 8; uint32_t ZRAN_INFLATE_USE_OFFSET = 16; uint32_t ZRAN_INFLATE_CLEAR_READBUF_OFFSETS = 32; uint32_t ZRAN_INFLATE_STOP_AT_BLOCK = 64; /* Macros used by _zran_inflate for testing flags. */ #define inflate_init_stream( flags) ((flags & ZRAN_INFLATE_INIT_Z_STREAM) > 0) #define inflate_free_stream( flags) ((flags & ZRAN_INFLATE_FREE_Z_STREAM) > 0) #define inflate_init_readbuf( flags) ((flags & ZRAN_INFLATE_INIT_READBUF) > 0) #define inflate_free_readbuf( flags) ((flags & ZRAN_INFLATE_FREE_READBUF) > 0) #define inflate_use_offset( flags) ((flags & ZRAN_INFLATE_USE_OFFSET) > 0) #define inflate_stop_at_block(flags) ((flags & ZRAN_INFLATE_STOP_AT_BLOCK) > 0) #define inflate_clear_readbuf_offsets(flags) \ ((flags & ZRAN_INFLATE_CLEAR_READBUF_OFFSETS) > 0) /* * Inflate (decompress) the specified number of bytes, or until the next * Z_BLOCK/Z_STREAM_END is reached. * * This is a complicated function which implements the core decompression * routine, and is used by both _zran_expand_index, and zran_read. It reads * compressed data from the file, starting from the specified compressed * offset, inflates (a.k.a. decompresses) it, and copies the decompressed * data to the provided output buffer. * * This function may be used in a re-entrant or non-re-entrant manner, * depending on the flags which are used. In the latter (more likely) case, * various pieces of information representing the current inflation state are * stored in fields of the zran_index_t struct. * * Specifically, this function does the following: * * 1. Figures out the starting offsets into the compressed/uncompressed * streams. If the ZRAN_INFLATE_USE_OFFSET flag is active, the index * point preceeding the specified offset is used as the starting point. * If there is no such point, ZRAN_INFLATE_NOT_COVERED is returned. If * ZRAN_INFLATE_USE_OFFSET is not active, index->inflate_cmp_offset and * index->inflate_uncmp_offset are used as the starting point. * * 2. Initialises the z_stream struct, if * ZRAN_INFLATE_INIT_Z_STREAM is active. Otherwise, the function assumes * that the z_stream struct is already initialised and ready to be used. * * 3. Create a read buffer, if ZRAN_INFLATE_INIT_READBUF is active. A * reference to the read buffer is stored at index->readbuf. If * ZRAN_INFLATE_INIT_READBUF is not set, the function assumes that the * read buffer already exists. * * 4. If the ZRAN_INFLATE_CLEAR_READBUF_OFFSETS flag is active, the read * buffer offset (index->readbuf_offset) and length (index->readbuf_end) * fields are both set to 0. Otherwise, the function assumes that the * current offset/length values are valid. * * 5. Read some compressed data from the file into the read buffer as needed. * * 6. Pass that data to the zlib inflate function, and store the resulting * uncompressed data in the provided data buffer. * * 7. Repeat steps 5 and 6 until one of the following is true: * * - The requested number of bytes have been read * * - The output buffer is full * * - End of file is reached * * - ZRAN_INFLATE_STOP_AT_BLOCK is active, and a block is reached * * 8. If ZRAN_INFLATE_FREE_READBUF is active, the file read buffer is * de-allocated. * * 9. If ZRAN_INFLATE_FREE_Z_STREAM is active, the memory used by the * z_stream struct is de-allocated (via the zlib inflateEnd function). * * The control flags can be a combination (bitwise OR) of the following: * * - ZRAN_INFLATE_INIT_Z_STREAM: Initialise the z_stream struct * before inflation. * * - ZRAN_INFLATE_FREE_Z_STREAM: Clean up the z_stream struct * after inflation. * * - ZRAN_INFLATE_INIT_READBUF: Allocate a read buffer before * inflation. * * - ZRAN_INFLATE_FREE_READBUF: Free the read buffer after * inflation. * * - ZRAN_INFLATE_USE_OFFSET: If set, use the provided offset * parameter; otherwise, use the * offsets stored in the index * struct. * * - ZRAN_INFLATE_CLEAR_READBUF_OFFSETS: If set, clear the read buffer * offset/length stored in the index * struct, otherwise assume that they * are valid. * * - ZRAN_INFLATE_STOP_AT_BLOCK: If set, stop inflation when a * deflate block boundary is reached * (the Z_BLOCK flag is passed to the * zlib inflate function). Otherwise, * inflation will continue until one * of the conditions in step 7, above, * are met. * * This function returns one of the following codes. Furthermore, if an error * did not occur (i.e. anything but ZRAN_INFLATE_ERROR or * ZRAN_INFLATE_NOT_COVERED was returned), the total_consumed and total_output * parameters are respectively updated to contain the total number of * compressed bytes that were read from the file, and total number of * decompressed bytes that were copied to the data buffer. * - ZRAN_INFLATE_OK: Inflation was successful and the requested * number of bytes were copied to the provided * data buffer. * * - ZRAN_INFLATE_NOT_COVERED: The requested compressed data offset is not * covered by the index. * * - ZRAN_INFLATE_OUTPUT_FULL: The provided data buffer has been filled. * * - ZRAN_INFLATE_BLOCK_BOUNDARY: A deflate block boundary was encountered. * This will only be returned if the * ZRAN_INFLATE_STOP_AT_BLOCK flag is active. * * - ZRAN_INFLATE_EOF: The end of file has been reached. * * - ZRAN_INFLATE_ERROR: A critical error has occurred. */ static int _zran_inflate( zran_index_t *index, /* Pointer to the index. */ z_stream *strm, /* Pointer to a z_stream struct. */ uint64_t offset, /* Compressed data offset to start inflation from. */ uint16_t flags, /* Control flags. */ uint32_t *total_consumed, /* Pointer which is updated to contain the total number of bytes that were read from the input file. */ uint32_t *total_output, /* Pointer which is updated to contain the total number of bytes that were inflated, and stored in data. */ uint32_t len, /* Maximum number of bytes to inflate. May be 0. */ uint8_t *data /* Place to store the inflated bytes. */ ); /* Initialise a zran_index_t struct for use with the given GZIP file. */ int zran_init(zran_index_t *index, FILE *fd, uint32_t spacing, uint32_t window_size, uint32_t readbuf_size, uint16_t flags) { zran_point_t *point_list = NULL; int64_t compressed_size; zran_log("zran_init(%u, %u, %u, %u)\n", spacing, window_size, readbuf_size, flags); if (spacing == 0) spacing = 1048576; if (window_size == 0) window_size = 32768; if (readbuf_size == 0) readbuf_size = 16384; /* * The zlib manual specifies that a window size of 32KB is 'always enough' * to initialise inflation/deflation with a set dictionary. Less than * that is not guaranteed to be enough. */ if (window_size < 32768) goto fail; /* * window_size bytes of uncompressed data are stored with each seek point * in the index. So it's a bit silly to have the distance between * consecutive points less than the window size. */ if (spacing <= window_size) goto fail; /* The file must be opened in read-only mode */ if (!is_readonly(fd)) goto fail; /* * Calculate the size of the compressed file */ if (FSEEK(fd, 0, SEEK_END) != 0) goto fail; compressed_size = FTELL(fd); if (compressed_size < 0) goto fail; if (FSEEK(fd, 0, SEEK_SET) != 0) goto fail; /* * Allocate some initial space * for the index point list */ point_list = calloc(1, sizeof(zran_point_t) * 8); if (point_list == NULL) { goto fail; } /* initialise the index struct */ index->fd = fd; index->flags = flags; index->compressed_size = compressed_size; index->uncompressed_size = 0; index->spacing = spacing; index->window_size = window_size; index->log_window_size = (int)round(log10(window_size) / log10(2)); index->readbuf_size = readbuf_size; index->readbuf_offset = 0; index->readbuf_end = 0; index->readbuf = NULL; index->npoints = 0; index->size = 8; index->uncmp_seek_offset = 0; index->inflate_cmp_offset = 0; index->inflate_uncmp_offset = 0; index->list = point_list; return 0; fail: free(point_list); return -1; } /* Returns the compressed or uncompressed index limit. */ uint64_t _zran_index_limit(zran_index_t *index, uint8_t compressed) { if (index->npoints == 0) return 0; if (compressed) return index->list[index->npoints - 1].cmp_offset; else return index->list[index->npoints - 1].uncmp_offset; } /* Expands the memory used to store the index points. */ int _zran_expand_point_list(zran_index_t *index) { zran_point_t *new_list; uint32_t new_size = index->size * 2; zran_log("_zran_expand_point_list(%i -> %i)\n", index->size, new_size); new_list = realloc(index->list, sizeof(zran_point_t) * new_size); if (new_list == NULL) { /* old list is still valid */ return -1; } index->list = new_list; index->size = new_size; return 0; } /* Frees any unused memory allocated for index storage. */ int _zran_free_unused(zran_index_t *index) { zran_point_t *new_list; zran_log("_zran_free_unused\n"); new_list = realloc(index->list, sizeof(zran_point_t) * index->npoints); if (new_list == NULL) { return -1; } index->list = new_list; index->size = index->npoints; return 0; } /* Deallocate memory used by a zran_index_t struct. */ void zran_free(zran_index_t *index) { uint32_t i; zran_point_t *pt; zran_log("zran_free\n"); for (i = 0; i < index->npoints; i++) { pt = &(index->list[i]); free(pt->data); } free(index->list); index->fd = NULL; index->spacing = 0; index->window_size = 0; index->readbuf_size = 0; index->npoints = 0; index->size = 0; index->list = NULL; index->uncmp_seek_offset = 0; } /* Discard all points in the index after the specified compressed offset. */ int _zran_invalidate_index(zran_index_t *index, uint64_t from) { uint64_t i; zran_point_t *p; if (index->npoints == 0) return 0; for (i = 0; i < index->npoints; i++) { p = &(index->list[i]); if (p->cmp_offset >= from) break; } /* * The index doesn't cover * the requested offest */ if (i == index->npoints) return 0; if (i <= 1) index->npoints = 0; else index->npoints = i - 1; return _zran_free_unused(index); } /* (Re-)Builds the full index. */ int zran_build_index(zran_index_t *index, uint64_t from, uint64_t until) { if (_zran_invalidate_index(index, from) != 0) return -1; if (until == 0) until = index->compressed_size; return _zran_expand_index(index, until); } /* Searches for and returns the index at the specified offset. */ int _zran_get_point_at( zran_index_t *index, uint64_t offset, uint8_t compressed, zran_point_t **point) { uint64_t cmp_max; uint64_t uncmp_max; zran_point_t *last; zran_point_t *prev; zran_point_t *curr; uint8_t bit; uint32_t i; *point = NULL; /* * Bad input - past the end of the compressed or * uncompressed streams (if the latter is known). */ if (compressed && offset >= index->compressed_size) goto eof; if (!compressed && index->uncompressed_size > 0 && offset >= index->uncompressed_size) goto eof; zran_log("_zran_get_point_at(%llu, c=%u)\n", offset, compressed); /* * Figure out how much of the compressed and * uncompressed data the index currently covers. * * No points - no coverage. */ if (index->npoints == 0) { cmp_max = 0; uncmp_max = 0; } /* * Otherwise the offsets of the * last point in the index. */ else { last = &(index->list[index->npoints - 1]); uncmp_max = last->uncmp_offset; cmp_max = last->cmp_offset; } if ( compressed && offset > cmp_max) goto not_covered; if (!compressed && offset > uncmp_max) goto not_covered; /* * We should have an index point * which corresponds to this offset, * so let's search for it. */ prev = index->list; for (i = 1; i < index->npoints; i++) { curr = &(index->list[i]); if (compressed) { /* * Adjust the offset for non * byte-aligned seek points. */ if (curr->bits > 0) bit = 1; else bit = 0; if (curr->cmp_offset > offset + bit) break; } else { if (curr->uncmp_offset > offset) break; } prev = curr; } *point = prev; return ZRAN_GET_POINT_OK; not_covered: *point = NULL; return ZRAN_GET_POINT_NOT_COVERED; eof: *point = NULL; return ZRAN_GET_POINT_EOF; } /* * Get the index point corresponding to the given offset, expanding * the index as needed if ZRAN_AUTO_BUILD is active. */ int _zran_get_point_with_expand(zran_index_t *index, uint64_t offset, uint8_t compressed, zran_point_t **point) { int result; uint64_t expand; uint64_t limit; zran_log("_zran_get_point_with_expand(%llu, %u, autobuild=%u)\n", offset, compressed, index->flags & ZRAN_AUTO_BUILD); if ((index->flags & ZRAN_AUTO_BUILD) == 0) { return _zran_get_point_at(index, offset, compressed, point); } /* * See if there is an index point that * covers the specified offset. If there's * not, we're going to expand the index * until there is. */ result = _zran_get_point_at(index, offset, compressed, point); while (result == ZRAN_GET_POINT_NOT_COVERED) { /* * If result == ZRAN_GET_POINT_NOT_COVERED, * get_point says that an index point for * this offset doesn't yet exist. So let's * expand the index. * * Guess how far we need to expand the index, * and expand it by that much. */ if (compressed == 0) expand = _zran_estimate_offset(index, offset, 0); else expand = offset; /* * If _zran_estimate_offset was unable to * estimate a sensible compressed offset * (i.e. smaller or at the current index * extent), we force it past the limit, * so that the expand_index function will * create at least one point. */ limit = _zran_index_limit(index, 1); if (expand <= limit) expand = limit + 10; /* * Expand the index */ if (_zran_expand_index(index, expand) != 0) { goto fail; } /* * Index has been expanded, so * there should now be a point * which covers the requested * offset. */ result = _zran_get_point_at(index, offset, compressed, point); /* * If we've made it to EOF, return * a ref to the eof point. */ if (result == ZRAN_GET_POINT_EOF) { *point = &index->list[index->npoints - 1]; if (offset < index->uncompressed_size) { result = ZRAN_GET_POINT_OK; } } } return result; fail: return ZRAN_GET_POINT_FAIL; } /* * Given an offset in one stream, estimates the corresponding offset into the * other stream. */ uint64_t _zran_estimate_offset( zran_index_t *index, uint64_t offset, uint8_t compressed) { zran_point_t *last; uint64_t estimate; /* * The first index in the list maps * indices 0 and 0, which won't help * us here. So we need at least two * index points. */ if (index->npoints <= 1) last = NULL; else last = &(index->list[index->npoints - 1]); /* * We have no reference. At least two * index points need to have been created. * The assumed correspondences between * the compressed streams are arbitrary. */ if (last == NULL) { if (compressed) estimate = offset * 2.0; else estimate = offset * 0.8; } /* * I'm just assuming a roughly linear correspondence * between the compressed/uncompressed data streams. */ else if (compressed) { estimate = round(offset * ((float)last->uncmp_offset / last->cmp_offset)); } else { estimate = round(offset * ((float)last->cmp_offset / last->uncmp_offset)); } zran_log("_zran_estimate_offset(%llu, %u) = %llu\n", offset, compressed, estimate); return estimate; } /* Add a new point to the index. */ int _zran_add_point(zran_index_t *index, uint8_t bits, uint64_t cmp_offset, uint64_t uncmp_offset, uint32_t data_offset, uint32_t data_size, uint8_t *data) { uint8_t *point_data = NULL; zran_point_t *next = NULL; int doff; zran_log("_zran_add_point(%i, c=%lld + %i, u=%lld, data=%u / %u)\n", index->npoints, cmp_offset, bits > 0, uncmp_offset, data_offset, data_size); #ifdef ZRAN_VERBOSE if (data_offset >= index->window_size) doff = data_offset - index->window_size; else doff = data_size = (index->window_size - data_offset); if (data != NULL) zran_log("Window data: [%02x %02x %02x %02x ...]\n", data[(data_offset - index->window_size + 0) % data_size], data[(data_offset - index->window_size + 1) % data_size], data[(data_offset - index->window_size + 2) % data_size], data[(data_offset - index->window_size + 3) % data_size]); #endif /* if list is full, make it bigger */ if (index->npoints == index->size) { if (_zran_expand_point_list(index) != 0) { goto fail; } } /* * Allocate memory to store the * uncompressed data (the "window") * associated with this point. The * first index point (where uncmp_offset == 0) * has no data associated with it for * obvious reasons. */ if (uncmp_offset == 0) { point_data = NULL; } else { point_data = calloc(1, index->window_size); if (point_data == NULL) goto fail; } next = &(index->list[index->npoints]); next->bits = bits; next->cmp_offset = cmp_offset; next->uncmp_offset = uncmp_offset; next->data = point_data; /* * The uncompressed data may not start at * the beginning of the data pointer, but * rather from an arbitrary point. So we * copy the beginning of the window from * the end of data, and the end of the * window from the beginning of data. Does * that make sense? */ if (uncmp_offset > 0) { if (data_offset >= index->window_size) { memcpy(point_data, data + (data_offset - index->window_size), index->window_size); zran_log("Copy %u bytes from %u to %u\n", index->window_size, data_offset - index->window_size, data_offset); } else { memcpy(point_data, data + (data_size - (index->window_size - data_offset)), (index->window_size - data_offset)); memcpy(point_data + (index->window_size - data_offset), data, data_offset); zran_log("Copy %u bytes from %u to %u, %u bytes from %u to %u\n", (index->window_size - data_offset), (data_size - (index->window_size - data_offset)), data_size, data_offset, 0, data_offset); } } index->npoints++; return 0; fail: free(point_data); return -1; } /* Initialise the given z_stream struct for decompression/inflation. */ int _zran_init_zlib_inflate(zran_index_t *index, z_stream *stream, zran_point_t *point) { int ret; int windowBits; int64_t seek_loc; windowBits = index->log_window_size; stream->zalloc = Z_NULL; stream->zfree = Z_NULL; stream->opaque = Z_NULL; stream->avail_in = 0; stream->avail_out = 0; stream->next_in = Z_NULL; /* * Seek to the required location in the compressed * data stream. If the provided index point is NULL, * we start from the beginning of the file. * * The compressed offset for index points correspond * to the first full byte of compressed data. So if * the index point is not byte-aligned (bits > 0), we * need to seek to the previous byte, and tell zlib * about it (via the inflatePrime call below). */ if (point == NULL) seek_loc = 0; else seek_loc = point->cmp_offset - (point->bits > 0); if (FSEEK(index->fd, seek_loc, SEEK_SET) != 0) goto fail; /* * If we're starting from the beginning * of the file, we tell inflateInit2 to * expect a file header */ if (point == NULL) { zran_log("zlib_init_zlib_inflate(0, n/a, n/a, %u + 32)\n", windowBits); if (inflateInit2(stream, windowBits + 32) != Z_OK) { goto fail; } } /* * Otherwise, we configure for raw inflation, * and initialise the inflation dictionary * from the uncompressed data associated with * the index point. */ else { zran_log("_zran_init_zlib_inflate(%lld, %llu, %llu, -%u)\n", seek_loc, point->cmp_offset, point->uncmp_offset, windowBits); if (inflateInit2(stream, -windowBits) != Z_OK) { goto fail; } /* The starting index point is not * byte-aligned, so we'll insert * the initial bits into the inflate * stream using inflatePrime */ if (point->bits > 0) { ret = getc(index->fd); if (ret == -1) goto fail; if (inflatePrime(stream, point->bits, ret >> (8 - point->bits)) != Z_OK) goto fail; } /* * Initialise the inflate stream * with the index point data. */ if (point->data != NULL) { if (inflateSetDictionary(stream, point->data, index->window_size) != Z_OK) goto fail; } } return 0; fail: return -1; } /* * Identify the location of the next compressed stream (if the file * contains concatenated streams). */ int _zran_find_next_stream(zran_index_t *index, z_stream *stream) { /* * Search for the beginning of * the next stream. GZIP files * start with 0x1f8b. */ int offset = 0; int found = 0; while (stream->avail_in >= 2) { if (stream->next_in[0] == 0x1f && stream->next_in[1] == 0x8b) { found = 1; break; } offset += 2; stream->next_in += 2; stream->avail_in -= 2; } /* * No header found for * the next stream. */ if (found == 0) goto not_found; zran_log("New stream found, re-initialising inflation\n"); /* * Re-configure for inflation * from the new stream. */ if (inflateEnd(stream) != Z_OK) goto fail; stream->zalloc = Z_NULL; stream->zfree = Z_NULL; stream->opaque = Z_NULL; if (inflateInit2(stream, index->log_window_size + 32) != Z_OK) goto fail; return offset; fail: return ZRAN_FIND_STREAM_ERROR; not_found: return ZRAN_FIND_STREAM_NOT_FOUND; } /* The workhorse. Inflate/decompress data from the file. */ static int _zran_inflate(zran_index_t *index, z_stream *strm, uint64_t offset, uint16_t flags, uint32_t *total_consumed, uint32_t *total_output, uint32_t len, uint8_t *data) { /* * Used to store and check return * values. f_ret is for fread, * z_ret is for zlib/zran functions. * return_val is the return value for * this function. */ size_t f_ret; int z_ret; int return_val = ZRAN_INFLATE_OK; /* * Offsets into the compressed * and uncompressed data streams, * and total number of bytes * decompressed and output. */ uint64_t cmp_offset; uint64_t uncmp_offset; uint32_t _total_consumed = 0; uint32_t _total_output = 0; /* * Index point to start from * (if ZRAN_INFLATE_USE_OFFSET * is active). */ zran_point_t *start = NULL; /* * If ZRAN_INFLATE_INIT_READBUF is not set, * make sure that a read buffer exists. * * If the opposite is true, the read buffer * from a prior call has not been cleaned up. */ if ((!inflate_init_readbuf(flags) && index->readbuf == NULL) || ( inflate_init_readbuf(flags) && index->readbuf != NULL)) { goto fail; } /* * It begins... */ zran_log("_zran_inflate(%llu, block=%u, use_offset=%u, init_stream=%u,\n" " free_stream=%u, init_readbuf=%u, free_readbuf=%u,\n" " clear_offsets=%u, nbytes=%u)\n", offset, inflate_stop_at_block( flags), inflate_use_offset( flags), inflate_init_stream( flags), inflate_free_stream( flags), inflate_init_readbuf( flags), inflate_free_readbuf( flags), inflate_clear_readbuf_offsets(flags), len); /* * The compressed/uncompressed offsets are initialised in * one of three ways. If ZRAN_INFLATE_USE_OFFSET is active, * they are either: * * - Both set to 0 * * - Initialised according to an existing index * point that preceeds the requested offset. * * Otherwise, they are initialised from index->inflate_cmp_offset * and index->inflate_uncmp_offset, which are assumed to have been * set in a prior call to _zran_inflate. */ if (inflate_use_offset(flags)) { cmp_offset = 0; uncmp_offset = 0; /* * If a non-zero offset has been specified, * search the index to see if we can start * inflating from a known location. */ if (offset > 0) { /* * In order to successfully decompress * data from the current uncompressed seek * location, we need to start decompressing * from the index point which preceeds it. */ z_ret = _zran_get_point_at(index, offset, 1, &start); if (z_ret == ZRAN_GET_POINT_NOT_COVERED) return ZRAN_INFLATE_NOT_COVERED; if (z_ret == ZRAN_GET_POINT_EOF) return ZRAN_INFLATE_EOF; } /* * Start inflating from the index point * corresponding to the offset (or keep * the offsets at 0 if no point was found). */ if (start != NULL) { cmp_offset = start->cmp_offset; uncmp_offset = start->uncmp_offset; } } /* * If ZRAN_INFLATE_USE_OFFSET is not active, * we initialise from offsets which were * stored on the last call to _zran_inflate. */ else { cmp_offset = index->inflate_cmp_offset; uncmp_offset = index->inflate_uncmp_offset; } zran_log("initialising to inflate from c=%llu, u=%llu\n", cmp_offset, uncmp_offset); /* * If ZRAN_INFLATE_INIT_Z_STREAM is active, * initialise the zlib struct for inflation. * The _zran_init_zlib_inflate function * seeks to the correct location in the file * for us. * * If ZRAN_INFLATE_INIT_Z_STREAM is not * active, we assume that the file is * already at the correct spot. */ if (inflate_init_stream(flags)) { if (_zran_init_zlib_inflate(index, strm, start) != 0) { goto fail; } } /* * If ZRAN_INFLATE_INIT_READBUF, * allocate memory for reading * compressed data from the file. * The buffer is attached to the * zran_index_t->readbuf pointer. */ if (inflate_init_readbuf(flags)) { index->readbuf = calloc(1, index->readbuf_size); if (index->readbuf == NULL) goto fail; } /* * If ZRAN_INFLATE_CLEAR_READBUF_OFFSETS, * we clear any stored information about * the read buffer, and start reading * from/writing to it from the beginning. */ if (inflate_clear_readbuf_offsets(flags)) { index->readbuf_offset = 0; index->readbuf_end = 0; } /* * Otherwise, assume that there is already * some input (compressed) data in the * readbuf, and that index->readbuf_offset * and index->readbuf_end were sert on a * prior call. * * - readbuf_offset tells us where in * readbuf the data starts * * - readbuf_end tells us where it ends. */ else { strm->next_in = index->readbuf + index->readbuf_offset; strm->avail_in = index->readbuf_end - index->readbuf_offset; } /* * Tell zlib where to store * the uncompressed data. */ strm->avail_out = len; strm->next_out = data; /* * Keep going until we run out of space. */ while (strm->avail_out > 0) { /* * We need to read in more data. We read in more * data when strm->avail_in < 2, because a GZIP * header is 2 bytes long, and when searching for * the next stream in a sequence of concatenated * streams, the _zran_find_next_stream function * might leave a byte in the input without * finding a new stream. */ if (strm->avail_in < 2) { if (feof(index->fd)) { return_val = ZRAN_INFLATE_EOF; break; } /* * If there are any unprocessed bytes * left over, put them at the beginning * of the read buffer */ if (strm->avail_in > 0) { memcpy(index->readbuf, strm->next_in, strm->avail_in); } zran_log("Reading from file %llu [ == %llu?]\n", FTELL(index->fd), cmp_offset); /* * Read a block of compressed data * (offsetting past any left over * bytes that we may have copied to * the beginning of the read buffer * above). */ f_ret = fread(index->readbuf + strm->avail_in, 1, index->readbuf_size - strm->avail_in, index->fd); if (ferror(index->fd)) goto fail; if (f_ret == 0) goto fail; zran_log("Read %lu bytes from file [c=%llu, u=%llu]\n", f_ret, cmp_offset, uncmp_offset); /* * Tell zlib about the block * of compressed data that we * just read in. */ index->readbuf_end = f_ret + strm->avail_in; strm->avail_in += f_ret; strm->next_in = index->readbuf; } /* * Decompress the block until it is * gone (or we've read enough bytes) */ z_ret = Z_OK; while (strm->avail_in > 0) { /* * Re-initialise inflation if we have * hit a new compressed stream. */ if (z_ret == Z_STREAM_END) { zran_log("End of stream - searching for another stream\n"); z_ret = _zran_find_next_stream(index, strm); /* * If _zran_find_next_stream can't find * a new stream, we are either out of * compressed input data, or at eof. In * either case, break and let the outer * loop deal with it. */ if (z_ret == ZRAN_FIND_STREAM_NOT_FOUND) break; else if (z_ret == ZRAN_FIND_STREAM_ERROR) goto fail; /* * _zran_find_next_stream has found a * new stream, and has told us how many * bytes it skipped over before finding * it. */ cmp_offset += z_ret; _total_consumed += z_ret; } /* * Optimistically update offsets - * we will adjust them after the * inflate call. */ cmp_offset += strm->avail_in; uncmp_offset += strm->avail_out; _total_consumed += strm->avail_in; _total_output += strm->avail_out; zran_log("Before inflate - avail_in=%u, avail_out=%u\n", strm->avail_in, strm->avail_out); /* * Inflate the block - the decompressed * data is output straight to the provided * data buffer. * * If ZRAN_INFLATE_STOP_AT_BLOCK is active, * Z_BLOCK tells inflate to stop inflating * at a compression block boundary. Otherwise, * inflate stops when it comes to the end of a * stream, or it runs out of input or output. */ if (inflate_stop_at_block(flags)) z_ret = inflate(strm, Z_BLOCK); else z_ret = inflate(strm, Z_NO_FLUSH); zran_log("After inflate - avail_in=%u, avail_out=%u\n", strm->avail_in, strm->avail_out); /* * Adjust our offsets according to what * was actually consumed/decompressed. */ cmp_offset -= strm->avail_in; uncmp_offset -= strm->avail_out; _total_consumed -= strm->avail_in; _total_output -= strm->avail_out; /* * Now we need to figure out what just happened. * * Z_BUF_ERROR indicates that the output buffer * is full; we clobber it though, as it makes the * code below a bit easier (and anyway, we can * tell if the output buffer is full by checking * strm->avail_out). */ if (z_ret == Z_BUF_ERROR) { z_ret = Z_OK; } /* * If z_ret is not Z_STREAM_END or * Z_OK, something has gone wrong. * * If the file comprises a sequence of * concatenated gzip streams, we will * encounter Z_STREAM_END before the end * of the file (where one stream ends and * the other begins). * * If at a new stream, we re-initialise * inflation on the next loop iteration. */ if (z_ret != Z_OK && z_ret != Z_STREAM_END) { zran_log("zlib inflate failed (code: %i, msg: %s)\n", z_ret, strm->msg); goto fail; } /* * End of a block? If INFLATE_STOP_AT_BLOCK * is active, we want to stop at a compression * block boundary. * * If we used Z_BLOCK above, and inflate * encountered a block boundary, it indicates * this in the the strm->data_type field. */ if (inflate_stop_at_block(flags) && ((strm->data_type & 128) && !(strm->data_type & 64))) { zran_log("At block or stream boundary, " "stopping inflation\n"); return_val = ZRAN_INFLATE_BLOCK_BOUNDARY; break; } /* * We've run out of space to * store decompressed data */ if (strm->avail_out == 0) { zran_log("Output buffer full - stopping inflation\n"); /* * We return OUTPUT_FULL if we haven't * decompressed the requested number of * bytes, or ZRAN_INFLATE_STOP_AT_BLOCK * is active and we haven't yet found a * block. */ if (inflate_stop_at_block(flags) || _total_output < len) { return_val = ZRAN_INFLATE_OUTPUT_FULL; } break; } /* * End of file. The GZIP file * footer takes up 8 bytes, which * do not get processed by the * inflate function. */ if (feof(index->fd) && strm->avail_in <= 8) { zran_log("End of file, stopping inflation\n"); return_val = ZRAN_INFLATE_EOF; /* * We now know how big the * uncompressed data is. */ if (index->uncompressed_size == 0) { zran_log("Updating uncompressed data " "size: %llu\n", uncmp_offset); index->uncompressed_size = uncmp_offset; } break; } /* * Some of the code above has decided that * it wants this _zran_inflate call to return. */ if (return_val != ZRAN_INFLATE_OK) { break; } } if (return_val != ZRAN_INFLATE_OK) { break; } } /* * If ZRAN_INFLATE_FREE_READBUF is * active, clear input buffer memory * and offsets. */ if (inflate_free_readbuf(flags)) { free(index->readbuf); index->readbuf = NULL; index->readbuf_offset = 0; index->readbuf_end = 0; } /* * Otherwise save the readbuf * offset for next time. */ else { index->readbuf_offset = index->readbuf_end - strm->avail_in; } /* * If ZRAN_INFLATE_FREE_Z_STREAM * is active, do just that. */ if (inflate_free_stream(flags)) { if (inflateEnd(strm) != Z_OK) goto fail; } /* * Update the total number of * bytes that were consumed/read */ *total_consumed = _total_consumed; *total_output = _total_output; /* * Update the compressed/uncompressed * offsets in case we need to use them * later. */ index->inflate_cmp_offset = cmp_offset; index->inflate_uncmp_offset = uncmp_offset; zran_log("Inflate finished - consumed=%u, output=%u,\n" " cmp_offset=%llu, uncmp_offset=%llu \n\n", *total_consumed, *total_output, cmp_offset, uncmp_offset); /* Phew. */ return return_val; fail: if (index->readbuf != NULL) { free(index->readbuf); index->readbuf = NULL; index->readbuf_offset = 0; index->readbuf_end = 0; } return ZRAN_INFLATE_ERROR; } /* * Expands the index to encompass the * compressed offset specified by 'until'. */ int _zran_expand_index(zran_index_t *index, uint64_t until) { /* * Used to store and check return values * from zlib and zran functions. */ int z_ret; /* Zlib stream struct */ z_stream strm; /* * Number of bytes read/decompressed * on each call to _zran_inflate. */ uint32_t bytes_consumed; uint32_t bytes_output; /* * Buffer to store uncompressed data, * size of said buffer, and current offset * into said buffef. We wrap the buffer * around to the beginning when it is * filled. * * Ideally, we only want to decompress * index->spacing bytes before creating a * new index point. But we may have to * decompress more than this before a * suitable location (a block/stream * boundary) is found, so we allocate * more space. */ uint8_t *data = NULL; uint32_t data_size = index->spacing * 4; uint32_t data_offset = 0; /* * _zran_inflate control flags. We need * to use different flags on the first * call - first_inflate is used to track * this. */ uint16_t inflate_flags; uint8_t first_inflate = 1; /* * Counters to keep track of where we are * in both the compressed and uncompressed * streams. */ uint64_t cmp_offset; uint64_t uncmp_offset; uint64_t last_uncmp_offset; /* * start is a reference to the last * point in the index when this function * is called. This is where we need * to start decompressing data from * before we can add more index points. * * last_created is a reference to the * most recent point that was added * to the index in this call to * _zran_expand_index. */ zran_point_t *start = NULL; zran_point_t *last_created = NULL; /* * In order to create a new index * point, we need to start reading * at the last index point, so that * we read enough data to initialise * the inflation. If we don't have * at least two points, we start * at the beginning of the file. */ start = NULL; if (index->npoints > 1) { start = &(index->list[index->npoints - 1]); /* * The index already covers the requested * offset. Nothing needs to be done. */ if (until <= start->cmp_offset) return 0; } /* * Allocate memory for the * uncompressed data buffer. */ data = calloc(1, data_size); if (data == NULL) goto fail; /* Let's do this. */ zran_log("_zran_expand_index(%llu)\n", until); /* * If the caller passed until == 0, * we force some data to be read. */ if (until == 0) { until = index->spacing; } /* * We start from the last point in * the index, or the beginning of * the file, if there are not enough * points in the index. */ if (start != NULL) { cmp_offset = start->cmp_offset; uncmp_offset = start->uncmp_offset; last_uncmp_offset = uncmp_offset; } else { cmp_offset = 0; uncmp_offset = 0; last_uncmp_offset = 0; } /* * Don't finish until we're at the end of the * file, or we've expanded the index past * the requested offset (and have created at * least one new index point - * last_created == NULL tells us whether a * point has been created). */ while ((cmp_offset < index->compressed_size) && (last_created == NULL || last_created->cmp_offset < until)) { /* * On the first call to _zran_inflate, we * tell it to initialise the zlib stream * struct, create a read buffer, and start * inflation from our starting point. */ if (first_inflate) { first_inflate = 0; inflate_flags = (ZRAN_INFLATE_INIT_Z_STREAM | ZRAN_INFLATE_INIT_READBUF | ZRAN_INFLATE_USE_OFFSET | ZRAN_INFLATE_CLEAR_READBUF_OFFSETS | ZRAN_INFLATE_STOP_AT_BLOCK); } /* * On subsequent calls, we tell _zran_inflate * to just continue where it left off on the * previous call. */ else { inflate_flags = ZRAN_INFLATE_STOP_AT_BLOCK; } zran_log("Searching for next block boundary\n" " c=%llu, u=%llu,\n" " data_offset=%u, data_space=%u\n", cmp_offset, uncmp_offset, data_offset, data_size - data_offset); /* * We wrap the data buffer around to its * beginning by using some trickery with * the data_offset. By doing this, the * _zran_add_point function will be able * to retrieve the data associated with * an index point even if some of it it * is contained at the end of the data * buffer, and the rest at the beginning. */ z_ret = _zran_inflate(index, &strm, cmp_offset, inflate_flags, &bytes_consumed, &bytes_output, data_size - data_offset, data + data_offset); cmp_offset += bytes_consumed; uncmp_offset += bytes_output; data_offset = (data_offset + bytes_output) % data_size; /* * Has the output buffer been filled? * If so, we just continue - the * data_offset trickery means that we * can ask the _zran_inflate function * to just keep filling the buffer * until we find a block. */ if (z_ret == ZRAN_INFLATE_OUTPUT_FULL) continue; /* * If z_ret != ZRAN_INFLATE_EOF or * ZRAN_INFLATE_BLOCK_BOUNDARY, * something has gone wrong. */ else if (z_ret != ZRAN_INFLATE_EOF && z_ret != ZRAN_INFLATE_BLOCK_BOUNDARY) { goto fail; } /* * If we're at the beginning of the file * (uncmp_offset == 0), or at the end of * the file (z_ret == ZRAN_INFLATE_EOF), * or at a compress block boundary, * and index->spacing bytes have passed * since the last index point that was * created, we'll create a new index * point at this location. */ if (z_ret == ZRAN_INFLATE_EOF || uncmp_offset == 0 || uncmp_offset - last_uncmp_offset >= index->spacing) { // TODO If at start or EOF, you should // pass in NULL for the window // data. You can then clean up // _zran_add_point a little bit. if (_zran_add_point(index, strm.data_type & 7, cmp_offset, uncmp_offset, data_offset, data_size, data) != 0) { goto fail; } last_created = &index->list[index->npoints - 1]; last_uncmp_offset = uncmp_offset; } /* And if at EOF, we are done. */ if (z_ret == ZRAN_INFLATE_EOF) break; } /* * A final call to _zran_inflate, to clean * up read buffer and z_stream memory. */ z_ret = _zran_inflate(index, &strm, 0, (ZRAN_INFLATE_CLEAR_READBUF_OFFSETS | ZRAN_INFLATE_FREE_Z_STREAM | ZRAN_INFLATE_FREE_READBUF), &bytes_consumed, &bytes_output, 0, data); if (z_ret != ZRAN_INFLATE_OK && z_ret != ZRAN_INFLATE_EOF) { goto fail; } /* * The index may have over-allocated * space for storing index points, so * here we free the unused memory. */ if (_zran_free_unused(index) != 0) { goto fail; } zran_log("Expansion finished (cmp_offset=%llu, last_created=%llu)\n", cmp_offset, last_created->cmp_offset); free(data); return 0; fail: free(data); return -1; } /* * Seek to the approximate location of the specified offset into * the uncompressed data stream. The whence argument must be * SEEK_SET or SEEK_CUR. */ int zran_seek(zran_index_t *index, int64_t offset, uint8_t whence, zran_point_t **point) { int result; zran_point_t *seek_point; zran_log("zran_seek(%lld, %i)\n", offset, whence); if (whence != SEEK_SET && whence != SEEK_CUR) { goto fail; } /* * SEEK_CUR: seek relative to * the current file position. */ if (whence == SEEK_CUR) { offset += index->uncmp_seek_offset; } /* Bad input */ if (offset < 0) { goto fail; } /* * Get the index point that * corresponds to this offset. */ result = _zran_get_point_with_expand(index, offset, 0, &seek_point); if (result == ZRAN_GET_POINT_FAIL) goto fail; if (result == ZRAN_GET_POINT_NOT_COVERED) goto not_covered; if (result == ZRAN_GET_POINT_EOF) goto eof; index->uncmp_seek_offset = offset; offset = seek_point->cmp_offset; /* * This index point is not byte-aligned. * Adjust the offset accordingly. */ if (seek_point->bits > 0) offset -= 1; /* * The caller wants a ref to the * index point corresponding to * the seek location. */ if (point != NULL) { *point = seek_point; } if (FSEEK(index->fd, offset, SEEK_SET) != 0) goto fail; return ZRAN_SEEK_OK; fail: return ZRAN_SEEK_FAIL; not_covered: return ZRAN_SEEK_NOT_COVERED; eof: index->uncmp_seek_offset = index->uncompressed_size; return ZRAN_SEEK_EOF; } /* Return the current seek position in the uncompressed data stream. */ long zran_tell(zran_index_t *index) { return index->uncmp_seek_offset; } /* Read len bytes from the uncompressed data stream, storing them in buf. */ int64_t zran_read(zran_index_t *index, void *buf, uint64_t len) { /* Used to store/check return values. */ int ret; /* * Number of bytes we try to read, and * number of bytes actually read/output * on each call to _zran_inflate. */ uint64_t bytes_to_read; uint32_t bytes_consumed; uint32_t bytes_output; /* * _zran_inflate control flags. We need * to pass different flags on thefirst * call to _zran_inflate. */ uint16_t inflate_flags; uint8_t first_inflate = 1; /* * Counters keeping track of the current * location in both the compressed and * uncompressed streams, and the total * number of bytes read. */ uint64_t uncmp_offset; uint64_t cmp_offset; uint64_t total_read; /* * Zlib stream struct and starting * index point for the read.. */ z_stream strm; zran_point_t *start = NULL; /* * Memory used to store bytes that we skip * over before reaching the appropriate * point in the uncompressed data stream. * * to_discard is used to store the number of * bytes that we want to discard on a single * call to _zran_inflate (which is limited by * the discard buffer size). * * total_discarded keeps track of the total * number of bytes discarded so far. * * discard_size is the size of the discard * * buffer. Ideally we will only have to * decompress (on average) spacing / 2 bytes * before reaching the seek location, but this * isn't a guarantee, so we allocate more to * reduce the number of reads that are required. */ uint8_t *discard = NULL; uint64_t to_discard = 0; uint64_t total_discarded = 0; uint64_t discard_size = index->spacing * 4; if (len == 0) return 0; if (len > INT64_MAX) goto fail; zran_log("zran_read(%llu)\n", len); /* * Search for the index point that * corresponds to our current seek * location in the uncompressed * data stream. */ ret = _zran_get_point_with_expand(index, index->uncmp_seek_offset, 0, &start); if (ret == ZRAN_GET_POINT_EOF) goto eof; if (ret == ZRAN_GET_POINT_NOT_COVERED) goto not_covered; /* * We have to start decompressing from * the index point that preceeds the seek * location, so we need to skip over bytes * until we get to that location. We use * the discard buffer to store those bytes. */ discard = malloc(discard_size); if (discard == NULL) { goto fail; } /* * Inflate and discard data until we * reach the current seek location * into the uncompresesd data stream. */ cmp_offset = start->cmp_offset; uncmp_offset = start->uncmp_offset; first_inflate = 1; total_discarded = 0; while (uncmp_offset < index->uncmp_seek_offset) { /* * On the first call to _zran_inflate, * we tell it to initialise the z_stream, * and create a read buffer. */ if (first_inflate) { first_inflate = 0; inflate_flags = (ZRAN_INFLATE_INIT_Z_STREAM | ZRAN_INFLATE_INIT_READBUF | ZRAN_INFLATE_CLEAR_READBUF_OFFSETS | ZRAN_INFLATE_USE_OFFSET); } /* * On subsequent calls, we just tell * _zran_inflate to continue where * it left off. */ else { inflate_flags = 0; } /* * Don't read past the uncompressed seek * location - at this point, we will need * to stop discarding bytes, and start * fulfilling the read request. */ to_discard = index->uncmp_seek_offset - uncmp_offset; if (to_discard > discard_size) to_discard = discard_size; zran_log("Discarding %llu bytes (%llu < %llu)\n", to_discard, uncmp_offset, index->uncmp_seek_offset); ret = _zran_inflate(index, &strm, cmp_offset, inflate_flags, &bytes_consumed, &bytes_output, to_discard, discard); /* * _zran_inflate should return 0 if * it runs out of output space (which * is ok), or it has read enough bytes * (which is perfect). Any other * return code means that something * has gone wrong. */ if (ret != ZRAN_INFLATE_OUTPUT_FULL && ret != ZRAN_INFLATE_EOF && ret != ZRAN_INFLATE_OK) goto fail; cmp_offset += bytes_consumed; uncmp_offset += bytes_output; total_discarded += bytes_output; } /* * Sanity check - we should be at the * correct location in the uncompressed * stream. * * TODO What happens here if we are at EOF? */ if (uncmp_offset != index->uncmp_seek_offset) goto fail; zran_log("Discarded %llu bytes, ready to " "read from %llu (== %llu)\n", total_discarded, uncmp_offset, index->uncmp_seek_offset); /* * At this point, we are ready to inflate * from the uncompressed seek location. */ total_read = 0; while (total_read < len) { /* * If we started at the correct location, * the discard loop above will not have * executed, and _zran_inflate will not * have initialised itself. So we repeat * the flag control stuff here. */ if (first_inflate) { first_inflate = 0; inflate_flags = (ZRAN_INFLATE_INIT_Z_STREAM | ZRAN_INFLATE_INIT_READBUF | ZRAN_INFLATE_CLEAR_READBUF_OFFSETS | ZRAN_INFLATE_USE_OFFSET); } else { inflate_flags = 0; } /* * _zran_inflate only allows us to * read max(uint32_t) at a time. If * len is greater than this, we need * to split it into multiple calls. */ bytes_to_read = len - total_read; if (bytes_to_read > 4294967295) { bytes_to_read = 4294967295; } ret = _zran_inflate(index, &strm, cmp_offset, inflate_flags, &bytes_consumed, &bytes_output, bytes_to_read, (uint8_t *)(buf) + total_read); cmp_offset += bytes_consumed; uncmp_offset += bytes_output; total_read += bytes_output; if (ret == ZRAN_INFLATE_EOF) break; else if (ret == ZRAN_INFLATE_OUTPUT_FULL) { /* * We might be reading 2**32 sized chunks * of data on each call to _zran_inflate. */ if (bytes_to_read == len) { break; } } else if (ret != ZRAN_INFLATE_OK) goto fail; zran_log("Read %u bytes (%llu / %llu)\n", bytes_output, total_read, len); } /* * A final call to _zran_inflate, * to clean up memory */ ret = _zran_inflate(index, &strm, 0, (ZRAN_INFLATE_CLEAR_READBUF_OFFSETS | ZRAN_INFLATE_FREE_Z_STREAM | ZRAN_INFLATE_FREE_READBUF), &bytes_consumed, &bytes_output, 0, discard); if (ret != ZRAN_INFLATE_OK && ret != ZRAN_INFLATE_EOF) { goto fail; } /* * Update the current uncompressed * seek position. */ index->uncmp_seek_offset += total_read; zran_log("Read succeeded - %llu bytes read [compressed offset: %ld]\n", total_read, FTELL(index->fd)); free(discard); return total_read; not_covered: return ZRAN_READ_NOT_COVERED; eof: return ZRAN_READ_EOF; fail: if (discard != NULL) free(discard); return ZRAN_READ_FAIL; } indexed-gzip-0.6.1/indexed_gzip/zran.h000066400000000000000000000174711317257227400177250ustar00rootroot00000000000000#ifndef __ZRAN_H__ #define __ZRAN_H__ /* * The zran module is an adaptation of the zran example, written by Mark * Alder, which ships with the zlib source code. It allows the creation * of an index into a compressed file, which is used to improve the speed * of random seek/read access to the uncompressed data. * * Author: Paul McCarthy */ #include #include struct _zran_index; struct _zran_point; typedef struct _zran_index zran_index_t; typedef struct _zran_point zran_point_t; /* * These values may be passed in as flags to the zran_init function. * They are specified as bit-masks, rather than bit locations. */ enum { ZRAN_AUTO_BUILD = 1, }; /* * Struct representing the index. None of the fields in this struct * should ever need to be accessed or modified directly. */ struct _zran_index { /* * Handle to the compressed file. */ FILE *fd; /* * Size of the compressed file. This * is calculated in zran_init. */ size_t compressed_size; /* * Size of the uncompressed data. This is * only updated when it becomes known. */ size_t uncompressed_size; /* * Spacing size in bytes, relative to the compressed * data stream, between adjacent index points */ uint32_t spacing; /* * Number of bytes of uncompressed data to store * for each index point. This must be a minimum * of 32768 bytes. */ uint32_t window_size; /* * Base2 logarithm of the window size - it * is needed to initialise zlib inflation. */ uint32_t log_window_size; /* * Size, in bytes, of buffer used to store * compressed data read from disk. */ uint32_t readbuf_size; /* * Number of index points that have been created. */ uint32_t npoints; /* * Number of index points that can be stored. */ uint32_t size; /* * List of index points. */ zran_point_t *list; /* * Most recently requested seek/read * location into the uncompressed data * stream - this is used to keep track * of where the calling code thinks it * is in the (uncompressed) file. */ uint64_t uncmp_seek_offset; /* * Flags passed to zran_init */ uint16_t flags; /* * All of the fields after this point are used * by the internal _zran_inflate function. */ /* * Reference to a file input * buffer of size readbuf_size. */ uint8_t *readbuf; /* * An offset into readbuf. */ uint32_t readbuf_offset; /* * The current end of the readbuf contents. */ uint32_t readbuf_end; /* * Current offsets into the uncompressed and * compressed data streams. */ uint64_t inflate_cmp_offset; uint64_t inflate_uncmp_offset; }; /* * Struct representing a single seek point in the index. */ struct _zran_point { /* * Location of this point in the compressed data * stream. This is the location of the first full * byte of compressed data - if the compressed * and uncompressed locations are not byte-aligned, * the bits field below specifies the bit offset. */ uint64_t cmp_offset; /* * Corresponding location of this point * in the uncompressed data stream. */ uint64_t uncmp_offset; /* * If this point is not byte-aligned, this specifies * the number of bits, in the compressed stream, * back from cmp_offset, that the uncompressed data * starts. */ uint8_t bits; /* * Chunk of uncompressed data preceeding this point. * This is required to initialise decompression from * this point onwards. */ uint8_t *data; }; /* * Initialise a zran_index_t struct for use with the given file. * * Passing in 0 for the spacing, window_size and readbuf_size arguments * will result in the follwoing values being used: * * spacing: 1048576 * window_size: 32768 * readbuf_size: 16384 * * The flags argument is a bit mask used to control the following options: * * ZRAN_AUTO_BUILD: Build the index automatically on demand. */ int zran_init( zran_index_t *index, /* The index */ FILE *fd, /* Open handle to the compressed file */ uint32_t spacing, /* Distance in bytes between index seek points */ uint32_t window_size, /* Number of uncompressed bytes to store with each point */ uint32_t readbuf_size, /* Number of bytes to read at a time */ uint16_t flags /* Flags controlling index behaviour */ ); /* * Frees the memory use by the given index. The zran_index_t struct * itself is not freed. */ void zran_free( zran_index_t *index /* The index */ ); /* * (Re-)Builds the index to cover the given range, which must be * specified relative to the compressed data stream. Pass in 0 * for both offsets to re-build the full index. * * Returns 0 on success, non-0 on failure. */ int zran_build_index( zran_index_t *index, /* The index */ uint64_t from, /* Build the index from this point */ uint64_t until /* Build the index to this point */ ); /* Return codes for zran_seek. */ enum { ZRAN_SEEK_FAIL = -1, ZRAN_SEEK_OK = 0, ZRAN_SEEK_NOT_COVERED = 1, ZRAN_SEEK_EOF = 2, }; /* * Seek to the specified offset in the uncompressed data stream. * If the index does not currently cover the offset, and it was * created with the ZRAN_AUTO_BUILD flag, the index is expanded * to cover the offset. * * Seeking from the end of the uncompressed stream is not supported * - you may only seek from the beginning of the file, or from the * current seek location. In other words, the whence argument must * be equal to SEEK_SET or SEEK_CUR. * * Returns: * - ZRAN_SEEK_OK for success. * * - ZRAN_SEEK_FAIL to indicate failure of some sort. * * - ZRAN_SEEK_NOT_COVERED to indicate that the index does not * cover the requested offset (will never happen if * ZRAN_AUTO_BUILD is active). * - ZRAN_SEEK_EOF to indicate that the requested offset * is past the end of the uncompressed stream. */ int zran_seek( zran_index_t *index, /* The index */ int64_t offset, /* Uncompressed offset to seek to */ uint8_t whence, /* Must be SEEK_SET or SEEK_CUR */ zran_point_t **point /* Optional place to store corresponding zran_point_t */ ); /* * Returns the current seek location in the uncompressed data stream * (just returns zran_index_t.uncmp_seek_offset). */ long zran_tell( zran_index_t *index /* The index */ ); /* Return codes for zran_read. */ enum { ZRAN_READ_NOT_COVERED = -1, ZRAN_READ_EOF = -2, ZRAN_READ_FAIL = -3 }; /* * Read len bytes from the current location in the uncompressed * data stream, storing them in buf. If the index was created with * the ZRAN_AUTO_BUILD flag, it is expanded as needed. * * Returns: * - Number of bytes read for success. * * - ZRAN_READ_NOT_COVERED to indicate that the index does not * cover the requested region (will never happen if * ZRAN_AUTO_BUILD is active). * * - ZRAN_READ_EOF to indicate that the read could not be completed * because the current uncompressed seek point is at EOF. * * - ZRAN_READ_FAIL to indicate that the read failed for some reason. */ int64_t zran_read( zran_index_t *index, /* The index */ void *buf, /* Buffer to store len bytes */ uint64_t len /* Number of bytes to read */ ); #endif /* __ZRAN_H__ */ indexed-gzip-0.6.1/indexed_gzip/zran.pxd000066400000000000000000000026461317257227400202670ustar00rootroot00000000000000# # Cython declaration for the zran library. # # Author: Paul McCarthy # from libc.stdio cimport FILE from libc.stdint cimport uint8_t, uint16_t, uint32_t, uint64_t, int64_t from posix.types cimport off_t cdef extern from "zran.h": ctypedef struct zran_index_t: pass ctypedef struct zran_point_t: pass enum: ZRAN_AUTO_BUILD = 1, ZRAN_SEEK_FAIL = -1, ZRAN_SEEK_OK = 0, ZRAN_SEEK_NOT_COVERED = 1, ZRAN_SEEK_EOF = 2, ZRAN_READ_NOT_COVERED = -1, ZRAN_READ_EOF = -2, ZRAN_READ_FAIL = -3, bint zran_init(zran_index_t *index, FILE *fd, uint32_t spacing, uint32_t window_size, uint32_t readbuf_size, uint16_t flags) void zran_free(zran_index_t *index) bint zran_build_index(zran_index_t *index, uint64_t from_, uint64_t until) nogil; long zran_tell(zran_index_t *index); int zran_seek(zran_index_t *index, int64_t offset, uint8_t whence, zran_point_t **point) nogil; int64_t zran_read(zran_index_t *index, void *buf, uint64_t len) nogil; indexed-gzip-0.6.1/performance.png000066400000000000000000000727521317257227400171430ustar00rootroot00000000000000PNG  IHDRk;bKGD pHYsaa?itIME0 IDATx}e?0b(H&ieiYi*f`4T,k_ͱ~rˇ"X2KW{0Wqd1])PCDAFf9g83 3a8羮ou>^םsI^?~ kߗ IKT I`X``|hOmB(&I} \%:Iޛd!Kl+MI~^װ/Ueߗm?` (.4%À_y]um:1m|1oO,춅vNLro瓼$%١}$.R]{$^<~oO'$J2h okIￚ$'K ZIv"$C\IIrxkI$kIv`ǽZX>X)mڬ-\>}7>IF?ϥ?2ɯRx$%9$$sdF+I^(-\ع8OaUH.WXR"a3$9\ ۺp9~c A}6Ͱ(~>G¦l cKu֦〩X*ܔB=)tkOc[>O;+64%~ߘ❅ ӽftu}kåwgE?cUpf $3ɿgShp\}O/IRd“{Oe.BKiӐ$o([ rvNaF]S|mIgʕB#kSuz sOZ>NaGRR@r[ v/۶C֘BN$IaS!c՞ҵ~x=ƔL!j.Q)i/I%\hLa@ԓKK,LGSAy^Y61%>B(Q.۾.oM/j7icH Kj|}p ?s;{mڞ_vC)] ),iV9S vHo aGyokLrj;UٴtewN!jL3a%yXm..;[¥lvKSc)/t>캴־4ӫ1{۩3|L7fAmL2X#-Vzc|(!g40{zmt`>YVVQiY;Cv. I!0jLNxc6/G)+mhqk'uF'u)X;eUI>,՘Bh]l/캤Θc}Pz.d[ Ľl/{vOWuX㶲{|ֲ6#ɷRQ|2Z8~Oο7vUB4okxOvR!ogӳJ!ԞBll}9/k;)؍:mgX̦08eI0?Ս/v>ŚlcdϨvUe1m8kWiZs ϦT t~K7vNcZI)wߧ1#x?>\.7zp bC7?[=wISilz.}IV_3*qo(a[Kܓ_փv줬&ff,/^qΦPhl'VK+бKm>IքKlJ_=$G3}oI2ye￙2ݟHaI|sՕ*vίݪt^6NnsmIkwl 3lJ?}%ٯB~x|B K!(TO;;D0sy1j,^ﶪ_S>#nl5-X6B+ֽ듼)/=Gi<y{eq`zVGG;lq\T$mEuZkI$?ۿ*ɏ3I=n {{=<ߛ_g_fS Vf%=B{J)ɜ>*wH2=:L.5Pg£$O'9oKaUS :GxnMtC//&9!iOD1ZZ/k޸>;%y I>ddY.qC .@ZK)qoCRx.R$o*ۯ7¥ο7:9Opp]dVO)lj US {>Ҟ3RT|\?fޜ$oM!X*c} !MSlWf.sJАB`S_]64Dc[ˎ_ ^(ޒO!k,/m[#zg.={)>sM&YߙWSq` p~WlcC&yO0#O祉)<3$Cy} ?dr?!I\?,wJDC:}!&p$ǦpivSX.paqlw)^G%}fy.韓oIdŲmGN[O'yW .-y%I>Ɍ.'}{%ɔCq,K9cgX!)i'\:xcQ>>Е6}K@|0Np?.04U)<$W σĴ^ `:9]_N$},$%yy L$/2F%bIV&iH| v>u*C,Yl69rdV^m )w w e{*[l)w w 9]&\˄Ktp ..e%L@UaժUYfM YzuV^m **Kvm=%>n[SO :4x466m`ܸq2eJ&N8 _Ї͟??]vYѣ3h'9R555eٲe3gN.$p [?7Jt:7n479Gyd>? |QVSO=~%H 4({l,YUV w Mk֬I=`@3f̘$K/4]G577'YK^655 sw*]&\˄Ktp YpaN<Č;6kN<,[,CСC{rKK{?яz6;zs1=c=YT5k2d]:,onnNڵݪL2%j˶o=K.5Џ0K?nysss>=ߤ /SO=5j8?/:G}ts^v_wygsAK;o>;inΣwygjRw$y 2?)_wuYfM&LeYgk /0aB&LgƷzKCMycZf0 RU?nKIr=$IN=Nlo}ݗ>y^y?~ .… <'|r.$M7'ӆ r׶ԩS뮻f9餓Zʮ^YD0 6,;ao{[1bD[ޗf<3-oҥKl%fn,yI'uc/O|>QFCP?|g+LRxvQGK?fuJΝ~W\qE8ԩSf͚]9ӣqn{~^|9cs哟dƎ|5kV^xWrsMO=O=u`<`ۿ qN;̙3'GuTCfE]?;krw}wwgȐ!=zt>dܹ׬&?Or5dĉMuuusL<9<@&Of6KՠA9p̌=ԌxڭS /Yڷ}uQG娣Rn}衇v{$0aBϟ砃V|9MۚKPՠAK%յ= p ..e5g6g]&\˄Ktp ..e%L@ 2]&\˄Ktp `[\xrs~~&knQ?pkܹ~mt .̉'cf]w͉'e˖9r--G?}s1=c= Lڜ{C/~,9Ej^kwuWLW_}eߞK9/cڵ햯[טK/s/~97xpVzh들eIs%=VY_xᅜzyWsg_bvex9{cƌG$s=+vݟ:p `YncLߖ`i峟;&7G?z6466g?[o~ϩ{{\wuYfM&LeYgk /cM0!&L3CM7=7$IyS+o~Ι0ad󳟽3UJSihhhJill̅^q oxCOj)u .Ⱦ~y:lƍ9sf&L#GfĈ9rWݾ^_|q6۾qƭj{$zfe'xb+lUgduzĉS[[}7\pA֭[^/Z3s ` ެpOWdʆ̙2'*\rIolw\|-rG׿u˶Gye^y?~ .… /555}00%o|waɓGmtM[sk_Z[omUg̙aÆkOk/}K?~vy̜93K.sgĈY`A.袖~$aO|"N;%IקznWZ$5jTrH9vS__-wL6lX>O%IzVn$ɥ^O>9Ç>쓯~aÆ\{I+2SNͮÇ礓Nj)[-v13L6lؐs=7{n^~r)9餓z~w_NaXmCv{Gs'IƎ;3Gk|%{wO>9^ziBHm@k%fk~u_;r=Z޿ CIߎ?Æ K͞O$I>m'|2ׯOL:u@NHRX*l׾|7o^?>ַz~w$#FH|xҬgy&'ON}}}v}u]7n\ŮqdR~mU:#ٽߓMK?>&Ia׾d7d|g3 :('6رc\|&Oox[}^~}V^y͚5y{I*8s[™)S$I8V[.GbŊ1"s~WW:! IDATbsioHMM!ڿ~E KmДnz6Gp֭kLtҘYM6dȐ-)-sbۯYIsmẘ+R__38վӦMO?9Ӳrʜqi;OI&%Inʖ,Y9s椺:Gydr)o~aÆ;ȁ2CRJ6+zr?p `;vh.-o|e{ܗSҥ$ozN7M|7ߜ5k駟E]Yv!g}vSO7ߜUV!->-oҥK[wWg?Yvm|׿3&sO.ߝ'ۆ /w^-[k{gԩ-%I]]]Νw1v[9䐊\/| YdI^|tM9shl%K4MF5?|S)S'|2ӧO 7ܐ߿ۿᆿ鉬_n{;<7nY:?OsǶVMS__9㏷[]]7nV!'tRΝn n!'On_|q6nܘK.;GqDߚ'ɵ^s9w{;ٸum[{ϻˡn[nճ{:?!rJ.-.7{oWyK)͟tdr衻dfG{ߓ?tJ:th|SʨQ;䨣}n'?5\'6s=3y<[ :$92gΜuQ>|x?]tQVҶtw~w ѣG3LΝ*Xk*}? f.K/m)w zWg.L푙K%L@U;0s n.QUUUI&}LA/j.QKdٲeg}6IR[[;]Gn7n\̙c!MMM;wn{n}ה)Sre|=،3f@.Ŗ <8fܹY`AO> AЇM81I8_ .m;{wOs.q'NĉjժKȣ]#Gի D 4(r)r%~bv?jӱQF3T92]&\˄K{o$_46W_ISS:[dFI֧muwM$0 25\-ɗdaN~??%A&٘dnSI%x&y}IIhޓI%I.M2Iޝ@kj ҖLNaҿmې$$3$o*~JK&yc*7\ꪃ1ɺ6)+/KTV$NM pitel_1eꋯ)RPK; ^[e9ڠ^IC;ۇE̥ei]#Gf5@S))c{0ɑIvNI%%i.oիWʌ5*=|S)S)lǶe~Bvfٶ$LPn3>dx=O+MH[|=IL!Xd=9\I*oNrB$?H!\JO$JdDǓ|0{sO5$+A.e%L@ 2]&\˄Ktp ..-^7/+gٳҼyk6@RrYinhHscc6m:+f̲3RUS}fJIn1s H/^憆$ɲ3bV`)IRxAK@d3躺Sy$gȲx@Rx iٌY9kV_|躺vc`0s heiZ`,QNlfi2xV.XBln̙f,%Lg01p VV̜̥$f0-1C4 muuy…$`؄K@dٛKg,1mfٳ $\$CǏOUMMRIyTUSIR;iR5+g3lӦj ?>&H4iQGe2]&\˄Ktp ..e%L@ 2]&\˄Ktp ..e%L@ 2]&\˶ETU|K/͛gwZgyi<l+aI>Iޒdx$/&y"ɃInK_:+ inlӦmVg̙Y6cFjjϬY4ۭA}$W&Nm$YdQq.W/^憆$ɲ3bV`)IRxAk5sIF$1I~lT%y_)I.xv?476H?6U$g4`[UI-.mNa҂$_ RxY-uFյd¥˷r{/66`,ՠm|$56-om%`@T$3lKa$7%Ɋ3[XJ 3V̜ipRR]>'דWY)O_4@b̖%񒴚l 0`T*\da瓓H$M$'~?h,.躺m&`TCW> ]I^-~M7~[9{fRK{LYrll*.=)pd$ ? ?>U55IZK%SUMMoZuڽ:7d$˒QV~XEj'M>f~~cڴT ǧv$l*.k KOIb$oLb1r_4iQGm_Y|: =@3U5siVӜ.@[¢rO2.$OK$K,7KoK|h9I>ߒl(n!əI.N1пTWo%1UmoHrekǠ {pŝ1; ?@RpiyAI3J-IH2?$OO2-g2}/ JҜIn(O$/&l<& $W'yoqmO%U p0l*.[|HaInq JK\*%0 JK$`JK\`1By }ToI>d!~TjYI$&:3I۩7%?*.V6eIЏT*\:l*.mh?&W|T?viP>:,Nr_8|?wI#$_NDqtKT*\ I^jS$ uKH;8ɵRד)Iu$%yoYI0$E*o̙-K%i5iٌ&C*.$sdto%'.I>;IvTN`it]]޶pa[ TԸ1)3ǧ&I`diZ ?ޠkBs4MmoNaIL2KIJ0ɑ~ݓI>dnq۩$Ir\2~IϬY_8yfu6-Ug4ɠkR҂$N23?]*3ŲA&y}IIhޓI.%6ɥI&t_sH!|p Iu<^u۾+SxƧߟRXnc~5`iK&oe6$&%)̠ZY6(pM)IFwp?&Yf#ep6ܞ>K?(TvߚB8SnR{2YF'Ye),7l[sOl$Mx@J\,~Z<.gK"w4$c U_V^6߸%:VpIY.$ɧSp$;}hYV9rd.*ɨQ ;ػ8˪.nI4bsTcLx%Qj41OM&c5QQ&3&rE^"8 L H]t PS~^ukծޯ ŗx0*\ڳl'Wo,U}çQYSmr;r+Kj*S;;Na՚KT_=5XhCl*\du\c[߷;謣87]ZuG`F5-h0+utWϪN^՝GUY]SU}zkT X:zpی*\Z]=6nWu?Spzp[UKsTgv7Tu6~w\ju 10W+V'U)}'G6XG1nxiSzmյ7 +JKLejQu^o0W҆=aӆOl* d^꟪(?x5:g&.0k%fM 563^V~~X˨¥CwOTWTGVr ǨS[F˪5ެeT[?zS)?xUpA,mrUD˨\:xZQ}rW*?xU_sWhM ZݧS}v%{/(?xUkuqu6ڮU+?xuVY}Y˨¥_No#_{F˨¥'V>xKՊQ0^F.) BW>W}zLR}fuT#ZD䈯F OWOJ0jҪjV'tf1Wp y`¥PJoR QKV_jE}US~25޳rjzvyjh`|*\z[z@ZyOVT~2iSF;+?xU4U]{+T+?xUWnmA,/ N^cxnIS}e0^FtOTGW'U }xy]0^Fx>R=?WW(=WWTfarמVA {4ʹT˨¥;UZöLwQKoZݵ(MnN~U˨¥gTQ}ڰSeTҒҾV~2p0^F.Mk.m2SMUolһ`L'5jUVV(?xUz~a FI}pY?S#{Z=cb/ W8<ޥ:Ցշ19J-AA ¥7+?xUt@uO^a0^F.4_P=:K\U-Q~25ή^ PzjG-^]eTέ&W_ݢi79` *\juL-ږTﭾej׾:m篬)=TfK 5&\`֦FxjjQZkG.Q:IϟTߪXg.G.ݵ::Պjm:zHcc.åWVh0BܷniWy,5l{x/(=p k_)\swY}jEZLVQ]TZ\K?PN-7T߭RJ_ouGmjjQZY]kj׾w ~z`uP7\:OeOs.Uyujb3UerKë}ë?:ˑKo>S=-Sm/u \\Wuz7njH0^2\z,>Ze.å~ZVz }Vz\sվk߫h0i]ku`Xk12;7"jQuecJ0~FpT')-3(F.=zv⭩.n0-އWz3#Vzykun ˪/5XwiVTB% IDATOSFK?[0>rsFjR>pi{?cd.åoV/駯ijo*?x5^_}nwV+uծՁՃe.åUXID53R~25;zlOu#Ukb:LԈ{yE_&^ O`lpiZe.\zm{՞wT_ ejf}'nCߟe-.^]T=DuHuG`nåsw͢Sg`-ON~G`nåa}펅KLjI};]zް9qذyL&ZgUl^O31n}7V֧{on0-Y8'5 Y=0R-n F0RPk'nunkӫ2\Lu,n݁u}r`һ8zO7?` =[@iu[:gwS{t`bW4tZTz7Xky-jݢ}56;d)k]q~|w{w , MOOtRS;;NaEzcտl1յ[h53lݮ O.]ڪUNS;yl⭩~ @;zsՋUyfoT}WV{ucuTup5YGVW X:zpی*\zm?4n^uLí ¥WoW-ΩR6 Z[ L;=6~w¥ 3<2pJ 0L*55kOTϮR2P=Z:ܞ^}Ұ;8z}VzjWqbhT#zxTUߵOTv vWqFsL37aC{\|y+-kbzrJ{ydT#Y\ԍRպ)ճY˖u[o f֯oy)`F.[}Vڿ=NdN|eeTu 'G+`F5-ՓL-Oa'i*MAҊeZy)m}Y}[d#nλI{l # v0Ssxs[Ջ>li#`55Ϥ0[%fM 565?:گբn1KTaD8/ T=zuu&UeTSwV+`*\:ƭ__eTEa0^F.}zEufϯ[]-ëVl,-TwXQ\Z[=ձ`LnJlr҃4XciI5q!n֯؅oݺ.j}wQF.zgu}u~Z`<\t/+7l>Ї ^oO|MNN(NjTWrept1to]׾vmOxžpwvFwޗ` ƹ^^vc4==G~nnϬ̌F5rߪ{+/eVf Xzhٲ7433ӧ?׼GY^}{ЃvW̨F.`J^-Vfر߯.w]:tsT511ѓWo|AϽ&0*\:zpx+o]cҗټ[-{,_\p;QMժ땧 v`W]u{k-[p}>fc7499;QKLuTu2mlb fU-Z4)X jZ?%XЇygs[e F.}zxv߭}omu[Lu n>~UШ;I{U:v }EqcϬjɒ:'}KW73̛=QwRШ¥W3߹>3#mGщ'_6lK.SO]u~ߢN9n-X`Ѩ51gN>Ҿ۸ƶڥ}{KhѤbF.Ni`<qw%״~L;)vv;i`+s.`?~?-s. ,zG~x\Kor <2WҺr <09^W=Vڟ9UZp+ 129k]V+?xp4nq|0^ZTPMT/R}&}fTY}P\KnUh00OLUZgR -&\`քKڔiviBspiu+`Gk6V5==SW/z )UtI5s;n[>+:w[ٳ~Lؕ}cWWqĄ\K vh^53g‰;.}+׶zKZxl?E浹 NVJ`ٰa7sw瞃?֮ة줓.OtEx-Yby[`,-8㌫zv,U-\8qsOUtW(0;jndT;~d-/}nu/kTҞ?U\M4669ú ?|-hdkl쪫6(0jZU^RݧAkήQ~`Gx2]t[w7fƪ-280ꯞ_N[[[[z;_7CҦ) k Td_XR=YSWwϦ~>o3/ǿ[*0jͥ F/ݷݸiۿ˖hf?_Wq՝y浛?Y{w!*0*\rOǧW7&WW_T~`GW׊]V9sܬߑG] F.zRkzSuOgUR~`G711;Ž+;ܭC/X ^oSS&hQKn\V=txcabb>uIGWut;Ow*;QKW{?UU-D[b;]ꁷ~0^&ݫZe.{xu6.W0^2\o՛3o m9]/s.Tu5QzU73SZg.åk[ս8`usa:Zgj~yubWj0-Uw+?xU_Tw߫L!`|*\:wGV7lq/ 䈮{괶6Ato/ r+0FF.su\m=KR~25NQ}aYR}zQ\z@Տn?^_=p12#U}Z53X:zO^!?+ٛӟjeuDo߃Tm.U:z[HM}}`yf޽Źu{Uw<` kor-ڷZ8ܥu'M>~V+q~E5Qto^Zpyy-j0 Mݢ}K8u}#wڸy+zSu{/R؁piMnj{݂ tҽwjժz+̌v6å:zsՋUy_?u51!]21'?ztU֚:' 8Oj㣪אַΪ>Rڿ!Ջ~#qjcٯef},կ>Vuixs{]3կ7_z^:zJu|7'y=51fo}up`q'MMM?&;I%`{8~gqu2mMl .7|ʮꆛYvcx%̽3փ /\ӟ~^ĕ_ f^ճu^瞻CяC`;R?eUaUq//,P4Ό\`]w>Y-C`0r CwSg_굝v]p6n{Nگ-1È%؁&&&z~.]ڪUv0`քK̚p Y.0k%fM 5&\`քK̚p Y.0k%fM 5&\`քK̚p Y.0k%fM 5&\`քK̚p Y.0k%fMvgSoSO3P,'3ciff6lhcYK/oŲeMLOwSNi#T8sF.q=f֯jŲe]|V훂[{yK.}t36lV,[ŋ/*X: ۦ6IЂNjUWms 'ls<`<;dcN|,XMǶ`:`K0 .]|K5tp;7Tm5iŲe&gKn7 G}>{5L0]VzVҁ'^jMh0]vX Xcj2`na)SJǑGvSNiyGoĂ-<8HEy@ǑG薂'`<Y.0k%fMM)ù^ק?. '{vOܫ #.3upa_u7k[䢎9f9f&&& v@%~f?/8۸+Љ'?X[z v@%~&Vk/~~wn>+:zU8=>խ]{Y#XgSS7N ;I%8kzF4p%⮿~]U =u:j}oB%n+uf=v/\csPN;튾ūZa&-^|yn~ 3 ;>py\>vN>Ц'[t>x}Ox^߭oׂjCs]WО{nټӊ ;ql8y?K7^};EU}[k_vL˗/`]ؕUd ;UWmK޽袉O׽611?\؅jݺ?^ss̽[xƴx&[MnҒ%Spx⊪z.vi Yqsߟ5*6쀌\6kMO k7b3ϼv1xP2%\U/G{a숄K&ӓ={U)˶{[ӧ?}UU3#q&&&zK3k_{@–._ݫ}v=g`GsxeU'xqWЋ_K'}W'rq6 oӮޘc.UW0fKf|9fN9ee33WSNY]z~+W'?qwQX7ж\ڭza!]_Auf PKox jriy c 6Q-Փê'V>\j~>ж҉UW7>LTO^1fQ]nnM\e#9mTzL9՛[VΔ1J3eY nx13ۗ@c-]{T7h۝oZjjskթ&ZO.]1 ޼gIuwo zrjlb{W;wd>S1S@La [ɥxU5?Sf>|S1S@La [_߱oQ]0m`zr=Y)[_Q+-Oվ{Vט~UuzuL%Օo4KUzӫkN_P{zTH.U=zQ qݫ~_϶\ [=cziVJr \`$X1%VLr \`$X1%tX_Aufꎚܦ:꠹7>Z^zk2}d֤ݪV!]gՑMX1e/_SͫWW_=zWu> 1EXSQSuSչխ4 ІV{5So䩳^_v;L|Qz479M]}^AjĐWϭ[}:߭5Sm|qAs}/+ݍ/پzDFF(*}`8h JO)ۥz#\:x3^v̾3ew}Lٮg#&5eJ7kQC?Stk`MǔC9b lnɡ6FUX/kR}o`7\ڽqz?޹fu/w^;`mCUm|pglLYf]&!>W}VX¥:MV81cfvn{mWk|s ; }~ ?Cnژf1/[]OÚv٩rZc W7lJL6OQU)(\H.5ژsI:`wnI=T7>YL,id6|S檍ow~9u>#C6I5, T=1̟VZLf=1;QU)(\ YP~v`qKĊœ_N?]\g;GV#lxksXW})4F݆QQ2pc[jlƷkn7uTbJYemd5bȦ97yosqBL\}Yuߴ> pQ}.u$'a5nmj\<9mc\+;GV#nL#\S`vϻT?GV)p#8shdD2:@gL/w\,91iOw`Z1ՙ+7>9YL&4nuk(*}`M9:zL)y6Lo5}Lk}g0ŘGϔXs<:&5fSL8dՈ!o,{\L;zk6_xy=kz(ESQ5]S{1:zz>ͳG5>1d3^gMu1l{c#LϟaMz\c4>y9CUQV=oթn ٘_Ô0=>4=b>r ౚ'}}`YH,1T3]x9`ӍT߯\]gAVi|Sgeˍ%:U鏠Ekr ٳz}C %cʞSk)@Q%] '}dݦ: ޼:::IiYߩ:ί A95.5݇x Y `ru7ӪOUOv< VPg='W;{o5^%qWoN)M>w_WT_lLȓYoN]scML瞵_#I3uiקU/]pwtޯNkI_4Bl:Wk$s~8Km<¨jQd_6z15jKYrjN鷪7ՙ]si,;G _LV+|kyG{Vi$vXyN޳Z_{)vMFCFBYi3SzZYFի?zWc5 K#QG43k,9e: 5FY=: #Un4;:nNl<1:Fbf&TJS;WIuשʚ=t̓&ݾͯUs}?HN-LӱN^K4t٧XTȷM:z\S=SvÍD 0ߑM˹Zu sFҪAǧk\ΞҕkKouH)Xsh|ֿT~ 5*)T6Νm<ܡS+ȑK#c~6S/[Ε?f:-O_a#6卍Hv2w٤̮lZ۴xZo>N4י?s˼{Mǿfov"?ku~\]o6rTū:zCiχxՙD"Wh-cZ<ֿ[cd|卑w+r%K1lm𨑈Fsi$pfݷJ#isřQn7;uyHo}n$v; {4; t]3Pl*u>lg8zlc$'#1wM{L׵d6S=ک_cjoӵqz:1nS>ۅ{g$]xfKpwzubu_s~x~ޯYVH,: ϳcn}u$O5`~pu|^؆z2xܥ,[&of7:GuƔ{OmL w&|ީV2ctz-W̵ {Zc.h^zb#)mK[Fr G4>6L[6tQ6Wj${_ۺTy|? fcVxӫgU/n$U{zX9K#NoLw 7b߬v5dc5F"= 4kMykjj#z: 꾽6)GʯH>}EX-#{Yc[<%QUn$ 1?5A{4b5fܸz޸Jok$&5㿧}vZuͼT/WV;LNϯ4]jSݩb굍Yn$A[H|1w5FVyU7WTCkUmP[vl91B')ݞӆinH0gTr?s|#0?5իv9$XzWo$iNo$^HNHtmmzkEV6zT{cDڦF.XSST{k]1&)[;9Nǻ\Q-Ӧ dܢf7h^PԿ6F]^Y}-KŮq1wzg58Sz r:Ml 4ۙN"eIENDB`indexed-gzip-0.6.1/setup.cfg000066400000000000000000000000251317257227400157350ustar00rootroot00000000000000[aliases] test=pytestindexed-gzip-0.6.1/setup.py000066400000000000000000000130011317257227400156240ustar00rootroot00000000000000#!/usr/bin/env python import sys import os import glob import os.path as op import shutil from setuptools import setup from setuptools import Extension from setuptools import Command # Custom 'clean' command class Clean(Command): user_options = [] def initialize_options(self): pass def finalize_options( self): pass def run(self): base = op.dirname(__file__) igzbase = op.join(base, 'indexed_gzip') shutil.rmtree(op.join(base, 'build'), ignore_errors=True) shutil.rmtree(op.join(base, 'dist'), ignore_errors=True) shutil.rmtree(op.join(base, 'indexed_gzip.egg-info'), ignore_errors=True) shutil.rmtree(op.join(base, '.eggs'), ignore_errors=True) shutil.rmtree(op.join(base, '__pycache__'), ignore_errors=True) shutil.rmtree(op.join(igzbase, '__pycache__'), ignore_errors=True) shutil.rmtree(op.join(igzbase, 'tests', '__pycache__'), ignore_errors=True) files = [ '*.so', op.join(igzbase, 'indexed_gzip.c'), op.join(igzbase, '*.pyc'), op.join(igzbase, '*.so'), op.join(igzbase, 'tests', '*.so'), op.join(igzbase, 'tests', '*.pyc'), op.join(igzbase, 'tests', 'ctest_zran.c'), op.join(igzbase, 'tests', 'ctest_indexed_gzip.c')] for f in files: for g in glob.glob(f): try: os.remove(g) except OSError: pass # Platform information python2 = sys.version_info[0] == 2 noc99 = python2 or (sys.version_info[0] == 3 and sys.version_info[1] <= 4) windows = sys.platform.startswith("win") # If cython is present, we'll compile # the pyx files from scratch. Otherwise, # we'll compile the pre-generated c # files (which are assumed to be present). have_cython = True have_numpy = True try: from Cython.Build import cythonize except: have_cython = False # We need numpy to compile the test modules try: import numpy as np except: have_numpy = False include_dirs = ['indexed_gzip'] lib_dirs = [] libs = [] extra_compile_args = [] # If numpy is present, we need # to include the headers if have_numpy: include_dirs.append(np.get_include()) if windows: ZLIB_HOME = os.environ.get("ZLIB_HOME", "c:/Program Files (x86)/GnuWin32") include_dirs.append(os.path.join(ZLIB_HOME, "include")) libs.append('zlib') lib_dirs.append(os.path.join(ZLIB_HOME, "lib")) # For stdint.h which is not included in the old Visual C # compiler used for Python 2 if python2: include_dirs.append('compat') # Some C functions might not be present when compiling against # older versions of python if noc99: extra_compile_args += ['-DNO_C99'] else: libs.append('z') extra_compile_args += ['-Wall', '-pedantic', '-Wno-unused-function'] # Compile from cython files if # possible, or compile from c. if have_cython: pyx_ext = 'pyx' else: pyx_ext = 'c' # The indexed_gzip module igzip_ext = Extension( 'indexed_gzip.indexed_gzip', [op.join('indexed_gzip', 'indexed_gzip.{}'.format(pyx_ext)), op.join('indexed_gzip', 'zran.c')], libraries=libs, library_dirs=lib_dirs, include_dirs=include_dirs, extra_compile_args=extra_compile_args) # Optional test modules test_exts = [ Extension( 'indexed_gzip.tests.ctest_indexed_gzip', [op.join('indexed_gzip', 'tests', 'ctest_indexed_gzip.{}'.format(pyx_ext))], libraries=libs, library_dirs=lib_dirs, include_dirs=include_dirs, extra_compile_args=extra_compile_args) ] if not windows: # Uses POSIX memmap API so won't work on Windows test_exts.append(Extension( 'indexed_gzip.tests.ctest_zran', [op.join('indexed_gzip', 'tests', 'ctest_zran.{}'.format(pyx_ext)), op.join('indexed_gzip', 'zran.c')], libraries=libs, library_dirs=lib_dirs, include_dirs=include_dirs, extra_compile_args=extra_compile_args)) # If we have numpy, we can compile the tests if have_numpy: extensions = [igzip_ext] + test_exts else: extensions = [igzip_ext] # Cythonize if we can if have_cython: extensions = cythonize(extensions) # find the version number def readVersion(): version = {} initfile = op.join(op.dirname(__file__), 'indexed_gzip', '__init__.py') with open(initfile, 'rt') as f: for line in f: if line.startswith('__version__'): exec(line, version) break return version.get('__version__') setup( name='indexed_gzip', packages=['indexed_gzip', 'indexed_gzip.tests'], version=readVersion(), author='Paul McCarthy', author_email='pauldmccarthy@gmail.com', description='Fast random access of gzip files in Python', url='https://github.com/pauldmccarthy/indexed_gzip', license='zlib', classifiers=[ 'Development Status :: 3 - Alpha', 'Intended Audience :: Developers', 'License :: OSI Approved :: zlib/libpng License', 'Programming Language :: C', 'Programming Language :: Cython', 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 3', 'Topic :: System :: Archiving :: Compression', ], cmdclass={'clean' : Clean}, ext_modules=extensions, setup_requires=['pytest-runner'], tests_require=['pytest', 'numpy'], test_suite='tests', )