pax_global_header00006660000000000000000000000064147620362150014520gustar00rootroot0000000000000052 comment=5e320f523676c831cd3e99ae89d6e84e1fec86d4 python-isal-1.7.2/000077500000000000000000000000001476203621500137765ustar00rootroot00000000000000python-isal-1.7.2/.github/000077500000000000000000000000001476203621500153365ustar00rootroot00000000000000python-isal-1.7.2/.github/PULL_REQUEST_TEMPLATE.md000066400000000000000000000001621476203621500211360ustar00rootroot00000000000000 ### Checklist - [ ] Pull request details were added to CHANGELOG.rst - [ ] Documentation was updated (if needed) python-isal-1.7.2/.github/release_checklist.md000066400000000000000000000015371476203621500213370ustar00rootroot00000000000000Release checklist - [ ] Check outstanding issues on JIRA and Github. - [ ] Check [latest documentation](https://python-isal.readthedocs.io/en/latest/) looks fine. - [ ] Create a release branch. - [ ] Change current development version in `CHANGELOG.rst` to stable version. - [ ] Check if the address sanitizer does not find any problems using `tox -e asan` - [ ] Merge the release branch into `main`. - [ ] Created an annotated tag with the stable version number. Include changes from CHANGELOG.rst. - [ ] Push tag to remote. This triggers the wheel/sdist build on github CI. - [ ] merge `main` branch back into `develop`. - [ ] Build the new tag on readthedocs. Only build the last patch version of each minor version. So `1.1.1` and `1.2.0` but not `1.1.0`, `1.1.1` and `1.2.0`. - [ ] Create a new release on github. - [ ] Update the package on conda-forge. python-isal-1.7.2/.github/workflows/000077500000000000000000000000001476203621500173735ustar00rootroot00000000000000python-isal-1.7.2/.github/workflows/ci.yml000066400000000000000000000230051476203621500205110ustar00rootroot00000000000000name: Continous integration on: pull_request: push: branches: - develop - main tags: - "*" jobs: lint: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 with: submodules: recursive - name: Set up Python 3.8 uses: actions/setup-python@v5 with: python-version: 3.8 - name: Install tox run: pip install tox - name: Lint run: tox -e lint package-checks: strategy: matrix: tox_env: - docs - twine_check runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 with: submodules: recursive - name: Set up Python 3.8 uses: actions/setup-python@v5 with: python-version: 3.8 - name: Install isal run: sudo apt-get install libisal-dev - name: Install tox and upgrade setuptools and pip run: pip install --upgrade tox setuptools pip - name: Run tox -e ${{ matrix.tox_env }} run: tox -e ${{ matrix.tox_env }} env: PYTHON_ISAL_LINK_DYNAMIC: True test-static: runs-on: ${{ matrix.os }} strategy: matrix: python-version: - "3.8" - "3.9" - "3.10" - "3.11" - "3.12" - "3.13" - "pypy-3.9" - "pypy-3.10" os: ["ubuntu-latest"] include: - os: "macos-13" python-version: "3.8" - os: "macos-14" python-version: "3.10" - os: "windows-latest" python-version: "3.8" steps: - uses: actions/checkout@v4 with: submodules: recursive - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install tox and upgrade setuptools run: pip install --upgrade tox setuptools - name: Install build dependencies (Linux) # Yasm in pypa/manylinux images. run: sudo apt install nasm if: runner.os == 'Linux' - name: Install build dependencies (Macos) # Install yasm because nasm does not work when building wheels. # Probably because of nasm-filter.sh not filtering all flags that can not be used. run: brew install nasm if: runner.os == 'macOS' - name: Set MSVC developer prompt uses: ilammy/msvc-dev-cmd@v1 if: runner.os == 'Windows' - name: Install nasm (Windows) uses: ilammy/setup-nasm@v1 if: runner.os == 'Windows' - name: Run tests run: tox - name: Upload coverage report uses: codecov/codecov-action@v1 test-arch: if: startsWith(github.ref, 'refs/tags') || github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/main' runs-on: "ubuntu-latest" strategy: matrix: python_version: - "3.8" steps: - uses: actions/checkout@v4 with: submodules: recursive - uses: uraimo/run-on-arch-action@v3 name: Build & run test with: arch: none distro: none base_image: "--platform=linux/arm64 quay.io/pypa/manylinux2014_aarch64" # versioningit needs an accessible git repository but the container # is run as root, which is different from the repository user. # use git config to override this. run: |- git config --global --add safe.directory $PWD CFLAGS="-DNDEBUG -g0" python${{matrix.python_version}} -m pip install . pytest python${{matrix.python_version}} -m pytest tests # Test if the python-isal conda package can be build. Which is linked # dynamically to the conda isa-l package. test-dynamic: runs-on: ${{ matrix.os }} defaults: run: # This is needed for miniconda, see: # https://github.com/marketplace/actions/setup-miniconda#important. shell: bash -l {0} strategy: matrix: os: ["ubuntu-latest", "macos-13", "windows-latest"] python_version: [ "python" ] include: - os: "ubuntu-latest" python_version: "pypy" steps: - uses: actions/checkout@v4 with: submodules: recursive - name: Install miniconda. uses: conda-incubator/setup-miniconda@v3 # https://github.com/conda-incubator/setup-miniconda. with: channels: conda-forge,defaults - name: Install requirements (universal) run: conda install isa-l ${{ matrix.python_version}} tox - name: Set MSVC developer prompt uses: ilammy/msvc-dev-cmd@v1 if: runner.os == 'Windows' - name: Run tests (dynamic link) run: tox env: PYTHON_ISAL_LINK_DYNAMIC: True deploy: if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') runs-on: ${{ matrix.os }} needs: - lint - package-checks - test-static - test-dynamic - test-arch strategy: matrix: os: - ubuntu-latest - macos-13 - macos-14 - windows-latest cibw_archs_linux: ["x86_64"] cibw_before_all_linux: - >- curl -o nasm-2.15.05.tar.gz https://www.nasm.us/pub/nasm/releasebuilds/2.15.05/nasm-2.15.05.tar.gz && tar -xzvf nasm-2.15.05.tar.gz && cd nasm-2.15.05/ && ./autogen.sh && ./configure && make nasm && install -c nasm /usr/bin/nasm build_sdist: [true] include: - os: "ubuntu-latest" cibw_archs_linux: "aarch64" cibw_before_all_linux: "true" # The true command exits with 0 steps: - uses: actions/checkout@v4 with: submodules: recursive fetch-depth: 0 # Fetch everything to get accurately versioned tag. - uses: actions/setup-python@v2 # Some issues where caused by higher versions. name: Install Python - name: Install cibuildwheel twine wheel run: python -m pip install cibuildwheel twine wheel - name: Install build dependencies (Macos) run: brew install nasm if: runner.os == 'macOS' - name: Set MSVC developer prompt uses: ilammy/msvc-dev-cmd@v1 if: runner.os == 'Windows' - name: Install nasm (Windows) uses: ilammy/setup-nasm@v1 if: runner.os == 'Windows' - name: Set up QEMU if: ${{runner.os == 'Linux' && matrix.cibw_archs_linux == 'aarch64'}} uses: docker/setup-qemu-action@v3 with: platforms: arm64 - name: Build wheels run: cibuildwheel --output-dir dist env: CIBW_SKIP: "*-win32 *-manylinux_i686 cp38-macosx_*arm64 cp39-macosx_*arm64" # Skip 32 bit and problematic mac builds. CIBW_ARCHS_LINUX: ${{ matrix.cibw_archs_linux }} CIBW_BEFORE_ALL_LINUX: ${{ matrix.cibw_before_all_linux }} # Fully test the build wheels again. CIBW_TEST_REQUIRES: "pytest" # Simple tests that requires the project to be build correctly # Skip extensive compatibility testing which is slow. CIBW_TEST_COMMAND_LINUX: >- pytest -v {project}/tests/test_igzip.py {project}/tests/test_gzip_compliance.py {project}/tests/test_zlib_compliance.py {project}/tests/test_igzip_lib.py -k 'not test_compress_decompress' CIBW_TEST_COMMAND_MACOS: >- pytest -v {project}/tests/test_igzip.py {project}/tests/test_gzip_compliance.py {project}/tests/test_zlib_compliance.py {project}/tests/test_igzip_lib.py -k 'not test_compress_decompress' # Windows does not have the test module apparently. Do more expensive # tests to verify build. CIBW_TEST_COMMAND_WINDOWS: >- pytest {project}/tests/test_igzip.py {project}/tests/test_igzip_lib.py {project}/tests/test_compat.py CIBW_ENVIRONMENT_LINUX: >- PYTHON_ISAL_BUILD_CACHE=True PYTHON_ISAL_BUILD_CACHE_FILE=/tmp/build_cache CFLAGS="-g0 -DNDEBUG" CIBW_ENVIRONMENT_WINDOWS: >- PYTHON_ISAL_BUILD_CACHE=True PYTHON_ISAL_BUILD_CACHE_FILE=${{ runner.temp }}\build_cache CIBW_ENVIRONMENT_MACOS: >- PYTHON_ISAL_BUILD_CACHE=True PYTHON_ISAL_BUILD_CACHE_FILE=${{ runner.temp }}/build_cache CFLAGS="-g0 -DNDEBUG" - name: Build sdist if: ${{runner.os == 'Linux' && matrix.cibw_archs_linux == 'x86_64'}} run: | pip install build python -m build --sdist - uses: actions/upload-artifact@v4 with: name: "dist-${{ runner.os }}-${{ runner.arch }}-${{ matrix.cibw_archs_linux }}" path: "dist/" - name: Publish package to TestPyPI # pypa/gh-action-pypi-publish@master does not work on OSX # Alpha, Beta and dev releases contain a - in the tag. if: contains(github.ref, '-') && startsWith(github.ref, 'refs/tags') run: twine upload --skip-existing --repository-url https://test.pypi.org/legacy/ dist/* env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.TEST_PYPI_API_TOKEN }} - name: Publish package to PyPI if: "!contains(github.ref, '-') && startsWith(github.ref, 'refs/tags')" run: twine upload --skip-existing dist/* env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} python-isal-1.7.2/.gitignore000066400000000000000000000034351476203621500157730ustar00rootroot00000000000000src/isal/_version.py # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class # C extensions *.so # Distribution / packaging .Python build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ wheels/ pip-wheel-metadata/ share/python-wheels/ *.egg-info/ .installed.cfg *.egg MANIFEST # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .nox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *.cover *.py,cover .hypothesis/ .pytest_cache/ # Translations *.mo *.pot # Django stuff: *.log local_settings.py db.sqlite3 db.sqlite3-journal # Flask stuff: instance/ .webassets-cache # Scrapy stuff: .scrapy # Sphinx documentation docs/_build/ # PyBuilder target/ # Jupyter Notebook .ipynb_checkpoints # IPython profile_default/ ipython_config.py # pyenv .python-version # pipenv # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. # However, in case of collaboration, if having platform-specific dependencies or dependencies # having no cross-platform support, pipenv may install dependencies that don't work, or not # install all needed dependencies. #Pipfile.lock # PEP 582; used by e.g. github.com/David-OConnor/pyflow __pypackages__/ # Celery stuff celerybeat-schedule celerybeat.pid # SageMath parsed files *.sage.py # Environments .env .venv env/ venv/ ENV/ env.bak/ venv.bak/ # Spyder project settings .spyderproject .spyproject # Rope project settings .ropeproject # mkdocs documentation /site # mypy .mypy_cache/ .dmypy.json dmypy.json # Pyre type checker .pyre/ python-isal-1.7.2/.gitmodules000066400000000000000000000001361476203621500161530ustar00rootroot00000000000000[submodule "src/isal/isa-l"] path = src/isal/isa-l url = https://github.com/intel/isa-l.git python-isal-1.7.2/.readthedocs.yml000066400000000000000000000004271476203621500170670ustar00rootroot00000000000000version: 2 formats: [] # Do not build epub and pdf python: install: - requirements: "requirements-docs.txt" - method: "pip" path: "." sphinx: configuration: docs/conf.py build: os: "ubuntu-22.04" tools: python: "3" apt_packages: - libisal-dev python-isal-1.7.2/CHANGELOG.rst000066400000000000000000000334641476203621500160310ustar00rootroot00000000000000========== Changelog ========== .. Newest changes should be on top. .. This document is user facing. Please word the changes in such a way .. that users understand how the changes affect the new version. version 1.7.2 ----------------- + Use upstream ISA-L version 2.31.1 which includes patches to make installation on MacOS ARM64 possible. + Fix a bug where bytes were copied in the wrong order on big endian architectures. Fixes test failures on s390x. + Enable building on GNU/Hurd platforms. version 1.7.1 ----------------- + Fix a bug where flushing files when writing in threaded mode did not work properly. + Prevent threaded opening from blocking python exit when an error is thrown in the calling thread. version 1.7.0 ----------------- + Include a patched ISA-L version 2.31. The applied patches make compilation and wheelbuilding on MacOS ARM64 possible. + Fix a bug where READ and WRITE in isal.igzip were inconsistent with the values in gzip on Python 3.13 + Small simplifications to the ``igzip.compress`` function, which should lead to less overhead. version 1.6.1 ----------------- + Fix a bug where streams that were passed to igzip_threaded.open where closed. version 1.6.0 ----------------- + Fix a bug where compression levels for IGzipFile where checked in read mode. + Update statically linked ISA-L release to 2.31.0 + Fix an error that occurred in the ``__close__`` function when a threaded writer was initialized with incorrect parameters. version 1.5.3 ----------------- + Fix a bug where append mode would not work when using ``igzip_threaded.open``. version 1.5.2 ----------------- + Fix a bug where a filehandle remained opened when ``igzip_threaded.open`` was used for writing with a wrong compression level. + Fix a memory leak that occurred when an error was thrown for a gzip header with the wrong magic numbers. + Fix a memory leak that occurred when isal_zlib.decompressobj was given a wrong wbits value. version 1.5.1 ----------------- + Fix a memory leak in the GzipReader.readall implementation. version 1.5.0 ----------------- + Make a special case for threads==1 in ``igzip_threaded.open`` for writing files. This now combines the writing and compression thread for less overhead. + Maximize time spent outside the GIL for ``igzip_threaded.open`` writing. This has decreased wallclock time significantly. version 1.4.1 ----------------- + Fix several errors related to unclosed files and buffers. version 1.4.0 ----------------- + Drop support for python 3.7 and PyPy 3.8 as these are no longer supported. Add testing and support for python 3.12 and PyPy 3.10. + Added an experimental ``isal.igzip_threaded`` module which has an ``open`` function. This can be used to read and write large files in a streaming fashion while escaping the GIL. + The internal ``igzip._IGzipReader`` has been rewritten in C. As a result the overhead of decompressing files has significantly been reduced and ``python -m isal.igzip`` is now very close to the C ``igzip`` application. + The ``igzip._IGZipReader`` in C is now used in ``igzip.decompress``. The ``_GzipReader`` also can read from objects that support the buffer protocol. This has reduced overhead significantly. version 1.3.0 ----------------- + Gzip headers are now actively checked for a BGZF extra field. If found the block size is taken into account when decompressing. This has further improved bgzf decompression speed by 5% on some files compared to the more generic solution of 1.2.0. + Integrated CPython 3.11 code for reading gzip headers. This leads to more commonality between the python-isal code and the upstream gzip.py code. This has enabled the change above. It comes at the cost of a slight increase in overhead at the ``gzip.decompress`` function. version 1.2.0 ----------------- + Bgzip files are now detected and a smaller reading buffer is used to accomodate the fact that bgzip blocks are typically less than 64K. (Unlike normal gzip files that consist of one block that spans the entire file.) This has reduced decompression time for bgzip files by roughly 12%. + Speed-up source build by using ISA-L Unix-specific makefile rather than the autotools build. + Simplify build setup. ISA-L release flags are now used and not overwritten with python release flags when building the included static library. + Fix bug where zdict's could not be set for ``isal_zlib.decompressobj`` and ``igzip_lib.IgzipDecompressor``. + Escape GIL when calling inflate, deflate, crc32 and adler32 functions just like in CPython. This allows for utilising more CPU cores in combination with the threading module. This comes with a very slight cost in efficiency for strict single-threaded applications. version 1.1.0 ----------------- + Added tests and support for Python 3.11. version 1.0.1 ------------------ + Fixed failing tests and wheel builds for PyPy. version 1.0.0 ------------------ Python-isal has been rewritten as a C-extension (first implementation was in Cython). This has made the library faster in many key areas. + Since the module now mostly contains code copied from CPython and then modified to work with ISA-L the license has been changed to the Python Software Foundation License version 2. + Python versions lower than 3.7 are no longer supported. Python 3.6 is out of support since December 2021. + Stub files with type information have now been updated to correctly display positional-only arguments. + Expose ``READ`` and ``WRITE`` constants on the ``igzip`` module. These are also present in Python's stdlib ``gzip`` module and exposing them allows for better drop-in capability of ``igzip``. Thanks to @alexander-beedie in https://github.com/pycompression/python-isal/pull/115. + A ``--no-name`` flag has been added to ``python -m isal.igzip``. + Reduced wheel size by not including debug symbols in the binary. Thanks to @marcelm in https://github.com/pycompression/python-isal/pull/108. + Cython is no longer required as a build dependency. + isal_zlib.compressobj and isal_zlib.decompressobj are now about six times faster. + igzip.decompress has 30% less overhead when called. + Error structure has been simplified. There is only ``IsalError`` which has ``Exception`` as baseclass instead of ``OSError``. ``isal_zlib.IsalError``, ``igzip_lib.IsalError``, ``isal_zlib.error`` and ``igzip_lib.error`` are all aliases of the same error class. + GzipReader now uses larger input and output buffers (128k) by default and IgzipDecompressor.decompress has been updated to allocate ``maxsize`` buffers when these are of reasonable size, instead of growing the buffer to maxsize on every call. This has improved gzip decompression speeds by 7%. + Patch statically linked included library (ISA-L 2.30.0) to fix the following: + ISA-L library version variables are now available on windows as well, for the statically linked version available on PyPI. + Wheels are now always build with nasm for the x86 architecture. Previously yasm was used for Linux and MacOS due to build issues. + Fixed a bug upstream in ISA-L were zlib headers would be created with an incorrect wbits value. + Python-isal shows up in Python profiler reports. + Support and tests for Python 3.10 were added. + Due to a change in the deployment process wheels should work for older versions of pip. + Added a ``crc`` property to the IgzipDecompressor class. Depending on the decompression flag chosen, this will update with an adler32 or crc32 checksum. + All the decompression NO_HDR flags on igzip_lib were incorrectly documented. This is now fixed. version 0.11.1 ------------------ + Fixed an issue which occurred rarely that caused IgzipDecompressor's unused_data to report back incorrectly. This caused checksum errors when reading gzip files. The issue was more likely to trigger in multi-member gzip files. version 0.11.0 ------------------ In this release the ``python -m isal.igzip`` relatively slow decompression rate has been improved in both speed and usability. Previously it was 19% slower than ``igzip`` when used with the ``-d`` flag for decompressing, now it is just 8% slower. Also some extra flags were added to make it easier to select the output file. + Prompt when an output file is overwritten with the ``python -m isal.igzip`` command line utility and provide the ``-f`` or ``--force`` flags to force overwriting. + Added ``-o`` and ``--output`` flags to the ``python -m isal.igzip`` command line utility to allow the user to select the destination of the output file. + Reverse a bug in the build system which caused some docstring and parameter information on ``igzip_lib`` and ``isal_zlib`` to disappear in the documentation and the REPL. + Increase the buffer size for ``python -m isal.igzip`` so it is now closer to speeds reached with ``igzip``. + Add a ``READ_BUFFER_SIZE`` attribute to ``igzip`` which allows setting the amount of raw data that is read at once. + Add an ``igzip_lib.IgzipDecompressor`` object which can decompress without using an unconsumed_tail and is therefore more efficient. version 0.10.0 ------------------ + Added an ``igzip_lib`` module which allows more direct access to ISA-L's igzip_lib API. This allows features such as headerless compression and decompression, as well as setting the memory levels manually. + Added more extensive documentation. version 0.9.0 ----------------- + Fix a bug where a AttributeError was triggered when zlib.Z_RLE or zlib.Z_FIXED were not present. + Add support for Linux aarch64 builds. + Add support for pypy by adding pypy tests to the CI and setting up wheel building support. version 0.8.1 ----------------- + Fix a bug where multi-member gzip files where read incorrectly due to an offset error. This was caused by ISA-L's decompressobj having a small bitbuffer which was not taken properly into account in some circumstances. version 0.8.0 ----------------- + Speed up ``igzip.compress`` and ``igzip.decompress`` by improving the implementation. + Make sure compiler arguments are passed to ISA-L compilation step. Previously ISA-L was compiled without optimisation steps, causing the statically linked library to be significantly slower. + A unused constant from the ``isal_zlib`` library was removed: ``ISAL_DEFAULT_HIST_BITS``. + Refactor isal_zlib.pyx to work almost the same as zlibmodule.c. This has made the code look cleaner and has reduced some overhead. version 0.7.0 ----------------- + Remove workarounds in the ``igzip`` module for the ``unconsumed_tail`` and ``unused_data`` bugs. ``igzip._IGzipReader`` now functions the same as ``gzip._GzipReader`` with only a few calls replaced with ``isal_zlib`` calls for speed. + Correctly implement ``unused_data`` and ``unconsumed_tail`` on ``isal_zlib.Decompress`` objects. It works the same as in CPython's zlib now. + Correctly implement flush implementation on ``isal_zlib.Compress`` and ``isal_zlib.Decompress`` objects. It works the same as in CPython's zlib now. version 0.6.1 ----------------- + Fix a crash that occurs when opening a file that did not end in ``.gz`` while outputting to stdout using ``python -m isal.igzip``. version 0.6.0 ----------------- + ``python -m gzip``'s behaviour has been changed since fixing bug: `bpo-43316 `_. This bug was not present in ``python -m isal.igzip`` but it handled the error differently than the solution in CPython. This is now corrected and ``python -m isal.igzip`` handles the error the same as the fixed ``python -m gzip``. + Installation on Windows is now supported. Wheels are provided for Windows as well. version 0.5.0 ----------------- + Fix a bug where negative integers were not allowed for the ``adler32`` and ``crc32`` functions in ``isal_zlib``. + Provided stubs (type-hint files) for ``isal_zlib`` and ``_isal`` modules. Package is now tested with mypy to ensure correct type information. + The command-line interface now reads in blocks of 32K instead of 8K. This improves performance by about 6% when compressing and 11% when decompressing. A hidden ``-b`` flag was added to adjust the buffer size for benchmarks. + A ``-c`` or ``--stdout`` flag was added to the CLI interface of isal.igzip. This allows it to behave more like the ``gzip`` or ``pigz`` command line interfaces. version 0.4.0 ----------------- + Move wheel building to cibuildwheel on github actions CI. Wheels are now provided for Mac OS as well. + Make a tiny change in setup.py so python-isal can be build on Mac OS X. version 0.3.0 ----------------- + Set included ISA-L library at version 2.30.0. + Python-isal now comes with a source distribution of ISA-L in its source distribution against which python-isal is linked statically upon installation by default. Dynamic linking against system libraries is now optional. Wheels with the statically linked ISA-L are now provided on PyPI. version 0.2.0 ----------------- + Fixed a bug where writing of the gzip header would crash if an older version of Python 3.7 was used such as on Debian or Ubuntu. This is due to differences between point releases because of a backported feature. The code now checks if the backported feature is present. + Added Python 3.9 to the testing. + Fixed ``setup.py`` to list setuptools as a requirement. + Changed homepage to reflect move to pycompression organization. version 0.1.0 ----------------- + Publish API documentation on readthedocs. + Add API documentation. + Ensure the igzip module is fully compatible with the gzip stdlib module. + Add compliance tests from CPython to ensure isal_zlib and igzip are validated to the same standards as the zlib and gzip modules. + Added a working gzip app using ``python -m isal.igzip`` + Add test suite that tests all possible settings for functions on the isal_zlib module. + Create igzip module which implements all gzip functions and methods. + Create isal_zlib module which implements all zlib functions and methods. python-isal-1.7.2/LICENSE000066400000000000000000000046641476203621500150150ustar00rootroot00000000000000PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 -------------------------------------------- 1. This LICENSE AGREEMENT is between the Python Software Foundation ("PSF"), and the Individual or Organization ("Licensee") accessing and otherwise using this software ("Python") in source or binary form and its associated documentation. 2. Subject to the terms and conditions of this License Agreement, PSF hereby grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, analyze, test, perform and/or display publicly, prepare derivative works, distribute, and otherwise use Python alone or in any derivative version, provided, however, that PSF's License Agreement and PSF's notice of copyright, i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021 Python Software Foundation; All Rights Reserved" are retained in Python alone or in any derivative version prepared by Licensee. 3. In the event Licensee prepares a derivative work that is based on or incorporates Python or any part thereof, and wants to make the derivative work available to others as provided herein, then Licensee hereby agrees to include in any such work a brief summary of the changes made to Python. 4. PSF is making Python available to Licensee on an "AS IS" basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT INFRINGE ANY THIRD PARTY RIGHTS. 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. 6. This License Agreement will automatically terminate upon a material breach of its terms and conditions. 7. Nothing in this License Agreement shall be deemed to create any relationship of agency, partnership, or joint venture between PSF and Licensee. This License Agreement does not grant permission to use PSF trademarks or trade name in a trademark sense to endorse or promote products or services of Licensee, or any third party. 8. By copying, installing or otherwise using Python, Licensee agrees to be bound by the terms and conditions of this License Agreement. python-isal-1.7.2/MANIFEST.in000066400000000000000000000002441476203621500155340ustar00rootroot00000000000000graft src/isal/isa-l include src/isal/*.h prune tests prune docs prune benchmark_scripts exclude requirements-docs.txt exclude codecov.yml exclude .readthedocs.yml python-isal-1.7.2/README.rst000066400000000000000000000223371476203621500154740ustar00rootroot00000000000000.. image:: https://img.shields.io/pypi/v/isal.svg :target: https://pypi.org/project/isal/ :alt: .. image:: https://img.shields.io/conda/v/conda-forge/python-isal.svg :target: https://github.com/conda-forge/python-isal-feedstock :alt: .. image:: https://img.shields.io/pypi/pyversions/isal.svg :target: https://pypi.org/project/isal/ :alt: .. image:: https://img.shields.io/pypi/l/isal.svg :target: https://github.com/pycompression/python-isal/blob/main/LICENSE :alt: .. image:: https://img.shields.io/conda/pn/conda-forge/python-isal.svg :target: https://github.com/conda-forge/python-isal-feedstock :alt: .. image:: https://github.com/pycompression/python-isal//actions/workflows/ci.yml/badge.svg :target: https://github.com/pycompression/python-isal/actions :alt: .. image:: https://codecov.io/gh/pycompression/python-isal/branch/develop/graph/badge.svg :target: https://codecov.io/gh/pycompression/python-isal :alt: .. image:: https://readthedocs.org/projects/python-isal/badge :target: https://python-isal.readthedocs.io :alt: python-isal =========== .. introduction start Faster zlib and gzip compatible compression and decompression by providing Python bindings for the ISA-L library. This package provides Python bindings for the `ISA-L `_ library. The Intel(R) Intelligent Storage Acceleration Library (ISA-L) implements several key algorithms in `assembly language `_. This includes a variety of functions to provide zlib/gzip-compatible compression. ``python-isal`` provides the bindings by offering four modules: + ``isal_zlib``: A drop-in replacement for the zlib module that uses ISA-L to accelerate its performance. + ``igzip``: A drop-in replacement for the gzip module that uses ``isal_zlib`` instead of ``zlib`` to perform its compression and checksum tasks, which improves performance. + ``igzip_threaded`` offers an ``open`` function which returns buffered read or write streams that can be used to read and write large files while escaping the GIL using one or multiple threads. This functionality only works for streaming, seeking is not supported. + ``igzip_lib``: Provides compression functions which have full access to the API of ISA-L's compression functions. ``isal_zlib`` and ``igzip`` are almost fully compatible with ``zlib`` and ``gzip`` from the Python standard library. There are some minor differences see: differences-with-zlib-and-gzip-modules_. .. introduction end Quickstart ---------- .. quickstart start The python-isal modules can be imported as follows .. code-block:: python from isal import isal_zlib from isal import igzip from isal import igzip_lib ``isal_zlib`` and ``igzip`` are meant to be used as drop in replacements so their api and functions are the same as the stdlib's modules. Except where ISA-L does not support the same calls as zlib (See differences below). A full API documentation can be found on `our readthedocs page `_. ``python -m isal.igzip`` implements a simple gzip-like command line application (just like ``python -m gzip``). Full usage documentation can be found on `our readthedocs page `_. .. quickstart end Installation ------------ - with pip: ``pip install isal`` - with conda: ``conda install python-isal`` Installation is supported on Linux, Windows and MacOS. For more advanced installation options check the `documentation `_. python-isal as a dependency in your project ------------------------------------------- .. dependency start Python-isal supports a limited amount of platforms for which wheels have been made available. To prevent your users from running into issues when installing your project please list a python-isal dependency as follows. ``setup.cfg``:: install_requires = isal; platform.machine == "x86_64" or platform.machine == "AMD64" or platform.machine == "aarch64" ``setup.py``:: extras_require={ ":platform.machine == 'x86_64' or platform.machine == 'AMD64' or platform.machine == 'aarch64'": ['isal'] }, .. dependency end .. _differences-with-zlib-and-gzip-modules: Differences with zlib and gzip modules -------------------------------------- .. differences start + Compression level 0 in ``zlib`` and ``gzip`` means **no compression**, while in ``isal_zlib`` and ``igzip`` this is the **lowest compression level**. This is a design choice that was inherited from the ISA-L library. + Compression levels range from 0 to 3, not 1 to 9. ``isal_zlib.Z_DEFAULT_COMPRESSION`` has been aliased to ``isal_zlib.ISAL_DEFAULT_COMPRESSION`` (2). + ``isal_zlib`` only supports ``NO_FLUSH``, ``SYNC_FLUSH``, ``FULL_FLUSH`` and ``FINISH_FLUSH``. Other flush modes are not supported and will raise errors. + ``zlib.Z_DEFAULT_STRATEGY``, ``zlib.Z_RLE`` etc. are exposed as ``isal_zlib.Z_DEFAULT_STRATEGY``, ``isal_zlib.Z_RLE`` etc. for compatibility reasons. However, ``isal_zlib`` only supports a default strategy and will give warnings when other strategies are used. + ``zlib`` supports different memory levels from 1 to 9 (with 8 default). ``isal_zlib`` supports memory levels smallest, small, medium, large and largest. These have been mapped to levels 1, 2-3, 4-6, 7-8 and 9. So ``isal_zlib`` can be used with zlib compatible memory levels. + ``igzip.open`` returns a class ``IGzipFile`` instead of ``GzipFile``. Since the compression levels are not compatible, a difference in naming was chosen to reflect this. ``igzip.GzipFile`` does exist as an alias of ``igzip.IGzipFile`` for compatibility reasons. + ``igzip._GzipReader`` has been rewritten in C. Since this is a private member it should not affect compatibility, but it may cause some issues for instances where this code is used directly. If such issues should occur, please report them so the compatibility issues can be fixed. .. differences end Contributing ------------ .. contributing start Please make a PR or issue if you feel anything can be improved. Bug reports are also very welcome. Please report them on the `github issue tracker `_. .. contributing end Development ----------- .. development start The repository needs to be cloned recursively to make sure the `ISA-L `_ repository is checked out: ``git clone --recursive https://github.com/pycompression/python-isal.git``. If the repository is already checked out you can use ``git submodule update --init``. Patches should be made on a feature branch. To run the testing install ``tox`` with ``pip install tox`` and run the commands ``tox -e lint`` and ``tox``. That will run most of the testing that is also performed by the CI. For changes to the documentation run ``tox -e docs``. For changes to the C code please also run ``tox -e asan`` to check for memory leaks. This requires libasan to be installed. Building requires the `ISA-L build requirements `_ as well. .. development end Acknowledgements ---------------- .. acknowledgements start This project builds upon the software and experience of many. Many thanks to: + The `ISA-L contributors `_ for making ISA-L. Special thanks to @gbtucker for always being especially helpful and responsive. + The `Cython contributors `_ for making it easy to create an extension and helping a novice get start with pointer addresses. + The `CPython contributors `_. Python-isal mimicks ``zlibmodule.c`` and ``gzip.py`` from the standard library to make it easier for python users to adopt it. + `@marcelm `_ for taking a chance on this project and make it a dependency for his `xopen `_ and by extension `cutadapt `_ projects. This gave python-isal its first users who used python-isal in production. + Mark Adler (@madler) for the excellent comments in his pigz code which made it very easy to replicate the behaviour for writing gzip with multiple threads using the ``threading`` and ``isal_zlib`` modules. Another thanks for his permissive license, which allowed the crc32_combine code to be included in the project. (ISA-L does not provide a crc32_combine function, unlike zlib.) And yet another thanks to Mark Adler and also for Jean-loup Gailly for creating the gzip format which is very heavily used in bioinformatics. Without that, I would have never written this library from which I have learned so much. + The `github actions team `_ for creating the actions CI service that enables building and testing on all three major operating systems. + `@animalize `_ for explaining how to test and build python-isal for ARM 64-bit platforms. + And last but not least: everyone who submitted a bug report or a feature request. These make the project better! Python-isal would not have been possible without you! .. acknowledgements end python-isal-1.7.2/benchmark_scripts/000077500000000000000000000000001476203621500174775ustar00rootroot00000000000000python-isal-1.7.2/benchmark_scripts/benchmark.py000066400000000000000000000116151476203621500220070ustar00rootroot00000000000000import argparse import gzip import io # noqa: F401 used in timeit strings import timeit import zlib from pathlib import Path from typing import Dict from isal import igzip, isal_zlib # noqa: F401 used in timeit strings DATA_DIR = Path(__file__).parent.parent / "tests" / "data" COMPRESSED_FILE = DATA_DIR / "test.fastq.gz" with gzip.open(str(COMPRESSED_FILE), mode="rb") as file_h: data = file_h.read() sizes: Dict[str, bytes] = { "0b": b"", "8b": data[:8], "128b": data[:128], "1kb": data[:1024], "8kb": data[:8 * 1024], "16kb": data[:16 * 1024], "32kb": data[:32 * 1024], "64kb": data[:64 * 1024], # "128kb": data[:128*1024], # "512kb": data[:512*1024] } compressed_sizes = {name: zlib.compress(data_block) for name, data_block in sizes.items()} compressed_sizes_gzip = {name: gzip.compress(data_block) for name, data_block in sizes.items()} def show_sizes(): print("zlib sizes") print("name\t" + "\t".join(str(level) for level in range(-1, 10))) for name, data_block in sizes.items(): orig_size = max(len(data_block), 1) rel_sizes = ( str(round(len(zlib.compress(data_block, level)) / orig_size, 3)) for level in range(-1, 10)) print(name + "\t" + "\t".join(rel_sizes)) print("isal sizes") print("name\t" + "\t".join(str(level) for level in range(0, 4))) for name, data_block in sizes.items(): orig_size = max(len(data_block), 1) rel_sizes = ( str(round(len(isal_zlib.compress(data_block, level)) / orig_size, 3)) for level in range(0, 4)) print(name + "\t" + "\t".join(rel_sizes)) def benchmark(name: str, names_and_data: Dict[str, bytes], isal_string: str, zlib_string: str, number: int = 10_000, **kwargs): print(name) print("name\tisal\tzlib\tratio") for name, data_block in names_and_data.items(): timeit_kwargs = dict(globals=dict(**globals(), **locals()), number=number, **kwargs) isal_time = timeit.timeit(isal_string, **timeit_kwargs) zlib_time = timeit.timeit(zlib_string, **timeit_kwargs) isal_microsecs = round(isal_time * (1_000_000 / number), 2) zlib_microsecs = round(zlib_time * (1_000_000 / number), 2) ratio = round(isal_time / zlib_time, 2) print("{0}\t{1}\t{2}\t{3}".format(name, isal_microsecs, zlib_microsecs, ratio)) # show_sizes() def argument_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser() parser.add_argument("--all", action="store_true") parser.add_argument("--checksums", action="store_true") parser.add_argument("--functions", action="store_true") parser.add_argument("--gzip", action="store_true") parser.add_argument("--sizes", action="store_true") parser.add_argument("--objects", action="store_true") return parser if __name__ == "__main__": args = argument_parser().parse_args() if args.checksums or args.all: benchmark("CRC32", sizes, "isal_zlib.crc32(data_block)", "zlib.crc32(data_block)") benchmark("Adler32", sizes, "isal_zlib.adler32(data_block)", "zlib.adler32(data_block)") if args.functions or args.all: benchmark("zlib compression", sizes, "isal_zlib.compress(data_block, 1)", "zlib.compress(data_block, 1)") benchmark("zlib decompression", compressed_sizes, "isal_zlib.decompress(data_block)", "zlib.decompress(data_block)") if args.gzip or args.all: benchmark("gzip compression", sizes, "igzip.compress(data_block, 1)", "gzip.compress(data_block, 1)") benchmark("gzip decompression", compressed_sizes_gzip, "igzip.decompress(data_block)", "gzip.decompress(data_block)") if args.objects or args.all: benchmark("zlib Compress instantiation", {"": b""}, "a = isal_zlib.compressobj()", "a = zlib.compressobj()") benchmark("zlib Decompress instantiation", {"": b""}, "a = isal_zlib.decompressobj()", "a = zlib.decompressobj()") benchmark("Gzip Writer instantiation", {"": b""}, "a = igzip.GzipFile(fileobj=io.BytesIO(), mode='wb' )", "a = gzip.GzipFile(fileobj=io.BytesIO(), mode='wb')") benchmark("Gzip Reader instantiation", {"": b""}, "a = igzip.GzipFile(fileobj=io.BytesIO(), mode='rb' )", "a = gzip.GzipFile(fileobj=io.BytesIO(), mode='rb')") if args.sizes or args.all: show_sizes() python-isal-1.7.2/benchmark_scripts/benchmark_cgzipreader.py000066400000000000000000000004221476203621500243600ustar00rootroot00000000000000import sys from isal.isal_zlib import _GzipReader if __name__ == "__main__": with open(sys.argv[1], "rb") as f: reader = _GzipReader(f, 512 * 1024) while True: block = reader.read(128 * 1024) if not block: break python-isal-1.7.2/benchmark_scripts/gzipread128kblocks.py000066400000000000000000000002701476203621500234610ustar00rootroot00000000000000import sys from isal import igzip with igzip.open(sys.argv[1], "rb") as gzip_file: while True: block = gzip_file.read(128 * 1024) if not block: break python-isal-1.7.2/benchmark_scripts/gzipreadlines.py000066400000000000000000000001751476203621500227140ustar00rootroot00000000000000import sys from isal import igzip with igzip.open(sys.argv[1], "rb") as gzip_file: for line in gzip_file: pass python-isal-1.7.2/benchmark_scripts/gzipthreadsread128kblocks.py000066400000000000000000000003121476203621500250310ustar00rootroot00000000000000import sys from isal import igzip_threaded with igzip_threaded.open(sys.argv[1], "rb") as gzip_file: while True: block = gzip_file.read(128 * 1024) if not block: break python-isal-1.7.2/benchmark_scripts/gzipwrite128kblocks.py000066400000000000000000000004401476203621500236770ustar00rootroot00000000000000import os import sys from isal import igzip with open(sys.argv[1], "rb") as in_file: with igzip.open(os.devnull, "wb") as out_gzip: while True: block = in_file.read(128 * 1024) if block == b"": break out_gzip.write(block) python-isal-1.7.2/benchmark_scripts/gzipwritelines.py000066400000000000000000000003101476203621500231220ustar00rootroot00000000000000import os import sys from isal import igzip with open(sys.argv[1], "rb") as in_file: with igzip.open(os.devnull, "wb") as out_gzip: for line in in_file: out_gzip.write(line) python-isal-1.7.2/benchmark_scripts/memory_leak_test.py000066400000000000000000000011131476203621500234100ustar00rootroot00000000000000import gc import resource import sys from isal import igzip for _ in range(10): with igzip.open(sys.argv[1], "rb") as reader: a = reader.read() print(len(a)) gc.collect() memory_usage = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss memory_usage_mb = memory_usage / 1024 print(f"Maximum memory usage: {memory_usage_mb:.2f} MiB") del a objects_and_size = [(sys.getsizeof(obj), type(obj)) for obj in gc.get_objects()] objects_and_size.sort(key=lambda x: x[0], reverse=True) print(objects_and_size[:10]) python-isal-1.7.2/benchmark_scripts/profile_igziplinewriter.py000066400000000000000000000004631476203621500250230ustar00rootroot00000000000000import cProfile import os import sys from isal import igzip def main(): with open(sys.argv[1], mode="rb") as in_file: with igzip.open(os.devnull, mode="wb") as gzip_h: for line in in_file: gzip_h.write(line) if __name__ == "__main__": cProfile.run("main()") python-isal-1.7.2/benchmark_scripts/profile_igzipreader.py000066400000000000000000000004421476203621500240760ustar00rootroot00000000000000import cProfile import sys from isal import igzip def main(): with igzip.open(sys.argv[1], mode="rb") as gzip_h: while True: block = gzip_h.read(128*1024) if block == b"": return if __name__ == "__main__": cProfile.run("main()") python-isal-1.7.2/codecov.yml000066400000000000000000000002771476203621500161510ustar00rootroot00000000000000coverage: status: project: default: target: 90 # let's try to hit high standards patch: default: target: 90 # Tests should be written for new features python-isal-1.7.2/docs/000077500000000000000000000000001476203621500147265ustar00rootroot00000000000000python-isal-1.7.2/docs/Makefile000066400000000000000000000011721476203621500163670ustar00rootroot00000000000000# Minimal makefile for Sphinx documentation # # You can set these variables from the command line, and also # from the environment for the first two. SPHINXOPTS ?= SPHINXBUILD ?= sphinx-build SOURCEDIR = . BUILDDIR = _build # Put it first so that "make" without argument is like "make help". help: @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) .PHONY: help Makefile # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) python-isal-1.7.2/docs/conf.py000066400000000000000000000034171476203621500162320ustar00rootroot00000000000000# Configuration file for the Sphinx documentation builder. # # This file only contains a selection of the most common options. For a full # list see the documentation: # https://www.sphinx-doc.org/en/master/usage/configuration.html from isal import __version__ # -- Project information ----------------------------------------------------- # Get package information from the installed package. project = 'python-isal' copyright = '2020, Leiden University Medical Center' author = 'Leiden University Medical Center' # The short X.Y version version = __version__ # The full version, including alpha/beta/rc tags release = __version__ # -- General configuration --------------------------------------------------- # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = ["sphinx.ext.autodoc", 'sphinxarg.ext'] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. # includes/* prevents double indexing exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', 'includes/*'] # -- Options for HTML output ------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # html_theme = 'sphinx_rtd_theme' # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static']python-isal-1.7.2/docs/includes/000077500000000000000000000000001476203621500165345ustar00rootroot00000000000000python-isal-1.7.2/docs/includes/CHANGELOG.rst000077700000000000000000000000001476203621500232172../../CHANGELOG.rstustar00rootroot00000000000000python-isal-1.7.2/docs/includes/README.rst000077700000000000000000000000001476203621500223332../../README.rstustar00rootroot00000000000000python-isal-1.7.2/docs/index.rst000066400000000000000000000111511476203621500165660ustar00rootroot00000000000000.. python-isal documentation master file, created by sphinx-quickstart on Fri Sep 11 15:42:56 2020. You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. ======================================= Welcome to python-isal's documentation! ======================================= .. toctree:: :maxdepth: 2 :caption: Contents: ============ Introduction ============ .. include:: includes/README.rst :start-after: .. introduction start :end-before: .. introduction end ========== Quickstart ========== .. include:: includes/README.rst :start-after: .. quickstart start :end-before: .. quickstart end ============ Installation ============ Installation with pip --------------------- :: pip install isal Installation is supported on Linux, MacOS and Windows. On most platforms wheels are provided. The installation will include a staticallly linked version of ISA-L. If a wheel is not provided for your system the installation will build ISA-L first in a temporary directory. Please check the `ISA-L homepage `_ for the build requirements. The latest development version of python-isal can be installed with:: pip install git+https://github.com/rhpvorderman/python-isal.git This requires having the build requirements installed. If you wish to link dynamically against a version of libisal installed on your system use:: PYTHON_ISAL_LINK_DYNAMIC=true pip install isal --no-binary isal ISA-L is available in numerous Linux distro's as well as on conda via the conda-forge channel. Checkout the `ports documentation `_ on the ISA-L project wiki to find out how to install it. It is important that the development headers are also installed. On Debian and Ubuntu the ISA-L libraries (including the development headers) can be installed with:: sudo apt install libisal-dev Installation via conda ---------------------- Python-isal can be installed via conda, for example using the `miniconda `_ installer with a properly setup `conda-forge `_ channel. When used with bioinformatics tools setting up `bioconda `_ provides a clear set of installation instructions for conda. python-isal is available on conda-forge and can be installed with:: conda install python-isal This will automatically install the ISA-L library dependency as well, since it is available on conda-forge. =========================================== python-isal as a dependency in your project =========================================== .. include:: includes/README.rst :start-after: .. dependency start :end-before: .. dependency end .. _differences-with-zlib-and-gzip-modules: ====================================== Differences with zlib and gzip modules ====================================== .. include:: includes/README.rst :start-after: .. differences start :end-before: .. differences end ============================ API Documentation: isal_zlib ============================ .. automodule:: isal.isal_zlib :members: .. autoclass:: Compress :members: .. autoclass:: Decompress :members: ======================== API-documentation: igzip ======================== .. automodule:: isal.igzip :members: compress, decompress, open, BadGzipFile, GzipFile, READ_BUFFER_SIZE .. autoclass:: IGzipFile :members: :special-members: __init__ ================================= API-documentation: igzip_threaded ================================= .. automodule:: isal.igzip_threaded :members: open ============================ API Documentation: igzip_lib ============================ .. automodule:: isal.igzip_lib :members: compress, decompress, .. autoclass:: IgzipDecompressor :members: ========================== python -m isal.igzip usage ========================== .. argparse:: :module: isal.igzip :func: _argument_parser :prog: python -m isal.igzip ============ Contributing ============ .. include:: includes/README.rst :start-after: .. contributing start :end-before: .. contributing end =========== Development =========== .. include:: includes/README.rst :start-after: .. development start :end-before: .. development end ================ Acknowledgements ================ .. include:: includes/README.rst :start-after: .. acknowledgements start :end-before: .. acknowledgements end .. include:: includes/CHANGELOG.rst python-isal-1.7.2/docs/make.bat000066400000000000000000000014331476203621500163340ustar00rootroot00000000000000@ECHO OFF pushd %~dp0 REM Command file for Sphinx documentation if "%SPHINXBUILD%" == "" ( set SPHINXBUILD=sphinx-build ) set SOURCEDIR=. set BUILDDIR=_build if "%1" == "" goto help %SPHINXBUILD% >NUL 2>NUL if errorlevel 9009 ( echo. echo.The 'sphinx-build' command was not found. Make sure you have Sphinx echo.installed, then set the SPHINXBUILD environment variable to point echo.to the full path of the 'sphinx-build' executable. Alternatively you echo.may add the Sphinx directory to PATH. echo. echo.If you don't have Sphinx installed, grab it from echo.http://sphinx-doc.org/ exit /b 1 ) %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% goto end :help %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% :end popd python-isal-1.7.2/pyproject.toml000066400000000000000000000003421476203621500167110ustar00rootroot00000000000000[build-system] requires = ["setuptools>=64", "versioningit>=1.1.0"] build-backend = "setuptools.build_meta" [tool.versioningit.vcs] method="git" default-tag = "v0.0.0" [tool.versioningit.write] file = "src/isal/_version.py" python-isal-1.7.2/requirements-docs.txt000066400000000000000000000001371476203621500202110ustar00rootroot00000000000000# https://github.com/sphinx-doc/sphinx/issues/13415 sphinx <8 sphinx-rtd-theme sphinx-argparse python-isal-1.7.2/setup.cfg000066400000000000000000000000401476203621500156110ustar00rootroot00000000000000[metadata] license_files=LICENSEpython-isal-1.7.2/setup.py000066400000000000000000000171551476203621500155210ustar00rootroot00000000000000# Copyright (c) 2020 Leiden University Medical Center # Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, # 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022 # Python Software Foundation; All Rights Reserved # This file is part of python-isal which is distributed under the # PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2. import functools import os import platform import shutil import subprocess import sys import tempfile from pathlib import Path from setuptools import Extension, find_packages, setup from setuptools.command.build_ext import build_ext import versioningit ISA_L_SOURCE = os.path.join("src", "isal", "isa-l") SYSTEM_IS_BSD = (sys.platform.startswith("freebsd") or sys.platform.startswith("netbsd")) SYSTEM_IS_UNIX = (sys.platform.startswith("linux") or sys.platform.startswith("darwin") or sys.platform.startswith("gnu") or SYSTEM_IS_BSD) SYSTEM_IS_WINDOWS = sys.platform.startswith("win") # Since pip builds in a temp directory by default, setting a fixed file in # /tmp works during the entire session. DEFAULT_CACHE_FILE = Path(tempfile.gettempdir() ).absolute() / ".isal_build_cache" BUILD_CACHE = os.environ.get("PYTHON_ISAL_BUILD_CACHE") BUILD_CACHE_FILE = Path(os.environ.get("PYTHON_ISAL_BUILD_CACHE_FILE", DEFAULT_CACHE_FILE)) EXTENSIONS = [ Extension("isal.isal_zlib", ["src/isal/isal_zlibmodule.c"]), Extension("isal.igzip_lib", ["src/isal/igzip_libmodule.c"]), Extension("isal._isal", ["src/isal/_isalmodule.c"]), ] class BuildIsalExt(build_ext): def build_extension(self, ext): # Add option to link dynamically for packaging systems such as conda. # Always link dynamically on readthedocs to simplify install. if (os.getenv("PYTHON_ISAL_LINK_DYNAMIC") is not None or os.environ.get("READTHEDOCS") is not None): # Check for isa-l include directories. This is useful when # installing in a conda environment. possible_prefixes = [sys.exec_prefix, sys.base_exec_prefix] for prefix in possible_prefixes: if Path(prefix, "include", "isa-l").exists(): ext.include_dirs = [os.path.join(prefix, "include")] ext.library_dirs = [os.path.join(prefix, "lib")] break # Only one include directory is needed. # On windows include is in Library apparently elif Path(prefix, "Library", "include", "isa-l").exists(): ext.include_dirs = [os.path.join(prefix, "Library", "include")] ext.library_dirs = [os.path.join(prefix, "Library", "lib")] break if SYSTEM_IS_UNIX: ext.libraries = ["isal"] # libisal.so* elif SYSTEM_IS_WINDOWS: ext.libraries = ["isa-l"] # isa-l.dll else: raise NotImplementedError( f"Unsupported platform: {sys.platform}") else: isa_l_build_dir = build_isa_l() if SYSTEM_IS_UNIX: ext.extra_objects = [ os.path.join(isa_l_build_dir, "bin", "isa-l.a")] elif SYSTEM_IS_WINDOWS: ext.extra_objects = [ os.path.join(isa_l_build_dir, "isa-l_static.lib")] else: raise NotImplementedError( f"Unsupported platform: {sys.platform}") ext.include_dirs = [isa_l_build_dir] super().build_extension(ext) # Use a cache to prevent isa-l from being build twice. According to the # functools docs lru_cache with maxsize None is faster. The shortcut called # 'cache' is only available from python 3.9 onwards. # see: https://docs.python.org/3/library/functools.html#functools.cache @functools.lru_cache(maxsize=None) def build_isa_l(): # Check for cache if BUILD_CACHE: if BUILD_CACHE_FILE.exists(): cache_path = Path(BUILD_CACHE_FILE.read_text()) if (cache_path / "isa-l.h").exists(): return str(cache_path) # Creating temporary directories build_dir = tempfile.mktemp() shutil.copytree(ISA_L_SOURCE, build_dir) # Build environment is a copy of OS environment to allow user to influence # it. build_env = os.environ.copy() if SYSTEM_IS_UNIX: build_env["CFLAGS"] = build_env.get("CFLAGS", "") + " -fPIC" if hasattr(os, "sched_getaffinity"): cpu_count = len(os.sched_getaffinity(0)) else: # sched_getaffinity not available on all platforms cpu_count = os.cpu_count() or 1 # os.cpu_count() can return None run_args = dict(cwd=build_dir, env=build_env) if SYSTEM_IS_UNIX: if platform.machine() == "aarch64": cflags_param = "CFLAGS_aarch64" else: cflags_param = "CFLAGS_" make_cmd = "make" if SYSTEM_IS_BSD: make_cmd = "gmake" subprocess.run([make_cmd, "-j", str(cpu_count), "-f", "Makefile.unx", "isa-l.h", "bin/isa-l.a", f"{cflags_param}={build_env.get('CFLAGS', '')}"], **run_args) elif SYSTEM_IS_WINDOWS: subprocess.run(["nmake", "/f", "Makefile.nmake"], **run_args) else: raise NotImplementedError(f"Unsupported platform: {sys.platform}") shutil.copytree(os.path.join(build_dir, "include"), os.path.join(build_dir, "isa-l")) if BUILD_CACHE: BUILD_CACHE_FILE.write_text(build_dir) return build_dir setup( name="isal", version=versioningit.get_version(), description="Faster zlib and gzip compatible compression and " "decompression by providing python bindings for the ISA-L " "library.", author="Leiden University Medical Center", author_email="r.h.p.vorderman@lumc.nl", # A placeholder for now long_description=Path("README.rst").read_text(), long_description_content_type="text/x-rst", cmdclass={"build_ext": BuildIsalExt}, license="PSF-2.0", keywords="isal isa-l compression deflate gzip igzip threads", zip_safe=False, packages=find_packages('src'), package_dir={'': 'src'}, package_data={'isal': ['*.pyi', 'py.typed', # Include isa-l LICENSE and other relevant files # with the binary distribution. 'isa-l/LICENSE', 'isa-l/README.md', 'isa-l/Release_notes.txt']}, url="https://github.com/pycompression/python-isal", classifiers=[ "Programming Language :: Python :: 3 :: Only", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Programming Language :: C", "Development Status :: 5 - Production/Stable", "Topic :: System :: Archiving :: Compression", "License :: OSI Approved :: Python Software Foundation License", "Operating System :: POSIX :: Linux", "Operating System :: MacOS", "Operating System :: Microsoft :: Windows", ], python_requires=">=3.8", # BadGzipFile imported ext_modules=EXTENSIONS ) python-isal-1.7.2/src/000077500000000000000000000000001476203621500145655ustar00rootroot00000000000000python-isal-1.7.2/src/isal/000077500000000000000000000000001476203621500155155ustar00rootroot00000000000000python-isal-1.7.2/src/isal/__init__.py000066400000000000000000000011221476203621500176220ustar00rootroot00000000000000# Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, # 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022 # Python Software Foundation; All Rights Reserved # This file is part of python-isal which is distributed under the # PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2. from ._isal import (ISAL_MAJOR_VERSION, ISAL_MINOR_VERSION, ISAL_PATCH_VERSION, ISAL_VERSION) from ._version import __version__ __all__ = [ "ISAL_MAJOR_VERSION", "ISAL_MINOR_VERSION", "ISAL_PATCH_VERSION", "ISAL_VERSION", "__version__" ] python-isal-1.7.2/src/isal/_isal.pyi000066400000000000000000000006251476203621500173320ustar00rootroot00000000000000# Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, # 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022 # Python Software Foundation; All Rights Reserved # This file is part of python-isal which is distributed under the # PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2. ISAL_MAJOR_VERSION: int ISAL_MINOR_VERSION: int ISAL_PATCH_VERSION: int ISAL_VERSION: str python-isal-1.7.2/src/isal/_isalmodule.c000066400000000000000000000024041476203621500201560ustar00rootroot00000000000000/* Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022 Python Software Foundation; All Rights Reserved This file is part of python-isal which is distributed under the PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2. This file is not originally from the CPython distribution. But it does contain mostly example code from the Python docs. Also dual licensing just for this one file seemed silly. */ #define PY_SSIZE_T_CLEAN #include #include static struct PyModuleDef _isal_module = { PyModuleDef_HEAD_INIT, "_isal", /* name of module */ NULL, /* module documentation, may be NULL */ -1, NULL }; PyMODINIT_FUNC PyInit__isal(void) { PyObject *m = PyModule_Create(&_isal_module); if (m == NULL) { return NULL; } PyModule_AddIntMacro(m, ISAL_MAJOR_VERSION); PyModule_AddIntMacro(m, ISAL_MINOR_VERSION); PyModule_AddIntMacro(m, ISAL_PATCH_VERSION); PyObject *isal_version = PyUnicode_FromFormat( "%d.%d.%d", ISAL_MAJOR_VERSION, ISAL_MINOR_VERSION, ISAL_PATCH_VERSION); if (isal_version == NULL) { return NULL; } PyModule_AddObject(m, "ISAL_VERSION", isal_version); return m; } python-isal-1.7.2/src/isal/_version.pyi000066400000000000000000000005141476203621500200640ustar00rootroot00000000000000# Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, # 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022 # Python Software Foundation; All Rights Reserved # This file is part of python-isal which is distributed under the # PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2. __version__: str python-isal-1.7.2/src/isal/crc32_combine.h000066400000000000000000000053631476203621500203050ustar00rootroot00000000000000/* pigz.c -- parallel implementation of gzip * Copyright (C) 2007-2023 Mark Adler * Version 2.8 19 Aug 2023 Mark Adler */ /* This software is provided 'as-is', without any express or implied warranty. In no event will the author be held liable for any damages arising from the use of this software. Permission is granted to anyone to use this software for any purpose, including commercial applications, and to alter it and redistribute it freely, subject to the following restrictions: 1. The origin of this software must not be misrepresented; you must not claim that you wrote the original software. If you use this software in a product, an acknowledgment in the product documentation would be appreciated but is not required. 2. Altered source versions must be plainly marked as such, and must not be misrepresented as being the original software. 3. This notice may not be removed or altered from any source distribution. Mark Adler madler@alumni.caltech.edu */ /* Alterations from original: - typedef for crc_t - local declarations replaced with static inline - g.block selector in crc32_comb removed */ #include #include typedef uint32_t crc_t; // CRC-32 polynomial, reflected. #define POLY 0xedb88320 // Return a(x) multiplied by b(x) modulo p(x), where p(x) is the CRC // polynomial, reflected. For speed, this requires that a not be zero. static inline crc_t multmodp(crc_t a, crc_t b) { crc_t m = (crc_t)1 << 31; crc_t p = 0; for (;;) { if (a & m) { p ^= b; if ((a & (m - 1)) == 0) break; } m >>= 1; b = b & 1 ? (b >> 1) ^ POLY : b >> 1; } return p; } // Table of x^2^n modulo p(x). static const crc_t x2n_table[] = { 0x40000000, 0x20000000, 0x08000000, 0x00800000, 0x00008000, 0xedb88320, 0xb1e6b092, 0xa06a2517, 0xed627dae, 0x88d14467, 0xd7bbfe6a, 0xec447f11, 0x8e7ea170, 0x6427800e, 0x4d47bae0, 0x09fe548f, 0x83852d0f, 0x30362f1a, 0x7b5a9cc3, 0x31fec169, 0x9fec022a, 0x6c8dedc4, 0x15d6874d, 0x5fde7a4e, 0xbad90e37, 0x2e4e5eef, 0x4eaba214, 0xa8a472c0, 0x429a969e, 0x148d302a, 0xc40ba6d0, 0xc4e22c3c}; // Return x^(n*2^k) modulo p(x). static inline crc_t x2nmodp(size_t n, unsigned k) { crc_t p = (crc_t)1 << 31; // x^0 == 1 while (n) { if (n & 1) p = multmodp(x2n_table[k & 31], p); n >>= 1; k++; } return p; } // This uses the pre-computed g.shift value most of the time. Only the last // combination requires a new x2nmodp() calculation. static inline unsigned long crc32_comb(unsigned long crc1, unsigned long crc2, size_t len2) { return multmodp(x2nmodp(len2, 3), crc1) ^ crc2; } python-isal-1.7.2/src/isal/igzip.py000066400000000000000000000361301476203621500172140ustar00rootroot00000000000000# Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, # 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022 # Python Software Foundation; All Rights Reserved # This file is part of python-isal which is distributed under the # PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2. # This file uses code from CPython's Lib/gzip.py # Changes compared to CPython: # - Subclassed GzipFile to IGzipFile. Methods that included calls to zlib have # been overwritten with the same methods, but now calling to isal_zlib. # - _GzipReader is implemented in C in isal_zlib and allows dropping the GIL. # - Gzip.compress does not use a GzipFile to compress in memory, but creates a # simple header using _create_simple_gzip_header and compresses the data with # igzip_lib.compress using the DECOMP_GZIP_NO_HDR flag. This change was # ported to Python 3.11, using zlib.compress(wbits=-15) in that instance. # - Gzip.decompress creates an isal_zlib.decompressobj and decompresses the # data that way instead of using GzipFile. This change was ported to # Python 3.11. # - The main() function's gzip utility has now support for a -c flag for easier # use. """Similar to the stdlib gzip module. But using the Intel Storage Accelaration Library to speed up its methods.""" import argparse import builtins import gzip import io import os import shutil import struct import sys import time from typing import Optional, SupportsInt from . import igzip_lib, isal_zlib from .isal_zlib import _GzipReader __all__ = ["IGzipFile", "open", "compress", "decompress", "BadGzipFile", "READ_BUFFER_SIZE"] _COMPRESS_LEVEL_FAST = isal_zlib.ISAL_BEST_SPEED _COMPRESS_LEVEL_TRADEOFF = isal_zlib.ISAL_DEFAULT_COMPRESSION _COMPRESS_LEVEL_BEST = isal_zlib.ISAL_BEST_COMPRESSION # The amount of data that is read in at once when decompressing a file. # Increasing this value may increase performance. # After 512K the performance does not increase anymore on a Ryzen 5 3600 test # system. READ_BUFFER_SIZE = 512 * 1024 FTEXT, FHCRC, FEXTRA, FNAME, FCOMMENT = 1, 2, 4, 8, 16 READ = gzip.READ WRITE = gzip.WRITE BadGzipFile = gzip.BadGzipFile # type: ignore # The open method was copied from the CPython source with minor adjustments. def open(filename, mode="rb", compresslevel=_COMPRESS_LEVEL_TRADEOFF, encoding=None, errors=None, newline=None): """Open a gzip-compressed file in binary or text mode. This uses the isa-l library for optimized speed. The filename argument can be an actual filename (a str or bytes object), or an existing file object to read from or write to. The mode argument can be "r", "rb", "w", "wb", "x", "xb", "a" or "ab" for binary mode, or "rt", "wt", "xt" or "at" for text mode. The default mode is "rb", and the default compresslevel is 2. For binary mode, this function is equivalent to the GzipFile constructor: GzipFile(filename, mode, compresslevel). In this case, the encoding, errors and newline arguments must not be provided. For text mode, a GzipFile object is created, and wrapped in an io.TextIOWrapper instance with the specified encoding, error handling behavior, and line ending(s). """ if "t" in mode: if "b" in mode: raise ValueError("Invalid mode: %r" % (mode,)) else: if encoding is not None: raise ValueError( "Argument 'encoding' not supported in binary mode") if errors is not None: raise ValueError("Argument 'errors' not supported in binary mode") if newline is not None: raise ValueError("Argument 'newline' not supported in binary mode") gz_mode = mode.replace("t", "") # __fspath__ method is os.PathLike if isinstance(filename, (str, bytes)) or hasattr(filename, "__fspath__"): binary_file = IGzipFile(filename, gz_mode, compresslevel) elif hasattr(filename, "read") or hasattr(filename, "write"): binary_file = IGzipFile(None, gz_mode, compresslevel, filename) else: raise TypeError("filename must be a str or bytes object, or a file") if "t" in mode: return io.TextIOWrapper(binary_file, encoding, errors, newline) else: return binary_file class IGzipFile(gzip.GzipFile): """The IGzipFile class simulates most of the methods of a file object with the exception of the truncate() method. This class only supports opening files in binary mode. If you need to open a compressed file in text mode, use the gzip.open() function. """ def __init__(self, filename=None, mode=None, compresslevel=isal_zlib.ISAL_DEFAULT_COMPRESSION, fileobj=None, mtime=None): """Constructor for the IGzipFile class. At least one of fileobj and filename must be given a non-trivial value. The new class instance is based on fileobj, which can be a regular file, an io.BytesIO object, or any other object which simulates a file. It defaults to None, in which case filename is opened to provide a file object. When fileobj is not None, the filename argument is only used to be included in the gzip file header, which may include the original filename of the uncompressed file. It defaults to the filename of fileobj, if discernible; otherwise, it defaults to the empty string, and in this case the original filename is not included in the header. The mode argument can be any of 'r', 'rb', 'a', 'ab', 'w', 'wb', 'x', or 'xb' depending on whether the file will be read or written. The default is the mode of fileobj if discernible; otherwise, the default is 'rb'. A mode of 'r' is equivalent to one of 'rb', and similarly for 'w' and 'wb', 'a' and 'ab', and 'x' and 'xb'. The compresslevel argument is an integer from 0 to 3 controlling the level of compression; 0 is fastest and produces the least compression, and 3 is slowest and produces the most compression. Unlike gzip.GzipFile 0 is NOT no compression. The default is 2. The mtime argument is an optional numeric timestamp to be written to the last modification time field in the stream when compressing. If omitted or None, the current time is used. """ if not (isal_zlib.ISAL_BEST_SPEED <= compresslevel <= isal_zlib.ISAL_BEST_COMPRESSION) and "r" not in mode: raise ValueError( f"Compression level should be between " f"{isal_zlib.ISAL_BEST_SPEED} and " f"{isal_zlib.ISAL_BEST_COMPRESSION}, got {compresslevel}." ) super().__init__(filename, mode, compresslevel, fileobj, mtime) if self.mode == WRITE: self.compress = isal_zlib.compressobj(compresslevel, isal_zlib.DEFLATED, -isal_zlib.MAX_WBITS, isal_zlib.DEF_MEM_LEVEL, 0) if self.mode == READ: raw = _GzipReader(self.fileobj, READ_BUFFER_SIZE) self._buffer = io.BufferedReader(raw) def __repr__(self): s = repr(self.fileobj) return '' def _write_gzip_header(self, compresslevel=_COMPRESS_LEVEL_TRADEOFF): # Python 3.9 added a `compresslevel` parameter to write gzip header. # This only determines the value of one extra flag. Because this change # was backported to 3.7 and 3.8 in later point versions, the attributes # of the function should be checked before trying to use the # compresslevel parameter. # The gzip header has an extra flag that can be set depending on the # compression level used. This should be set when either the fastest or # best method is used. ISAL level 0 is larger than gzip level 1 and # much faster, so setting the flag for fastest level is appropriate. # ISAL level 1,2 and 3 (best)are similar in size and fall around the # gzip level 3 size. So setting no extra flag # (by using COMPRESS_LEVEL_TRADEOFF) is appropriate here. if ("compresslevel" in super()._write_gzip_header.__code__.co_varnames and hasattr(gzip, "_COMPRESS_LEVEL_FAST") and hasattr(gzip, "_COMPRESS_LEVEL_TRADEOFF")): if compresslevel == _COMPRESS_LEVEL_FAST: super()._write_gzip_header(gzip._COMPRESS_LEVEL_FAST) else: super()._write_gzip_header(gzip._COMPRESS_LEVEL_TRADEOFF) else: super()._write_gzip_header() def write(self, data): self._check_not_closed() if self.mode != WRITE: import errno raise OSError(errno.EBADF, "write() on read-only IGzipFile object") if self.fileobj is None: raise ValueError("write() on closed IGzipFile object") if isinstance(data, bytes): length = len(data) else: # accept any data that supports the buffer protocol data = memoryview(data) length = data.nbytes if length > 0: self.fileobj.write(self.compress.compress(data)) self.size += length self.crc = isal_zlib.crc32(data, self.crc) self.offset += length return length # Aliases for improved compatibility with CPython gzip module. GzipFile = IGzipFile _IGzipReader = _GzipReader def compress(data, compresslevel: int = _COMPRESS_LEVEL_BEST, *, mtime: Optional[SupportsInt] = None) -> bytes: """Compress data in one shot and return the compressed string. Optional argument is the compression level, in range of 0-3. """ if mtime is None: mtime = time.time() # There is no best compression level. ISA-L only provides algorithms for # fast and medium levels. xfl = 4 if compresslevel == _COMPRESS_LEVEL_FAST else 0 # Pack ID1 and ID2 magic bytes, method (8=deflate), header flags (no extra # fields added to header), mtime, xfl and os (255 for unknown OS). header = struct.pack("