pax_global_header00006660000000000000000000000064125462216510014516gustar00rootroot0000000000000052 comment=824f31b5401904d82a74aa86fbde18db9fc48a4a c-blosc-1.7.0/000077500000000000000000000000001254622165100130455ustar00rootroot00000000000000c-blosc-1.7.0/.editorconfig000066400000000000000000000004231254622165100155210ustar00rootroot00000000000000; Top-most EditorConfig file root = true ; Global settings [*] end_of_line = LF indent_style = space trim_trailing_whitespace = true insert_final_newline = true ; C source files [*.{h,c}] indent_size = 2 ; CMake [CMakeLists.txt] indent_size = 4 [*.cmake] indent_size = 4 c-blosc-1.7.0/.gitignore000066400000000000000000000000231254622165100150300ustar00rootroot00000000000000bench/bench build/ c-blosc-1.7.0/.mailmap000066400000000000000000000004351254622165100144700ustar00rootroot00000000000000Francesc Alted FrancescAlted Francesc Alted FrancescAlted Francesc Alted FrancescAlted Francesc Alted FrancescAlted c-blosc-1.7.0/.travis.yml000066400000000000000000000005361254622165100151620ustar00rootroot00000000000000language: c os: - linux - osx compiler: - gcc - clang before_install: ./scripts/travis-before-install.sh #install: sudo apt-get install libsnappy-dev zlib1g-dev #install: sudo apt-get install liblz4-dev libsnappy-dev zlib1g-dev before_script: - mkdir build - cd build - cmake .. script: - cmake --build . --config Release - ctest c-blosc-1.7.0/ANNOUNCE.rst000066400000000000000000000043271254622165100150130ustar00rootroot00000000000000=============================================================== Announcing c-blosc 1.7.0 A blocking, shuffling and lossless compression library for C =============================================================== What is new? ============ This is a quite big release introducing some exciting new features: * A new 'bitshuffle' filter is here. This is similar that the existing 'shuffle' filter, but the shuffle takes place at bit level, and not at byte level. With it you can expect higher compression ratios but still having pretty good speed. For more info, see: http://blosc.org/blog/new-bitshuffle-filter.html * Implemented a new acceleration mode for LZ4 (updated to 1.7.0) and BloscLZ codecs that enters in operation with all compression levels except for the highest (9). This allows for an important boost in speed with minimal compression ratio loss. * Jack Pappas made great contributions allowing SSE2 operation in more scenarios (like types larger than 16 bytes or buffers not being a multiple of typesize * vectorsize). Another contribution is a much more comprehensive test suite for SSE2 and AVX2 operation. * Zbyszek Szmek fixed compilation on non-Intel archs (tested on ARM). For more info, please see the release notes in: https://github.com/Blosc/c-blosc/wiki/Release-notes What is it? =========== Blosc (http://www.blosc.org) is a high performance meta-compressor optimized for binary data. It has been designed to transmit data to the processor cache faster than the traditional, non-compressed, direct memory fetch approach via a memcpy() OS call. Blosc has internal support for different compressors like its internal BloscLZ, but also LZ4, LZ4HC, Snappy and Zlib. This way these can automatically leverage the multithreading and pre-filtering (shuffling) capabilities that comes with Blosc. Download sources ================ Please go to main web site: http://www.blosc.org/ and proceed from there. The github repository is over here: https://github.com/Blosc Blosc is distributed using the MIT license, see LICENSES/BLOSC.txt for details. Mailing list ============ There is an official Blosc mailing list at: blosc@googlegroups.com http://groups.google.es/group/blosc Enjoy Data! c-blosc-1.7.0/CMakeLists.txt000066400000000000000000000262411254622165100156120ustar00rootroot00000000000000# CMake build system for Blosc # ============================ # # Available options: # # BUILD_STATIC: default ON # build the static version of the Blosc library # BUILD_TESTS: default ON # build test programs and generates the "test" target # BUILD_BENCHMARKS: default ON # build the benchmark program # DEACTIVATE_LZ4: default OFF # do not include support for the LZ4 library # DEACTIVATE_SNAPPY: default OFF # do not include support for the Snappy library # DEACTIVATE_ZLIB: default OFF # do not include support for the Zlib library # PREFER_EXTERNAL_LZ4: default OFF # when found, use the installed LZ4 libs instead of included # sources # PREFER_EXTERNAL_SNAPPY: default ON # when found, use the installed Snappy libs instead of included # sources # PREFER_EXTERNAL_ZLIB: default ON # when found, use the installed zlib libs instead of included # sources # TEST_INCLUDE_BENCH_SHUFFLE_1: default ON # add a test that runs the benchmark program passing "shuffle" with 1 # thread as second parameter # TEST_INCLUDE_BENCH_SHUFFLE_N: default ON # add a test that runs the benchmark program passing "shuffle" with all # threads as second parameter # TEST_INCLUDE_BENCH_BITSHUFFLE_1: default ON # add a test that runs the benchmark program passing "bitshuffle" with 1 # thread as second parameter # TEST_INCLUDE_BENCH_BITSHUFFLE_N: default ON # add a test that runs the benchmark program passing "bitshuffle" with # all threads as second parameter # TEST_INCLUDE_BENCH_SUITE: default OFF # add a test that runs the benchmark program passing "suite" # as first parameter # TEST_INCLUDE_BENCH_SUITE_PARALLEL: default OFF # add a test that runs the benchmark program passing "parallel" # as first parameter # TEST_INCLUDE_BENCH_HARDSUITE: default OFF # add a test that runs the benchmark program passing "hardsuite" # as first parameter # TEST_INCLUDE_BENCH_EXTREMESUITE: default OFF # add a test that runs the benchmark program passing "extremesuite" # as first parameter # TEST_INCLUDE_BENCH_DEBUGSUITE: default OFF # add a test that runs the benchmark program passing "debugsuite" # as first parameter # # Components: # # LIB: includes blosc.so # DEV: static includes blosc.a and blosc.h cmake_minimum_required(VERSION 2.8.10) project(blosc) # parse the full version numbers from blosc.h file(READ ${CMAKE_CURRENT_SOURCE_DIR}/blosc/blosc.h _blosc_h_contents) string(REGEX REPLACE ".*#define[ \t]+BLOSC_VERSION_MAJOR[ \t]+([0-9]+).*" "\\1" BLOSC_VERSION_MAJOR ${_blosc_h_contents}) string(REGEX REPLACE ".*#define[ \t]+BLOSC_VERSION_MINOR[ \t]+([0-9]+).*" "\\1" BLOSC_VERSION_MINOR ${_blosc_h_contents}) string(REGEX REPLACE ".*#define[ \t]+BLOSC_VERSION_RELEASE[ \t]+([0-9]+).*" "\\1" BLOSC_VERSION_PATCH ${_blosc_h_contents}) string(REGEX REPLACE ".*#define[ \t]+BLOSC_VERSION_STRING[ \t]+\"([-0-9A-Za-z.]+)\".*" "\\1" BLOSC_VERSION_STRING ${_blosc_h_contents}) message("Configuring for Blosc version: " ${BLOSC_VERSION_STRING}) # options option(BUILD_STATIC "Build a static version of the blosc library." ON) option(BUILD_TESTS "Build test programs form the blosc compression library" ON) option(BUILD_BENCHMARKS "Build benchmark programs form the blosc compression library" ON) option(DEACTIVATE_LZ4 "Do not include support for the LZ4 library." OFF) option(DEACTIVATE_SNAPPY "Do not include support for the SNAPPY library." OFF) option(DEACTIVATE_ZLIB "Do not include support for the ZLIB library." OFF) option(PREFER_EXTERNAL_LZ4 "Find and use external LZ4 library instead of included sources." OFF) option(PREFER_EXTERNAL_SNAPPY "Find and use external Snappy library instead of included sources." ON) option(PREFER_EXTERNAL_ZLIB "Find and use external zlib library instead of included sources." ON) set(CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake") if(NOT DEACTIVATE_LZ4) if(PREFER_EXTERNAL_LZ4) find_package(LZ4) else() message(STATUS "Using LZ4 internal sources.") endif(PREFER_EXTERNAL_LZ4) # HAVE_LZ4 will be set to true because even if the library is # not found, we will use the included sources for it set(HAVE_LZ4 TRUE) endif(NOT DEACTIVATE_LZ4) if(NOT DEACTIVATE_SNAPPY) if(PREFER_EXTERNAL_SNAPPY) find_package(Snappy) else() message(STATUS "Using Snappy internal sources.") endif(PREFER_EXTERNAL_SNAPPY) # HAVE_SNAPPY will be set to true because even if the library is not found, # we will use the included sources for it set(HAVE_SNAPPY TRUE) endif(NOT DEACTIVATE_SNAPPY) if(NOT DEACTIVATE_ZLIB) # import the ZLIB_ROOT environment variable to help finding the zlib library if(PREFER_EXTERNAL_ZLIB) set(ZLIB_ROOT $ENV{ZLIB_ROOT}) find_package(ZLIB) if (NOT ZLIB_FOUND ) message(STATUS "No zlib found. Using internal sources.") endif (NOT ZLIB_FOUND ) else() message(STATUS "Using zlib internal sources.") endif(PREFER_EXTERNAL_ZLIB) # HAVE_ZLIB will be set to true because even if the library is not found, # we will use the included sources for it set(HAVE_ZLIB TRUE) endif(NOT DEACTIVATE_ZLIB) # create the config.h file configure_file ("blosc/config.h.in" "blosc/config.h" ) # now make sure that you set the build directory on your "Include" path when compiling include_directories("${PROJECT_BINARY_DIR}/blosc/") # If the build type is not set, default to Release. set(BLOSC_DEFAULT_BUILD_TYPE Release) if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) message(STATUS "No build type specified. Defaulting to '${BLOSC_DEFAULT_BUILD_TYPE}'.") set(CMAKE_BUILD_TYPE ${BLOSC_DEFAULT_BUILD_TYPE} CACHE STRING "Choose the type of build." FORCE) # Set the possible values of build type for cmake-gui set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo") endif() # Based on the target system's processor and the compiler being used, # set build variables indicating which hardware features can be targeted # by the compiler. Note we DO NOT check which hardware features are supported # by this (the host) system, because we want to be able to support compiling # for newer hardware on older machines as well as cross-compilation. message(STATUS "Building for system processor ${CMAKE_SYSTEM_PROCESSOR}") if(CMAKE_SYSTEM_PROCESSOR STREQUAL i386 OR CMAKE_SYSTEM_PROCESSOR STREQUAL i686 OR CMAKE_SYSTEM_PROCESSOR STREQUAL x86_64 OR CMAKE_SYSTEM_PROCESSOR STREQUAL amd64 OR CMAKE_SYSTEM_PROCESSOR STREQUAL AMD64) if(CMAKE_C_COMPILER_ID STREQUAL GNU) set(COMPILER_SUPPORT_SSE2 TRUE) if(CMAKE_C_COMPILER_VERSION VERSION_GREATER 4.7 OR CMAKE_C_COMPILER_VERSION VERSION_EQUAL 4.7) set(COMPILER_SUPPORT_AVX2 TRUE) else() set(COMPILER_SUPPORT_AVX2 FALSE) endif() elseif(CMAKE_C_COMPILER_ID STREQUAL Clang) set(COMPILER_SUPPORT_SSE2 TRUE) if(CMAKE_C_COMPILER_VERSION VERSION_GREATER 3.2 OR CMAKE_C_COMPILER_VERSION VERSION_EQUAL 3.2) set(COMPILER_SUPPORT_AVX2 TRUE) else() set(COMPILER_SUPPORT_AVX2 FALSE) endif() elseif(CMAKE_C_COMPILER_ID STREQUAL Intel) set(COMPILER_SUPPORT_SSE2 TRUE) if(CMAKE_C_COMPILER_VERSION VERSION_GREATER 14.0 OR CMAKE_C_COMPILER_VERSION VERSION_EQUAL 14.0) # icc (ICC) 15.0.3 does not work compiling AVX2 code # (perhaps my machine does not have AVX2 and the compiler # cannot generate code for that?) set(COMPILER_SUPPORT_AVX2 FALSE) else() set(COMPILER_SUPPORT_AVX2 FALSE) endif() elseif(MSVC) set(COMPILER_SUPPORT_SSE2 TRUE) if(CMAKE_C_COMPILER_VERSION VERSION_GREATER 18.00.30501 OR CMAKE_C_COMPILER_VERSION VERSION_EQUAL 18.00.30501) set(COMPILER_SUPPORT_AVX2 TRUE) else() set(COMPILER_SUPPORT_AVX2 FALSE) endif() else() set(COMPILER_SUPPORT_SSE2 FALSE) set(COMPILER_SUPPORT_AVX2 FALSE) # Unrecognized compiler. Emit a warning message to let the user know hardware-acceleration won't be available. message(WARNING "Unable to determine which ${CMAKE_SYSTEM_PROCESSOR} hardware features are supported by the C compiler (${CMAKE_C_COMPILER_ID} ${CMAKE_C_COMPILER_VERSION}).") endif() else() # If the target system processor isn't recognized, emit a warning message to alert the user # that hardware-acceleration support won't be available but allow configuration to proceed. message(WARNING "Unrecognized system processor ${CMAKE_SYSTEM_PROCESSOR}. Cannot determine which hardware features (${CMAKE_C_COMPILER_ID} ${CMAKE_C_COMPILER_VERSION}) supports, so hardware-accelerated implementations will not be available.") endif() # flags # @TODO: set -Wall # @NOTE: -O3 is enabled in Release mode (CMAKE_BUILD_TYPE="Release") # Set the "-msse2" build flag only if the CMAKE_C_FLAGS is not already set. # Probably "-msse2" should be appended to CMAKE_C_FLAGS_RELEASE. if(CMAKE_C_COMPILER_ID STREQUAL GNU OR CMAKE_C_COMPILER_ID STREQUAL Clang OR CMAKE_C_COMPILER_ID STREQUAL Intel) if(NOT CMAKE_C_FLAGS AND COMPILER_SUPPORT_SSE2) set(CMAKE_C_FLAGS -msse2 CACHE STRING "C flags." FORCE) endif(NOT CMAKE_C_FLAGS AND COMPILER_SUPPORT_SSE2) endif(CMAKE_C_COMPILER_ID STREQUAL GNU OR CMAKE_C_COMPILER_ID STREQUAL Clang OR CMAKE_C_COMPILER_ID STREQUAL Intel) if(MSVC) if(NOT CMAKE_C_FLAGS) set(CMAKE_C_FLAGS "/Ox" CACHE STRING "C flags." FORCE) endif(NOT CMAKE_C_FLAGS) # Turn off misguided "secure CRT" warnings in MSVC. # Microsoft wants people to use the MS-specific _s # versions of certain C functions but this is difficult to do # in platform-independent code. add_definitions( -D_CRT_SECURE_NO_WARNINGS ) endif(MSVC) if(WIN32) # For some supporting headers include_directories("${CMAKE_CURRENT_SOURCE_DIR}/blosc") endif(WIN32) # subdirectories add_subdirectory(blosc) if(BUILD_TESTS) enable_testing() add_subdirectory(tests) endif(BUILD_TESTS) if(BUILD_BENCHMARKS) add_subdirectory(bench) endif(BUILD_BENCHMARKS) # uninstall target configure_file( "${CMAKE_CURRENT_SOURCE_DIR}/cmake_uninstall.cmake.in" "${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake" IMMEDIATE @ONLY) add_custom_target(uninstall COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake) # packaging include(InstallRequiredSystemLibraries) set(CPACK_GENERATOR TGZ ZIP) set(CPACK_SOURCE_GENERATOR TGZ ZIP) set(CPACK_PACKAGE_VERSION_MAJOR ${BLOSC_VERSION_MAJOR}) set(CPACK_PACKAGE_VERSION_MINOR ${BLOSC_VERSION_MINOR}) set(CPACK_PACKAGE_VERSION_PATCH ${BLOSC_VERSION_PATCH}) set(CPACK_PACKAGE_VERSION ${BLOSC_STRING_VERSION}) set(CPACK_PACKAGE_DESCRIPTION_FILE "${CMAKE_CURRENT_SOURCE_DIR}/README.rst") set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "A blocking, shuffling and lossless compression library") set(CPACK_RESOURCE_FILE_LICENSE "${CMAKE_CURRENT_SOURCE_DIR}/LICENSES/BLOSC.txt") set(CPACK_SOURCE_IGNORE_FILES "/build.*;.*~;\\\\.git.*;\\\\.DS_Store") set(CPACK_STRIP_FILES TRUE) set(CPACK_SOURCE_STRIP_FILES TRUE) include(CPack) c-blosc-1.7.0/LICENSES/000077500000000000000000000000001254622165100142525ustar00rootroot00000000000000c-blosc-1.7.0/LICENSES/BITSHUFFLE.txt000066400000000000000000000021741254622165100164520ustar00rootroot00000000000000Bitshuffle - Filter for improving compression of typed binary data. Copyright (c) 2014 Kiyoshi Masui (kiyo@physics.ubc.ca) Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. c-blosc-1.7.0/LICENSES/BLOSC.txt000066400000000000000000000021741254622165100156610ustar00rootroot00000000000000Blosc - A blocking, shuffling and lossless compression library Copyright (C) 2009-2015 Francesc Alted Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. c-blosc-1.7.0/LICENSES/FASTLZ.txt000066400000000000000000000023121254622165100160140ustar00rootroot00000000000000FastLZ - lightning-fast lossless compression library Copyright (C) 2007 Ariya Hidayat (ariya@kde.org) Copyright (C) 2006 Ariya Hidayat (ariya@kde.org) Copyright (C) 2005 Ariya Hidayat (ariya@kde.org) Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. c-blosc-1.7.0/LICENSES/LZ4.txt000066400000000000000000000030101254622165100154160ustar00rootroot00000000000000LZ4 - Fast LZ compression algorithm Copyright (C) 2011-2014, Yann Collet. BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html - LZ4 source repository : http://code.google.com/p/lz4/ c-blosc-1.7.0/LICENSES/SNAPPY.txt000066400000000000000000000027031254622165100160270ustar00rootroot00000000000000Copyright 2011, Google Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Google Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. c-blosc-1.7.0/LICENSES/STDINT.txt000066400000000000000000000030401254622165100160150ustar00rootroot00000000000000ISO C9x compliant stdint.h for Microsoft Visual Studio Based on ISO/IEC 9899:TC2 Committee draft (May 6, 2005) WG14/N1124 Copyright (c) 2006-2013 Alexander Chemeris Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. Neither the name of the product nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. c-blosc-1.7.0/LICENSES/ZLIB.txt000066400000000000000000000017521254622165100155600ustar00rootroot00000000000000Copyright notice: (C) 1995-2013 Jean-loup Gailly and Mark Adler This software is provided 'as-is', without any express or implied warranty. In no event will the authors be held liable for any damages arising from the use of this software. Permission is granted to anyone to use this software for any purpose, including commercial applications, and to alter it and redistribute it freely, subject to the following restrictions: 1. The origin of this software must not be misrepresented; you must not claim that you wrote the original software. If you use this software in a product, an acknowledgment in the product documentation would be appreciated but is not required. 2. Altered source versions must be plainly marked as such, and must not be misrepresented as being the original software. 3. This notice may not be removed or altered from any source distribution. Jean-loup Gailly Mark Adler jloup@gzip.org madler@alumni.caltech.edu c-blosc-1.7.0/README.rst000066400000000000000000000254621254622165100145450ustar00rootroot00000000000000=============================================================== Blosc: A blocking, shuffling and lossless compression library =============================================================== :Author: Francesc Alted :Contact: francesc@blosc.org :URL: http://www.blosc.org :Travis CI: |travis| .. |travis| image:: https://travis-ci.org/Blosc/c-blosc.svg?branch=master :target: https://travis-ci.org/Blosc/c-blosc :Appveyor: |appveyor| .. |appveyor| image:: https://ci.appveyor.com/api/projects/status/gccmb03j8ghbj0ig/branch/master?svg=true :target: https://ci.appveyor.com/project/FrancescAlted/c-blosc/branch/master What is it? =========== Blosc [1]_ is a high performance compressor optimized for binary data. It has been designed to transmit data to the processor cache faster than the traditional, non-compressed, direct memory fetch approach via a memcpy() OS call. Blosc is the first compressor (that I'm aware of) that is meant not only to reduce the size of large datasets on-disk or in-memory, but also to accelerate memory-bound computations. It uses the blocking technique (as described in [2]_) to reduce activity on the memory bus as much as possible. In short, this technique works by dividing datasets in blocks that are small enough to fit in caches of modern processors and perform compression / decompression there. It also leverages, if available, SIMD instructions (SSE2) and multi-threading capabilities of CPUs, in order to accelerate the compression / decompression process to a maximum. Blosc is actually a metacompressor, that meaning that it can use a range of compression libraries for performing the actual compression/decompression. Right now, it comes with integrated support for BloscLZ (the original one), LZ4, LZ4HC, Snappy and Zlib. Blosc comes with full sources for all compressors, so in case it does not find the libraries installed in your system, it will compile from the included sources and they will be integrated into the Blosc library anyway. That means that you can trust in having all supported compressors integrated in Blosc in all supported platforms. You can see some benchmarks about Blosc performance in [3]_ Blosc is distributed using the MIT license, see LICENSES/BLOSC.txt for details. .. [1] http://www.blosc.org .. [2] http://blosc.org/docs/StarvingCPUs-CISE-2010.pdf .. [3] http://blosc.org/synthetic-benchmarks.html Meta-compression and other advantages over existing compressors =============================================================== Blosc is not like other compressors: it should rather be called a meta-compressor. This is so because it can use different compressors and pre-conditioners (programs that generally improve compression ratio). At any rate, it can also be called a compressor because it happens that it already integrates one compressor and one pre-conditioner, so it can actually work like so. Currently it comes with support of BloscLZ, a compressor heavily based on FastLZ (http://fastlz.org/), LZ4 and LZ4HC (http://fastcompression.blogspot.com.es/p/lz4.html), Snappy (https://github.com/google/snappy) and Zlib (http://www.zlib.net/), as well as a highly optimized (it can use SSE2 instructions, if available) Shuffle pre-conditioner (for info on how it works, see slide 17 of http://www.slideshare.net/PyData/blosc-py-data-2014). However, different compressors or pre-conditioners may be added in the future. Blosc is in charge of coordinating the compressor and pre-conditioners so that they can leverage the blocking technique (described above) as well as multi-threaded execution (if several cores are available) automatically. That makes that every compressor and pre-conditioner will work at very high speeds, even if it was not initially designed for doing blocking or multi-threading. Other advantages of Blosc are: * Meant for binary data: can take advantage of the type size meta-information for improved compression ratio (using the integrated shuffle pre-conditioner). * Small overhead on non-compressible data: only a maximum of (16 + 4 * nthreads) additional bytes over the source buffer length are needed to compress *every* input. * Maximum destination length: contrary to many other compressors, both compression and decompression routines have support for maximum size lengths for the destination buffer. When taken together, all these features set Blosc apart from other similar solutions. Compiling your application with a minimalistic Blosc ==================================================== The minimal Blosc consists of the next files (in `blosc/ directory `_):: blosc.h and blosc.c -- the main routines shuffle.h and shuffle.c -- the shuffle code blosclz.h and blosclz.c -- the blosclz compressor Just add these files to your project in order to use Blosc. For information on compression and decompression routines, see `blosc.h `_. To compile using GCC (4.4 or higher recommended) on Unix: .. code-block:: console $ gcc -O3 -msse2 -o myprog myprog.c blosc/*.c -Iblosc -lpthread Using Windows and MINGW: .. code-block:: console $ gcc -O3 -msse2 -o myprog myprog.c -Iblosc blosc\*.c Using Windows and MSVC (2010 or higher recommended): .. code-block:: console $ cl /Ox /Femyprog.exe /Iblosc myprog.c blosc\*.c In the `examples/ directory `_ you can find more hints on how to link your app with Blosc. I have not tried to compile this with compilers other than GCC, clang, MINGW, Intel ICC or MSVC yet. Please report your experiences with your own platforms. Adding support for other compressors with a minimalistic Blosc ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The official cmake files (see below) for Blosc try hard to include support for LZ4, LZ4HC, Snappy, Zlib inside the Blosc library, so using them is just a matter of calling the appropriate `blosc_set_compressor() API call `_. See an `example here `_. Having said this, it is also easy to use a minimalistic Blosc and just add the symbols HAVE_LZ4 (will include both LZ4 and LZ4HC), HAVE_SNAPPY and HAVE_ZLIB during compilation as well as the appropriate libraries. For example, for compiling with minimalistic Blosc but with added Zlib support do: .. code-block:: console $ gcc -O3 -msse2 -o myprog myprog.c blosc/*.c -Iblosc -lpthread -DHAVE_ZLIB -lz In the `bench/ directory `_ there a couple of Makefile files (one for UNIX and the other for MinGW) with more complete building examples, like switching between libraries or internal sources for the compressors. Compiling the Blosc library with CMake ====================================== Blosc can also be built, tested and installed using CMake_. Although this procedure is a bit more invloved than the one described above, it is the most general because it allows to integrate other compressors than BloscLZ either from libraries or from internal sources. Hence, serious library developers should use this way. The following procedure describes the "out of source" build. Create the build directory and move into it: .. code-block:: console $ mkdir build $ cd build Now run CMake configuration and optionally specify the installation directory (e.g. '/usr' or '/usr/local'): .. code-block:: console $ cmake -DCMAKE_INSTALL_PREFIX=your_install_prefix_directory .. CMake allows to configure Blosc in many different ways, like prefering internal or external sources for compressors or enabling/disabling them. Please note that configuration can also be performed using UI tools provided by CMake_ (ccmake or cmake-gui): .. code-block:: console $ ccmake .. # run a curses-based interface $ cmake-gui .. # run a graphical interface Build, test and install Blosc: .. code-block:: console $ cmake --build . $ ctest $ cmake --build . --target install The static and dynamic version of the Blosc library, together with header files, will be installed into the specified CMAKE_INSTALL_PREFIX. .. _CMake: http://www.cmake.org Once you have compiled your Blosc library, you can easily link your apps with it as shown in the `example/ directory `_. Adding support for other compressors (LZ4, LZ4HC, Snappy, Zlib) with CMake ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The CMake files in Blosc are configured to automatically detect other compressors like LZ4, LZ4HC, Snappy or Zlib by default. So as long as the libraries and the header files for these libraries are accessible, these will be used by default. See an `example here `_. *Note on Zlib*: the library should be easily found on UNIX systems, although on Windows, you can help CMake to find it by setting the environment variable 'ZLIB_ROOT' to where zlib 'include' and 'lib' directories are. Also, make sure that Zlib DDL library is in your '\Windows' directory. However, the full sources for LZ4, LZ4HC, Snappy and Zlib have been included in Blosc too. So, in general, you should not worry about not having (or CMake not finding) the libraries in your system because in this case, their sources will be automatically compiled for you. That effectively means that you can be confident in having a complete support for all the supported compression libraries in all supported platforms. If you want to force Blosc to use external libraries instead of the included compression sources: .. code-block:: console $ cmake -DPREFER_EXTERNAL_LZ4=ON .. You can also disable support for some compression libraries: .. code-block:: console $ cmake -DDEACTIVATE_SNAPPY=ON .. Mac OSX troubleshooting ======================= If you run into compilation troubles when using Mac OSX, please make sure that you have installed the command line developer tools. You can always install them with: .. code-block:: console $ xcode-select --install Wrapper for Python ================== Blosc has an official wrapper for Python. See: https://github.com/Blosc/python-blosc Command line interface and serialization format for Blosc ========================================================= Blosc can be used from command line by using Bloscpack. See: https://github.com/Blosc/bloscpack Filter for HDF5 =============== For those who want to use Blosc as a filter in the HDF5 library, there is a sample implementation in the blosc/hdf5 project in: https://github.com/Blosc/hdf5 Mailing list ============ There is an official mailing list for Blosc at: blosc@googlegroups.com http://groups.google.es/group/blosc Acknowledgments =============== See THANKS.rst. ---- **Enjoy data!** c-blosc-1.7.0/README_HEADER.rst000066400000000000000000000033231254622165100155450ustar00rootroot00000000000000Blosc Header Format =================== Blosc (as of Version 1.0.0) has the following 16 byte header that stores information about the compressed buffer:: |-0-|-1-|-2-|-3-|-4-|-5-|-6-|-7-|-8-|-9-|-A-|-B-|-C-|-D-|-E-|-F-| ^ ^ ^ ^ | nbytes | blocksize | ctbytes | | | | | | | | +--typesize | | +------flags | +----------versionlz +--------------version Datatypes of the Header Entries ------------------------------- All entries are little endian. :version: (``uint8``) Blosc format version. :versionlz: (``uint8``) Version of the internal compressor used. :flags and compressor enumeration: (``bitfield``) The flags of the buffer :bit 0 (``0x01``): Whether the byte-shuffle filter has been applied or not. :bit 1 (``0x02``): Whether the internal buffer is a pure memcpy or not. :bit 2 (``0x04``): Whether the bit-shuffle filter has been applied or not. :bit 3 (``0x08``): Reserved :bit 4 (``0x16``): Reserved :bit 5 (``0x32``): Part of the enumeration for compressors. :bit 6 (``0x64``): Part of the enumeration for compressors. :bit 7 (``0x64``): Part of the enumeration for compressors. The last three bits form an enumeration that allows to use alternative compressors. :``0``: ``blosclz`` :``1``: ``lz4`` or ``lz4hc`` :``2``: ``snappy`` :``3``: ``zlib`` :typesize: (``uint8``) Number of bytes for the atomic type. :nbytes: (``uint32``) Uncompressed size of the buffer. :blocksize: (``uint32``) Size of internal blocks. :ctbytes: (``uint32``) Compressed size of the buffer. c-blosc-1.7.0/README_THREADED.rst000066400000000000000000000030411254622165100157720ustar00rootroot00000000000000Blosc supports threading ======================== Threads are the most efficient way to program parallel code for multi-core processors, but also the more difficult to program well. Also, they has a non-negligible start-up time that does not fit well with a high-performance compressor as Blosc tries to be. In order to reduce the overhead of threads as much as possible, I've decided to implement a pool of threads (the workers) that are waiting for the main process (the master) to send them jobs (basically, compressing and decompressing small blocks of the initial buffer). Despite this and many other internal optimizations in the threaded code, it does not work faster than the serial version for buffer sizes around 64/128 KB or less. This is for Intel Quad Core2 (Q8400 @ 2.66 GHz) / Linux (openSUSE 11.2, 64 bit), but your mileage may vary (and will vary!) for other processors / operating systems. In contrast, for buffers larger than 64/128 KB, the threaded version starts to perform significantly better, being the sweet point at 1 MB (again, this is with my setup). For larger buffer sizes than 1 MB, the threaded code slows down again, but it is probably due to a cache size issue and besides, it is still considerably faster than serial code. This is why Blosc falls back to use the serial version for such a 'small' buffers. So, you don't have to worry too much about deciding whether you should set the number of threads to 1 (serial) or more (parallel). Just set it to the number of cores in your processor and your are done! Francesc Alted c-blosc-1.7.0/RELEASE_NOTES.rst000066400000000000000000000453231254622165100156360ustar00rootroot00000000000000================================= Release notes for c-blosc 1.7.0 ================================= :Author: Francesc Alted :Contact: francesc@blosc.org :URL: http://www.blosc.org Changes from 1.6.1 to 1.7.0 =========================== * Added a new 'bitshuffle' filter so that the shuffle takes place at a bit level and not just at a byte one, which is what it does the previous 'shuffle' filter. For activating this new bit-level filter you only have to pass the symbol BLOSC_BITSHUFFLE to `blosc_compress()`. For the previous byte-level one, pass BLOSC_SHUFFLE. For disabling the shuffle, pass BLOSC_NOSHUFFLE. This is a port of the existing filter in https://github.com/kiyo-masui/bitshuffle. Thanks to Kiyo Masui for changing the license and allowing its inclusion here. * New acceleration mode for LZ4 and BloscLZ codecs that enters in operation with complevel < 9. This allows for an important boost in speed with minimal compression ratio loss. Francesc Alted. * LZ4 codec updated to 1.7.0 (r130). * PREFER_EXTERNAL_COMPLIBS cmake option has been removed and replaced by the more fine grained PREFER_EXTERNAL_LZ4, PREFER_EXTERNAL_SNAPPY and PREFER_EXTERNAL_ZLIB. In order to allow the use of the new API introduced in LZ4 1.7.0, PREFER_EXTERNAL_LZ4 has been set to OFF by default, whereas PREFER_EXTERNAL_SNAPPY and PREFER_EXTERNAL_ZLIB continues to be ON. * Implemented SSE2 shuffle support for buffers containing a number of elements which is not a multiple of (typesize * vectorsize). Jack Pappas. * Added SSE2 shuffle/unshuffle routines for types larger than 16 bytes. Jack Pappas. * 'test_basic' suite has been split in components for a much better granularity on what's a possibly failing test. Also, lots of new tests have been added. Jack Pappas. * Fixed compilation on non-Intel archs (tested on ARM). Zbyszek Szmek. * Modifyied cmake files in order to inform that AVX2 on Visual Studio is supported only in 2013 update 2 and higher. * Added a replacement for stdbool.h for Visual Studio < 2013. * blosclz codec adds Win64/Intel as a platform supporting unaligned addressing. That leads to a speed-up of 2.2x in decompression. * New blosc_get_version_string() function for retrieving the version of the c-blosc library. Useful when linking with dynamic libraries and one want to know its version. * New example (win-dynamic-linking.c) that shows how to link a Blosc DLL dynamically in run-time (Windows only). * The `context.threads_started` is initialized now when decompressing. This could cause crashes in case you decompressed before compressing (e.g. directly deserializing blosc buffers). @atchouprakov. * The HDF5 filter has been removed from c-blosc and moved into its own repo at: https://github.com/Blosc/hdf5 * The MS Visual Studio 2008 has been tested with c-blosc for ensuring compatibility with extensions for Python 2.6 and up. Changes from 1.6.0 to 1.6.1 =========================== * Support for *runtime* detection of AVX2 and SSE2 SIMD instructions. These changes make it possible to compile one single binary that runs on a system that supports SSE2 or AVX2 (or neither), so the redistribution problem is fixed (see #101). Thanks to Julian Taylor and Jack Pappas. * Added support for MinGW and TDM-GCC compilers for Windows. Thanks to yasushima-gd. * Fixed a bug in blosclz that could potentially overwrite an area beyond the output buffer. See #113. * New computation for blocksize so that larger typesizes (> 8 bytes) would benefit of much better compression ratios. Speed is not penalized too much. * New parametrization of the hash table for blosclz codec. This allows better compression in many scenarios, while slightly increasing the speed. Changes from 1.5.4 to 1.6.0 =========================== * Support for AVX2 is here! The benchmarks with a 4-core Intel Haswell machine tell that both compression and decompression are accelerated around a 10%, reaching peaks of 9.6 GB/s during compression and 26 GB/s during decompression (memcpy() speed for this machine is 7.5 GB/s for writes and 11.7 GB/s for reads). Many thanks to @littlezhou for this nice work. * Support for HPET (high precision timers) for the `bench` program. This is particularly important for microbenchmarks like bench is doing; since they take so little time to run, the granularity of a less-accurate timer may account for a significant portion of the runtime of the benchmark itself, skewing the results. Thanks to Jack Pappas. Changes from 1.5.3 to 1.5.4 =========================== * Updated to LZ4 1.6.0 (r128). * Fix resource leak in t_blosc. Jack Pappas. * Better checks during testing. Jack Pappas. * Dynamically loadable HDF5 filter plugin. Kiyo Masui. Changes from 1.5.2 to 1.5.3 =========================== * Use llabs function (where available) instead of abs to avoid truncating the result. Jack Pappas. * Use C11 aligned_alloc when it's available. Jack Pappas. * Use the built-in stdint.h with MSVC when available. Jack Pappas. * Only define the __SSE2__ symbol when compiling with MS Visual C++ and targeting x64 or x86 with the correct /arch flag set. This avoids re-defining the symbol which makes other compilers issue warnings. Jack Pappas. * Reinitializing Blosc during a call to set_nthreads() so as to fix problems with contexts. Francesc Alted. Changes from 1.5.1 to 1.5.2 =========================== * Using blosc_compress_ctx() / blosc_decompress_ctx() inside the HDF5 compressor for allowing operation in multiprocess scenarios. See: https://github.com/PyTables/PyTables/issues/412 The drawback of this quick fix is that the Blosc filter will be only able to use a single thread until another solution can be devised. Changes from 1.5.0 to 1.5.1 =========================== * Updated to LZ4 1.5.0. Closes #74. * Added the 'const' qualifier to non SSE2 shuffle functions. Closes #75. * Explicitly call blosc_init() in HDF5 blosc_filter.c, fixing a segfault. * Quite a few improvements in cmake files for HDF5 support. Thanks to Dana Robinson (The HDF Group). * Variable 'class' caused problems compiling the HDF5 filter with g++. Thanks to Laurent Chapon. * Small improvements on docstrings of c-blosc main functions. Changes from 1.4.1 to 1.5.0 =========================== * Added new calls for allowing Blosc to be used *simultaneously* (i.e. lock free) from multi-threaded environments. The new functions are: - blosc_compress_ctx(...) - blosc_decompress_ctx(...) See the new docstrings in blosc.h for how to use them. The previous API should be completely unaffected. Thanks to Christopher Speller. * Optimized copies during BloscLZ decompression. This can make BloscLZ to decompress up to 1.5x faster in some situations. * LZ4 and LZ4HC compressors updated to version 1.3.1. * Added an examples directory on how to link apps with Blosc. * stdlib.h moved from blosc.c to blosc.h as suggested by Rob Lathm. * Fix a warning for {snappy,lz4}-free compilation. Thanks to Andrew Schaaf. * Several improvements for CMakeLists.txt (cmake). * Fixing C99 compatibility warnings. Thanks to Christopher Speller. Changes from 1.4.0 to 1.4.1 =========================== * Fixed a bug in blosc_getitem() introduced in 1.4.0. Added a test for blosc_getitem() as well. Changes from 1.3.6 to 1.4.0 =========================== * Support for non-Intel and non-SSE2 architectures has been added. In particular, the Raspberry Pi platform (ARM) has been tested and all tests pass here. * Architectures requiring strict access alignment are supported as well. Due to this, arquitectures with a high penalty in accessing unaligned data (e.g. Raspberry Pi, ARMv6) can compress up to 2.5x faster. * LZ4 has been updated to r119 (1.2.0) so as to fix a possible security breach. Changes from 1.3.5 to 1.3.6 =========================== * Updated to LZ4 r118 due to a (highly unlikely) security hole. For details see: http://fastcompression.blogspot.fr/2014/06/debunking-lz4-20-years-old-bug-myth.html Changes from 1.3.4 to 1.3.5 =========================== * Removed a pointer from 'pointer from integer without a cast' compiler warning due to a bad macro definition. Changes from 1.3.3 to 1.3.4 =========================== * Fixed a false buffer overrun condition. This bug made c-blosc to fail, even if the failure was not real. * Fixed the type of a buffer string. Changes from 1.3.2 to 1.3.3 =========================== * Updated to LZ4 1.1.3 (improved speed for 32-bit platforms). * Added a new `blosc_cbuffer_complib()` for getting the compression library for a compressed buffer. Changes from 1.3.1 to 1.3.2 =========================== * Fix for compiling Snappy sources against MSVC 2008. Thanks to Mark Wiebe! * Version for internal LZ4 and Snappy are now supported. When compiled against the external libraries, this info is not available because they do not support the symbols (yet). Changes from 1.3.0 to 1.3.1 =========================== * Fixes for a series of issues with the filter for HDF5 and, in particular, a problem in the decompression buffer size that made it impossible to use the blosc_filter in combination with other ones (e.g. fletcher32). See https://github.com/PyTables/PyTables/issues/21. Thanks to Antonio Valentino for the fix! Changes from 1.2.4 to 1.3.0 =========================== A nice handful of compressors have been added to Blosc: * LZ4 (http://code.google.com/p/lz4/): A very fast compressor/decompressor. Could be thought as a replacement of the original BloscLZ, but it can behave better is some scenarios. * LZ4HC (http://code.google.com/p/lz4/): This is a variation of LZ4 that achieves much better compression ratio at the cost of being much slower for compressing. Decompression speed is unaffected (and sometimes better than when using LZ4 itself!), so this is very good for read-only datasets. * Snappy (http://code.google.com/p/snappy/): A very fast compressor/decompressor. Could be thought as a replacement of the original BloscLZ, but it can behave better is some scenarios. * Zlib (http://www.zlib.net/): This is a classic. It achieves very good compression ratios, at the cost of speed. However, decompression speed is still pretty good, so it is a good candidate for read-only datasets. With this, you can select the compression library with the new function:: int blosc_set_complib(char* complib); where you pass the library that you want to use (currently "blosclz", "lz4", "lz4hc", "snappy" and "zlib", but the list can grow in the future). You can get more info about compressors support in you Blosc build by using these functions:: char* blosc_list_compressors(void); int blosc_get_complib_info(char *compressor, char **complib, char **version); Changes from 1.2.2 to 1.2.3 =========================== - Added a `blosc_init()` and `blosc_destroy()` so that the global lock can be initialized safely. These new functions will also allow other kind of initializations/destructions in the future. Existing applications using Blosc do not need to start using the new functions right away, as long as they calling `blosc_set_nthreads()` previous to anything else. However, using them is highly recommended. Thanks to Oscar Villellas for the init/destroy suggestion, it is a nice idea! Changes from 1.2.1 to 1.2.2 =========================== - All important warnings removed for all tested platforms. This will allow less intrusiveness compilation experiences with applications including Blosc source code. - The `bench/bench.c` has been updated so that it can be compiled on Windows again. - The new web site has been set to: http://www.blosc.org Changes from 1.2 to 1.2.1 ========================= - Fixed a problem with global lock not being initialized. This affected mostly to Windows platforms. Thanks to Christoph Gohlke for finding the cure! Changes from 1.1.5 to 1.2 ========================= - Now it is possible to call Blosc simultaneously from a parent threaded application without problems. This has been solved by setting a global lock so that the different calling threads do not execute Blosc routines at the same time. Of course, real threading work is still available *inside* Blosc itself. Thanks to Thibault North. - Support for cmake is now included. Linux, Mac OSX and Windows platforms are supported. Thanks to Thibault North, Antonio Valentino and Mark Wiebe. - Fixed many compilers warnings (specially about unused variables). - As a consequence of the above, as minimal change in the API has been introduced. That is, the previous API:: void blosc_free_resources(void) has changed to:: int blosc_free_resources(void) Now, a return value of 0 means that the resources have been released successfully. If the return value is negative, then it is not guaranteed that all the resources have been freed. - Many typos were fixed and docs have been improved. The script for generating nice plots for the included benchmarks has been improved too. Thanks to Valetin Haenel. Changes from 1.1.4 to 1.1.5 =========================== - Fix compile error with msvc compilers (Christoph Gohlke) Changes from 1.1.3 to 1.1.4 =========================== - Redefinition of the BLOSC_MAX_BUFFERSIZE constant as (INT_MAX - BLOSC_MAX_OVERHEAD) instead of just INT_MAX. This prevents to produce outputs larger than INT_MAX, which is not supported. - `exit()` call has been replaced by a ``return -1`` in blosc_compress() when checking for buffer sizes. Now programs will not just exit when the buffer is too large, but return a negative code. - Improvements in explicit casts. Blosc compiles without warnings (with GCC) now. - Lots of improvements in docs, in particular a nice ascii-art diagram of the Blosc format (Valentin Haenel). - Improvements to the plot-speeds.py (Valentin Haenel). - [HDF5 filter] Adapted HDF5 filter to use HDF5 1.8 by default (Antonio Valentino). - [HDF5 filter] New version of H5Z_class_t definition (Antonio Valentino). Changes from 1.1.2 to 1.1.3 =========================== - Much improved compression ratio when using large blocks (> 64 KB) and high compression levels (> 6) under some circumstances (special data distribution). Closes #7. Changes from 1.1.1 to 1.1.2 =========================== - Fixes for small typesizes (#6 and #1 of python-blosc). Changes from 1.1 to 1.1.1 ========================= - Added code to avoid calling blosc_set_nthreads more than necessary. That will improve performance up to 3x or more, specially for small chunksizes (< 1 MB). Changes from 1.0 to 1.1 ======================= - Added code for emulating pthreads API on Windows. No need to link explicitly with pthreads lib on Windows anymore. However, performance is a somewhat worse because the new emulation layer does not support the `pthread_barrier_wait()` call natively. But the big improvement in installation easiness is worth this penalty (most specially on 64-bit Windows, where pthreads-win32 support is flaky). - New BLOSC_MAX_BUFFERSIZE, BLOSC_MAX_TYPESIZE and BLOSC_MAX_THREADS symbols are available in blosc.h. These can be useful for validating parameters in clients. Thanks to Robert Smallshire for suggesting that. - A new BLOSC_MIN_HEADER_LENGTH symbol in blosc.h tells how many bytes long is the minimum length of a Blosc header. `blosc_cbuffer_sizes()` only needs these bytes to be passed to work correctly. - Removed many warnings (related with potentially dangerous type-casting code) issued by MSVC 2008 in 64-bit mode. - Fixed a problem with the computation of the blocksize in the Blosc filter for HDF5. - Fixed a problem with large datatypes. See http://www.pytables.org/trac/ticket/288 for more info. - Now Blosc is able to work well even if you fork an existing process with a pool of threads. Bug discovered when PyTables runs in multiprocess environments. See http://pytables.org/trac/ticket/295 for details. - Added a new `blosc_getitem()` call to allow the retrieval of items in sizes smaller than the complete buffer. That is useful for the carray project, but certainly for others too. Changes from 0.9.5 to 1.0 ========================= - Added a filter for HDF5 so that people can use Blosc outside PyTables, if they want to. - Many small improvements, specially in README files. - Do not assume that size_t is uint_32 for every platform. - Added more protection for large buffers or in allocation memory routines. - The src/ directory has been renamed to blosc/. - The `maxbytes` parameter in `blosc_compress()` has been renamed to `destsize`. This is for consistency with the `blosc_decompress()` parameters. Changes from 0.9.4 to 0.9.5 =========================== - Now, compression level 0 is allowed, meaning not compression at all. The overhead of this mode will be always BLOSC_MAX_OVERHEAD (16) bytes. This mode actually represents using Blosc as a basic memory container. - Supported a new parameter `maxbytes` for ``blosc_compress()``. It represents a maximum of bytes for output. Tests unit added too. - Added 3 new functions for querying different metadata on compressed buffers. A test suite for testing the new API has been added too. Changes from 0.9.3 to 0.9.4 =========================== - Support for cross-platform big/little endian compatibility in Blosc headers has been added. - Fixed several failures exposed by the extremesuite. The problem was a bad check for limits in the buffer size while compressing. - Added a new suite in bench.c called ``debugsuite`` that is appropriate for debugging purposes. Now, the ``extremesuite`` can be used for running the complete (and extremely long) suite. Changes from 0.9.0 to 0.9.3 =========================== - Fixed several nasty bugs uncovered by the new suites in bench.c. Thanks to Tony Theodore and Gabriel Beckers for their (very) responsive beta testing and feedback. - Added several modes (suites), namely ``suite``, ``hardsuite`` and ``extremehardsuite`` in bench.c so as to allow different levels of testing. Changes from 0.8.0 to 0.9 ========================= - Internal format version bumped to 2 in order to allow an easy way to indicate that a buffer is being saved uncompressed. This is not supported yet, but it might be in the future. - Blosc can use threads now for leveraging the increasing number of multi-core processors out there. See README-threaded.txt for more info. - Added a protection for MacOSX so that it has to not link against posix_memalign() funtion, which seems not available in old versions of MacOSX (for example, Tiger). At nay rate, posix_memalign() is not necessary on Mac because 16 bytes alignment is ensured by default. Thanks to Ivan Vilata. Fixes #3. c-blosc-1.7.0/RELEASING.rst000066400000000000000000000031271254622165100151130ustar00rootroot00000000000000================ Releasing Blosc ================ :Author: Francesc Alted :Contact: francesc@blosc.org :Date: 2014-01-15 Preliminaries ------------- - Make sure that ``RELEASE_NOTES.rst`` and ``ANNOUNCE.rst`` are up to date with the latest news in the release. - Check that *VERSION* symbols in blosc/blosc.h contains the correct info. Testing ------- Create a new build/ directory, change into it and issue:: $ cmake .. $ cmake --build . $ ctest To actually test Blosc the hard way, look at the end of: http://blosc.org/synthetic-benchmarks.html where instructions on how to intensively test (and benchmark) Blosc are given. Tagging ------- - Create a tag ``X.Y.Z`` from ``master``. Use the next message:: $ git tag -a vX.Y.Z -m "Tagging version X.Y.Z" - Push the tag to the github repo:: $ git push --tags Announcing ---------- - Update the release notes in the github wiki: https://github.com/Blosc/c-blosc/wiki/Release-notes - Send an announcement to the blosc, pytables-dev, bcolz and comp.compression lists. Use the ``ANNOUNCE.rst`` file as skeleton (possibly as the definitive version). Post-release actions -------------------- - Edit *VERSION* symbols in blosc/blosc.h in master to increment the version to the next minor one (i.e. X.Y.Z --> X.Y.(Z+1).dev). - Create new headers for adding new features in ``RELEASE_NOTES.rst`` and empty the release-specific information in ``ANNOUNCE.rst`` and add this place-holder instead: #XXX version-specific blurb XXX# That's all folks! .. Local Variables: .. mode: rst .. coding: utf-8 .. fill-column: 70 .. End: c-blosc-1.7.0/THANKS.rst000066400000000000000000000026721254622165100145760ustar00rootroot00000000000000I'd like to thank the PyTables community that have collaborated in the exhaustive testing of Blosc. With an aggregate amount of more than 300 TB of different datasets compressed *and* decompressed successfully, I can say that Blosc is pretty safe now and ready for production purposes. Other important contributions: * Valentin Haenel did a terrific work implementing the support for the Snappy compression, fixing typos and improving docs and the plotting script. * Thibault North, with ideas from Oscar Villellas, contributed a way to call Blosc from different threads in a safe way. Christopher Speller introduced contexts so that a global lock is not necessary anymore. * The CMake support was initially contributed by Thibault North, and Antonio Valentino and Mark Wiebe made great enhancements to it. * Christopher Speller also introduced the two new '_ctx' calls to avoid the use of the blosc_init() and blosc_destroy(). * Jack Pappas contributed important portability enhancements, specially runtime and cross-platform detection of SSE2/AVX2 as well as high precision timers (HPET) for the benchmark program. * @littlezhou implemented the AVX2 version of shuffle routines. * Julian Taylor contributed a way to detect AVX2 in runtime and calling the appropriate routines only if the undelying hardware supports it. * Kiyo Masui for relicensing his bitshuffle project for allowing the inclusion of part of his code in Blosc. c-blosc-1.7.0/THOUGHTS_FOR_2.0.txt000066400000000000000000000013761254622165100161470ustar00rootroot00000000000000Here there go some random thoughts for Blosc 2.0: * Try to find a classification for the return codes and standarize them. Use constants in blosc.h for naming them. * blosc_init() should return a structure that should serve as a context for other calls. So instead of blosc_set_blocksize(size_t size), you would have blosc_set_blocksize(blosc_context* context, size_t size). See: https://github.com/Blosc/c-blosc/pull/66/files#r20010877 for some discussion. * Add more space in header for allowing more compressors and filters. Discuss how much should that space be in the mailing list. * Make two different versions of the header so that it supports 32-bit sizes as well as 64-bit? That would allow to compress buffers larger than INT_MAX. c-blosc-1.7.0/appveyor.yml000066400000000000000000000012021254622165100154300ustar00rootroot00000000000000# AppVeyor CI build configuration for c-blosc # Before cloning the repo, configure git to handle line endings correctly. init: - git config --global core.autocrlf input # This is the build version displayed on AppVeyor's UI. # It's incrementally automatically like travis-ci but allows custom formatting. version: '{build}' build_script: - cmd: mkdir build - cmd: cd build - cmd: cmake -G "Visual Studio 12 Win64" .. - cmd: cmake --build . --config Release # Disable CPack for now as it conflicts with the Chocolatey 'cpack' alias # for 'choco pack' and causes the build to fail. #- cmd: cpack test_script: - cmd: ctest c-blosc-1.7.0/bench/000077500000000000000000000000001254622165100141245ustar00rootroot00000000000000c-blosc-1.7.0/bench/CMakeLists.txt000066400000000000000000000107571254622165100166760ustar00rootroot00000000000000# sources set(SOURCES bench.c) # targets add_executable(bench ${SOURCES}) if(UNIX AND NOT APPLE) # cmake is complaining about LINK_PRIVATE in original PR # and removing it does not seem to hurt, so be it. # target_link_libraries(bench LINK_PRIVATE rt) target_link_libraries(bench rt) endif(UNIX AND NOT APPLE) target_link_libraries(bench blosc_shared) # have to copy blosc dlls on Windows if(MSVC) add_custom_command( TARGET bench POST_BUILD COMMAND ${CMAKE_COMMAND} ARGS -E copy_if_different "${PROJECT_BINARY_DIR}/blosc/\$\(Configuration\)/blosc.dll" "${CMAKE_CURRENT_BINARY_DIR}/\$\(Configuration\)/blosc.dll") elseif(MINGW) add_custom_command( TARGET bench POST_BUILD COMMAND ${CMAKE_COMMAND} ARGS -E copy_if_different "${PROJECT_BINARY_DIR}/blosc/libblosc.dll" "${CMAKE_CURRENT_BINARY_DIR}/libblosc.dll") endif() # tests if(BUILD_TESTS) # The commented tests below take too much time to complete option(TEST_INCLUDE_BENCH_SHUFFLE_1 "Include bench shuffle (1 thread) in the tests" ON) if(TEST_INCLUDE_BENCH_SHUFFLE_1) set(SHUFFLE_1_OPTS shuffle test 1) add_test(test_blosclz_shuffle_1 bench blosclz ${SHUFFLE_1_OPTS}) if (HAVE_LZ4) add_test(test_lz4_shuffle_1 bench lz4 ${SHUFFLE_1_OPTS}) # add_test(test_lz4hc_shuffle_1 bench lz4hc ${SHUFFLE_1_OPTS}) endif (HAVE_LZ4) if (HAVE_SNAPPY) add_test(test_snappy_shuffle_1 bench snappy ${SHUFFLE_1_OPTS}) endif (HAVE_SNAPPY) if (HAVE_ZLIB) # add_test(test_zlib_shuffle_1 bench zlib ${SHUFFLE_1_OPTS}) endif (HAVE_ZLIB) endif(TEST_INCLUDE_BENCH_SHUFFLE_1) option(TEST_INCLUDE_BENCH_SHUFFLE_N "Include bench shuffle (multithread) in the tests" ON) if(TEST_INCLUDE_BENCH_SHUFFLE_N) set(SHUFFLE_N_OPTS shuffle test) add_test(test_blosclz_shuffle_n bench blosclz ${SHUFFLE_N_OPTS}) if (HAVE_LZ4) add_test(test_lz4_shuffle_n bench lz4 ${SHUFFLE_N_OPTS}) add_test(test_lz4hc_shuffle_n bench lz4hc ${SHUFFLE_N_OPTS}) endif (HAVE_LZ4) if (HAVE_SNAPPY) add_test(test_snappy_shuffle_n bench snappy ${SHUFFLE_N_OPTS}) endif (HAVE_SNAPPY) if (HAVE_ZLIB) add_test(test_zlib_shuffle_n bench zlib ${SHUFFLE_N_OPTS}) endif (HAVE_ZLIB) endif(TEST_INCLUDE_BENCH_SHUFFLE_N) option(TEST_INCLUDE_BENCH_BITSHUFFLE_1 "Include bench bitshuffle (1 thread) in the tests" ON) if(TEST_INCLUDE_BENCH_BITSHUFFLE_1) set(BITSHUFFLE_1_OPTS bitshuffle test 1) add_test(test_blosclz_bitshuffle_1 bench blosclz ${BITSHUFFLE_1_OPTS}) if (HAVE_LZ4) add_test(test_lz4_bitshuffle_1 bench lz4 ${BITSHUFFLE_1_OPTS}) # add_test(test_lz4hc_bitshuffle_1 bench lz4hc ${BITSHUFFLE_1_OPTS}) endif (HAVE_LZ4) if (HAVE_SNAPPY) add_test(test_snappy_bitshuffle_1 bench snappy ${BITSHUFFLE_1_OPTS}) endif (HAVE_SNAPPY) if (HAVE_ZLIB) # add_test(test_zlib_bitshuffle_1 bench zlib ${BITSHUFFLE_1_OPTS}) endif (HAVE_ZLIB) endif(TEST_INCLUDE_BENCH_BITSHUFFLE_1) option(TEST_INCLUDE_BENCH_BITSHUFFLE_N "Include bench bitshuffle (multithread) in the tests" ON) if(TEST_INCLUDE_BENCH_BITSHUFFLE_N) set(BITSHUFFLE_N_OPTS bitshuffle test 2 2097152 3 19) # typesize=3 add_test(test_blosclz_bitshuffle_n bench blosclz ${BITSHUFFLE_N_OPTS}) if (HAVE_LZ4) add_test(test_lz4_bitshuffle_n bench lz4 ${BITSHUFFLE_N_OPTS}) # add_test(test_lz4hc_bitshuffle_n bench lz4hc ${BITSHUFFLE_N_OPTS}) endif (HAVE_LZ4) if (HAVE_SNAPPY) add_test(test_snappy_bitshuffle_n bench snappy ${BITSHUFFLE_N_OPTS}) endif (HAVE_SNAPPY) if (HAVE_ZLIB) # add_test(test_zlib_bitshuffle_n bench zlib ${BITSHUFFLE_N_OPTS}) endif (HAVE_ZLIB) endif(TEST_INCLUDE_BENCH_BITSHUFFLE_N) option(TEST_INCLUDE_BENCH_SUITE "Include bench suite in the tests" OFF) if(TEST_INCLUDE_BENCH_SUITE) add_test(test_hardsuite blosc blosclz shuffle suite) endif(TEST_INCLUDE_BENCH_SUITE) option(TEST_INCLUDE_BENCH_DEBUGSUITE "Include bench debugsuite in the tests" OFF) if(TEST_INCLUDE_BENCH_DEBUGSUITE) add_test(test_debugsuite bench blosclz shuffle debugsuite) endif(TEST_INCLUDE_BENCH_DEBUGSUITE) endif(BUILD_TESTS) c-blosc-1.7.0/bench/Makefile000066400000000000000000000020461254622165100155660ustar00rootroot00000000000000CC = gcc # clang++, g++ or just gcc if not compiling Snappy (C++ code) CFLAGS = -O3 -g -msse2 -Wall LDFLAGS = -lpthread # for UNIX or for Windows with pthread installed #LDFLAGS = -static # for mingw SOURCES = $(wildcard ../blosc/*.c) EXECUTABLE = bench # Support for internal LZ4 and LZ4HC LZ4_DIR = ../internal-complibs/lz4-1.7.0 CFLAGS += -DHAVE_LZ4 -I$(LZ4_DIR) SOURCES += $(wildcard $(LZ4_DIR)/*.c) # Support for external LZ4 and LZ4HC #LDFLAGS += -DHAVE_LZ4 -llz4 # Support for internal Snappy #SNAPPY_DIR = ../internal-complibs/snappy-1.1.1 #CFLAGS += -DHAVE_SNAPPY -I$(SNAPPY_DIR) #SOURCES += $(wildcard $(SNAPPY_DIR)/*.cc) # Support for external Snappy LDFLAGS += -DHAVE_SNAPPY -lsnappy # Support for external Zlib LDFLAGS += -DHAVE_ZLIB -lz # Support for internal Zlib #ZLIB_DIR = ../internal-complibs/zlib-1.2.8 #CFLAGS += -DHAVE_ZLIB -I$(ZLIB_DIR) #SOURCES += $(wildcard $(ZLIB_DIR)/*.c) SOURCES += bench.c all: $(SOURCES) $(EXECUTABLE) $(EXECUTABLE): $(SOURCES) $(CC) $(CFLAGS) $(SOURCES) -o $@ $(LDFLAGS) clean: rm -rf $(EXECUTABLE) c-blosc-1.7.0/bench/Makefile.mingw000066400000000000000000000023611254622165100167060ustar00rootroot00000000000000# Makefile for the MinGW suite for Windows CC = g++ # clang++, g++ or just gcc if not compiling Snappy (C++ code) CFLAGS = -O3 -g -msse2 -Wall #LDFLAGS = -lpthread # for UNIX or for Windows with pthread installed LDFLAGS = -static # for mingw SOURCES = $(wildcard ../blosc/*.c) EXECUTABLE = bench # Support for internal LZ4 LZ4_DIR = ../internal-complibs/lz4-1.7.0 CFLAGS += -DHAVE_LZ4 -I$(LZ4_DIR) SOURCES += $(wildcard $(LZ4_DIR)/*.c) # Support for external LZ4 #LDFLAGS += -DHAVE_LZ4 -llz4 # Support for internal Snappy SNAPPY_DIR = ../internal-complibs/snappy-1.1.1 CFLAGS += -DHAVE_SNAPPY -I$(SNAPPY_DIR) SOURCES += $(wildcard $(SNAPPY_DIR)/*.cc) # Support for external Snappy #LDFLAGS += -DHAVE_SNAPPY -lsnappy # Support for the msvc zlib: ZLIB_ROOT=/libs/zlib128 LDFLAGS=-DHAVE_ZLIB -I$(ZLIB_ROOT)/include -lzdll -L$(ZLIB_ROOT)/lib # Support for the mingw zlib: #ZLIB_ROOT=/libs/libz-1.2.8 #LDFLAGS=-DHAVE_ZLIB -I$(ZLIB_ROOT)/include -lz -L$(ZLIB_ROOT)/lib # Support for internal Zlib #ZLIB_DIR = ../internal-complibs/zlib-1.2.8 #CFLAGS += -DHAVE_ZLIB -I$(ZLIB_DIR) #SOURCES += $(wildcard $(ZLIB_DIR)/*.c) all: $(SOURCES) $(EXECUTABLE) $(EXECUTABLE): $(SOURCES) $(CC) $(CFLAGS) bench.c $(SOURCES) -o $@ $(LDFLAGS) clean: rm -rf $(EXECUTABLE) c-blosc-1.7.0/bench/bench.c000066400000000000000000000432021254622165100153500ustar00rootroot00000000000000/********************************************************************* Small benchmark for testing basic capabilities of Blosc. You can select different degrees of 'randomness' in input buffer, as well as external datafiles (uncomment the lines after "For data coming from a file" comment). For usage instructions of this benchmark, please see: http://blosc.org/synthetic-benchmarks.html I'm collecting speeds for different machines, so the output of your benchmarks and your processor specifications are welcome! Author: Francesc Alted Note: Compiling this with VS2008 does not work well with cmake. Here it is a way to compile the benchmark (with added support for LZ4): > cl /DHAVE_LZ4 /arch:SSE2 /Ox /Febench.exe /Iblosc /Iinternal-complibs\lz4-1.7.0 bench\bench.c blosc\blosc.c blosc\blosclz.c blosc\shuffle.c blosc\shuffle-sse2.c blosc\shuffle-generic.c blosc\bitshuffle-generic.c blosc\bitshuffle-sse2.c internal-complibs\lz4-1.7.0\*.c See LICENSES/BLOSC.txt for details about copyright and rights to use. **********************************************************************/ #include #include #include #include #include #include #if defined(_WIN32) /* For QueryPerformanceCounter(), etc. */ #include #elif defined(__MACH__) #include #include #include #include #elif defined(__unix__) #include #if defined(__linux__) #include #else #include #endif #else #error Unable to detect platform. #endif #include "../blosc/blosc.h" #define KB 1024 #define MB (1024*KB) #define GB (1024*MB) #define NCHUNKS (32*1024) /* maximum number of chunks */ #define MAX_THREADS 16 int nchunks = NCHUNKS; int niter = 3; /* default number of iterations */ double totalsize = 0.; /* total compressed/decompressed size */ /* System-specific high-precision timing functions. */ #if defined(_WIN32) /* The type of timestamp used on this system. */ #define blosc_timestamp_t LARGE_INTEGER /* Set a timestamp value to the current time. */ void blosc_set_timestamp(blosc_timestamp_t* timestamp) { /* Ignore the return value, assume the call always succeeds. */ QueryPerformanceCounter(timestamp); } /* Given two timestamp values, return the difference in microseconds. */ double blosc_elapsed_usecs(blosc_timestamp_t start_time, blosc_timestamp_t end_time) { LARGE_INTEGER CounterFreq; QueryPerformanceFrequency(&CounterFreq); return (double)(end_time.QuadPart - start_time.QuadPart) / ((double)CounterFreq.QuadPart / 1e6); } #else /* The type of timestamp used on this system. */ #define blosc_timestamp_t struct timespec /* Set a timestamp value to the current time. */ void blosc_set_timestamp(blosc_timestamp_t* timestamp) { #ifdef __MACH__ // OS X does not have clock_gettime, use clock_get_time clock_serv_t cclock; mach_timespec_t mts; host_get_clock_service(mach_host_self(), CALENDAR_CLOCK, &cclock); clock_get_time(cclock, &mts); mach_port_deallocate(mach_task_self(), cclock); timestamp->tv_sec = mts.tv_sec; timestamp->tv_nsec = mts.tv_nsec; #else clock_gettime(CLOCK_MONOTONIC, timestamp); #endif } /* Given two timestamp values, return the difference in microseconds. */ double blosc_elapsed_usecs(blosc_timestamp_t start_time, blosc_timestamp_t end_time) { return (1e6 * (end_time.tv_sec - start_time.tv_sec)) + (1e-3 * (end_time.tv_nsec - start_time.tv_nsec)); } #endif /* Given two timeval stamps, return the difference in seconds */ double getseconds(blosc_timestamp_t last, blosc_timestamp_t current) { return 1e-6 * blosc_elapsed_usecs(last, current); } /* Given two timeval stamps, return the time per chunk in usec */ double get_usec_chunk(blosc_timestamp_t last, blosc_timestamp_t current, int niter, size_t nchunks) { double elapsed_usecs = (double)blosc_elapsed_usecs(last, current); return elapsed_usecs / (double)(niter * nchunks); } /* Define posix_memalign for Windows */ #if defined(_WIN32) #include int posix_memalign(void **memptr, size_t alignment, size_t size) { *memptr = _aligned_malloc(size, alignment); return 0; } /* Buffers allocated with _aligned_malloc need to be freed with _aligned_free. */ #define aligned_free(memptr) _aligned_free(memptr) #else /* If not using MSVC, aligned memory can be freed in the usual way. */ #define aligned_free(memptr) free(memptr) #endif /* defined(_WIN32) && !defined(__MINGW32__) */ int get_value(int i, int rshift) { int v; v = (i<<26)^(i<<18)^(i<<11)^(i<<3)^i; if (rshift < 32) { v &= (1 << rshift) - 1; } return v; } void init_buffer(void *src, int size, int rshift) { unsigned int i; int *_src = (int *)src; /* To have reproducible results */ srand(1); /* Initialize the original buffer */ for (i = 0; i < size/sizeof(int); ++i) { /* Choose one below */ /* _src[i] = 0; * _src[i] = 0x01010101; * _src[i] = 0x01020304; * _src[i] = i * 1/.3; * _src[i] = i; * _src[i] = rand() >> (32-rshift); */ _src[i] = get_value(i, rshift); } } void do_bench(char *compressor, char *shuffle, int nthreads, int size, int elsize, int rshift, FILE * ofile) { void *src, *srccpy; void *dest[NCHUNKS], *dest2; int nbytes = 0, cbytes = 0; int i, j, retcode; unsigned char *orig, *round; blosc_timestamp_t last, current; double tmemcpy, tshuf, tunshuf; int clevel, doshuffle; if (strcmp(shuffle, "shuffle") == 0) { doshuffle = BLOSC_SHUFFLE; } else if (strcmp(shuffle, "bitshuffle") == 0) { doshuffle = BLOSC_BITSHUFFLE; } else if (strcmp(shuffle, "noshuffle") == 0) { doshuffle = BLOSC_NOSHUFFLE; } blosc_set_nthreads(nthreads); if(blosc_set_compressor(compressor) < 0){ printf("Compiled w/o support for compressor: '%s', so sorry.\n", compressor); exit(1); } /* Initialize buffers */ srccpy = malloc(size); retcode = posix_memalign( (void **)(&src), 32, size); retcode = posix_memalign( (void **)(&dest2), 32, size); /* zero src to initialize byte on it, and not only multiples of 4 */ memset(src, 0, size); init_buffer(src, size, rshift); memcpy(srccpy, src, size); for (j = 0; j < nchunks; j++) { retcode = posix_memalign( (void **)(&dest[j]), 32, size+BLOSC_MAX_OVERHEAD); } fprintf(ofile, "--> %d, %d, %d, %d, %s, %s\n", nthreads, size, elsize, rshift, compressor, shuffle); fprintf(ofile, "********************** Run info ******************************\n"); fprintf(ofile, "Blosc version: %s (%s)\n", BLOSC_VERSION_STRING, BLOSC_VERSION_DATE); fprintf(ofile, "Using synthetic data with %d significant bits (out of 32)\n", rshift); fprintf(ofile, "Dataset size: %d bytes\tType size: %d bytes\n", size, elsize); fprintf(ofile, "Working set: %.1f MB\t\t", (size*nchunks) / (float)MB); fprintf(ofile, "Number of threads: %d\n", nthreads); fprintf(ofile, "********************** Running benchmarks *********************\n"); blosc_set_timestamp(&last); for (i = 0; i < niter; i++) { for (j = 0; j < nchunks; j++) { memcpy(dest[j], src, size); } } blosc_set_timestamp(¤t); tmemcpy = get_usec_chunk(last, current, niter, nchunks); fprintf(ofile, "memcpy(write):\t\t %6.1f us, %.1f MB/s\n", tmemcpy, (size * 1e6) / (tmemcpy*MB)); blosc_set_timestamp(&last); for (i = 0; i < niter; i++) { for (j = 0; j < nchunks; j++) { memcpy(dest2, dest[j], size); } } blosc_set_timestamp(¤t); tmemcpy = get_usec_chunk(last, current, niter, nchunks); fprintf(ofile, "memcpy(read):\t\t %6.1f us, %.1f MB/s\n", tmemcpy, (size * 1e6) / (tmemcpy*MB)); for (clevel=0; clevel<10; clevel++) { fprintf(ofile, "Compression level: %d\n", clevel); blosc_set_timestamp(&last); for (i = 0; i < niter; i++) { for (j = 0; j < nchunks; j++) { cbytes = blosc_compress(clevel, doshuffle, elsize, size, src, dest[j], size+BLOSC_MAX_OVERHEAD); } } blosc_set_timestamp(¤t); tshuf = get_usec_chunk(last, current, niter, nchunks); fprintf(ofile, "comp(write):\t %6.1f us, %.1f MB/s\t ", tshuf, (size * 1e6) / (tshuf*MB)); fprintf(ofile, "Final bytes: %d ", cbytes); if (cbytes > 0) { fprintf(ofile, "Ratio: %3.2f", size/(float)cbytes); } fprintf(ofile, "\n"); /* Compressor was unable to compress. Copy the buffer manually. */ if (cbytes == 0) { for (j = 0; j < nchunks; j++) { memcpy(dest[j], src, size); } } blosc_set_timestamp(&last); for (i = 0; i < niter; i++) { for (j = 0; j < nchunks; j++) { if (cbytes == 0) { memcpy(dest2, dest[j], size); nbytes = size; } else { nbytes = blosc_decompress(dest[j], dest2, size); } } } blosc_set_timestamp(¤t); tunshuf = get_usec_chunk(last, current, niter, nchunks); fprintf(ofile, "decomp(read):\t %6.1f us, %.1f MB/s\t ", tunshuf, (nbytes * 1e6) / (tunshuf*MB)); if (nbytes < 0) { fprintf(ofile, "FAILED. Error code: %d\n", nbytes); } /* fprintf(ofile, "Orig bytes: %d\tFinal bytes: %d\n", cbytes, nbytes); */ /* Check if data has had a good roundtrip. Byte-by-byte comparison is slow, so use 'memcmp' to check whether the roundtripped data is correct. If not, fall back to the slow path to print diagnostic messages. */ orig = (unsigned char *)srccpy; round = (unsigned char *)dest2; if (memcmp(orig, round, size) != 0) { for(i = 0; i %x, round-trip--> %x\n", orig[i], round[i]); break; } } } else { i = size; } if (i == size) fprintf(ofile, "OK\n"); } /* End clevel loop */ /* To compute the totalsize, we should take into account the 10 compression levels */ totalsize += (size * nchunks * niter * 10.); aligned_free(src); free(srccpy); aligned_free(dest2); for (i = 0; i < nchunks; i++) { aligned_free(dest[i]); } } /* Compute a sensible value for nchunks */ int get_nchunks(int size_, int ws) { int nchunks; nchunks = ws / size_; if (nchunks > NCHUNKS) nchunks = NCHUNKS; if (nchunks < 1) nchunks = 1; return nchunks; } void print_compress_info(void) { char *name = NULL, *version = NULL; int ret; printf("Blosc version: %s (%s)\n", BLOSC_VERSION_STRING, BLOSC_VERSION_DATE); printf("List of supported compressors in this build: %s\n", blosc_list_compressors()); printf("Supported compression libraries:\n"); ret = blosc_get_complib_info("blosclz", &name, &version); if (ret >= 0) printf(" %s: %s\n", name, version); ret = blosc_get_complib_info("lz4", &name, &version); if (ret >= 0) printf(" %s: %s\n", name, version); ret = blosc_get_complib_info("snappy", &name, &version); if (ret >= 0) printf(" %s: %s\n", name, version); ret = blosc_get_complib_info("zlib", &name, &version); if (ret >= 0) printf(" %s: %s\n", name, version); } int main(int argc, char *argv[]) { char compressor[32]; char shuffle[32] = "shuffle"; char bsuite[32]; int single = 1; int suite = 0; int hard_suite = 0; int extreme_suite = 0; int debug_suite = 0; int nthreads = 4; /* The number of threads */ int size = 2*MB; /* Buffer size */ int elsize = 8; /* Datatype size */ int rshift = 19; /* Significant bits */ int workingset = 256*MB; /* The maximum allocated memory */ int nthreads_, size_, elsize_, rshift_, i; FILE * output_file = stdout; blosc_timestamp_t last, current; float totaltime; char usage[256]; print_compress_info(); strncpy(usage, "Usage: bench [blosclz | lz4 | lz4hc | snappy | zlib] " "[noshuffle | shuffle | bitshuffle] " "[single | suite | hardsuite | extremesuite | debugsuite] " "[nthreads] [bufsize(bytes)] [typesize] [sbits]", 255); if (argc < 2) { printf("%s\n", usage); exit(1); } strcpy(compressor, argv[1]); if (strcmp(compressor, "blosclz") != 0 && strcmp(compressor, "lz4") != 0 && strcmp(compressor, "lz4hc") != 0 && strcmp(compressor, "snappy") != 0 && strcmp(compressor, "zlib") != 0) { printf("No such compressor: '%s'\n", compressor); printf("%s\n", usage); exit(2); } if (argc >= 3) { strcpy(shuffle, argv[2]); if (strcmp(shuffle, "shuffle") != 0 && strcmp(shuffle, "bitshuffle") != 0 && strcmp(shuffle, "noshuffle") != 0) { printf("No such shuffler: '%s'\n", shuffle); printf("%s\n", usage); exit(2); } } if (argc < 4) strcpy(bsuite, "single"); else strcpy(bsuite, argv[3]); if (strcmp(bsuite, "single") == 0) { single = 1; } else if (strcmp(bsuite, "test") == 0) { single = 1; workingset = 128*MB; } else if (strcmp(bsuite, "suite") == 0) { suite = 1; } else if (strcmp(bsuite, "hardsuite") == 0) { hard_suite = 1; workingset = 64*MB; /* Values here are ending points for loops */ nthreads = 2; size = 8*MB; elsize = 32; rshift = 32; } else if (strcmp(bsuite, "extremesuite") == 0) { extreme_suite = 1; workingset = 32*MB; niter = 1; /* Values here are ending points for loops */ nthreads = 4; size = 16*MB; elsize = 32; rshift = 32; } else if (strcmp(bsuite, "debugsuite") == 0) { debug_suite = 1; workingset = 32*MB; niter = 1; /* Warning: values here are starting points for loops. This is useful for debugging. */ nthreads = 1; size = 16*KB; elsize = 1; rshift = 0; } else { printf("%s\n", usage); exit(1); } printf("Using compressor: %s\n", compressor); printf("Using shuffle type: %s\n", shuffle); printf("Running suite: %s\n", bsuite); if (argc >= 5) { nthreads = atoi(argv[4]); } if (argc >= 6) { size = atoi(argv[5]); } if (argc >= 7) { elsize = atoi(argv[6]); } if (argc >= 8) { rshift = atoi(argv[7]); } if ((argc >= 9) || !(single || suite || hard_suite || extreme_suite)) { printf("%s\n", usage); exit(1); } nchunks = get_nchunks(size, workingset); blosc_set_timestamp(&last); blosc_init(); if (suite) { for (nthreads_=1; nthreads_ <= nthreads; nthreads_++) { do_bench(compressor, shuffle, nthreads_, size, elsize, rshift, output_file); } } else if (hard_suite) { /* Let's start the rshift loop by 4 so that 19 is visited. This is to allow a direct comparison with the plain suite, that runs precisely at 19 significant bits. */ for (rshift_ = 4; rshift_ <= rshift; rshift_ += 5) { for (elsize_ = 1; elsize_ <= elsize; elsize_ *= 2) { /* The next loop is for getting sizes that are not power of 2 */ for (i = -elsize_; i <= elsize_; i += elsize_) { for (size_ = 32*KB; size_ <= size; size_ *= 2) { nchunks = get_nchunks(size_+i, workingset); niter = 1; for (nthreads_ = 1; nthreads_ <= nthreads; nthreads_++) { do_bench(compressor, shuffle, nthreads_, size_+i, elsize_, rshift_, output_file); blosc_set_timestamp(¤t); totaltime = (float)getseconds(last, current); printf("Elapsed time:\t %6.1f s. Processed data: %.1f GB\n", totaltime, totalsize / GB); } } } } } } else if (extreme_suite) { for (rshift_ = 0; rshift_ <= rshift; rshift_++) { for (elsize_ = 1; elsize_ <= elsize; elsize_++) { /* The next loop is for getting sizes that are not power of 2 */ for (i = -elsize_*2; i <= elsize_*2; i += elsize_) { for (size_ = 32*KB; size_ <= size; size_ *= 2) { nchunks = get_nchunks(size_+i, workingset); for (nthreads_ = 1; nthreads_ <= nthreads; nthreads_++) { do_bench(compressor, shuffle, nthreads_, size_+i, elsize_, rshift_, output_file); blosc_set_timestamp(¤t); totaltime = (float)getseconds(last, current); printf("Elapsed time:\t %6.1f s. Processed data: %.1f GB\n", totaltime, totalsize / GB); } } } } } } else if (debug_suite) { for (rshift_ = rshift; rshift_ <= 32; rshift_++) { for (elsize_ = elsize; elsize_ <= 32; elsize_++) { /* The next loop is for getting sizes that are not power of 2 */ for (i = -elsize_*2; i <= elsize_*2; i += elsize_) { for (size_ = size; size_ <= 16*MB; size_ *= 2) { nchunks = get_nchunks(size_+i, workingset); for (nthreads_ = nthreads; nthreads_ <= 6; nthreads_++) { do_bench(compressor, shuffle, nthreads_, size_+i, elsize_, rshift_, output_file); blosc_set_timestamp(¤t); totaltime = (float)getseconds(last, current); printf("Elapsed time:\t %6.1f s. Processed data: %.1f GB\n", totaltime, totalsize / GB); } } } } } } /* Single mode */ else { do_bench(compressor, shuffle, nthreads, size, elsize, rshift, output_file); } /* Print out some statistics */ blosc_set_timestamp(¤t); totaltime = (float)getseconds(last, current); printf("\nRound-trip compr/decompr on %.1f GB\n", totalsize / GB); printf("Elapsed time:\t %6.1f s, %.1f MB/s\n", totaltime, totalsize*2*1.1/(MB*totaltime)); /* Free blosc resources */ blosc_free_resources(); blosc_destroy(); return 0; } c-blosc-1.7.0/bench/plot-speeds.py000066400000000000000000000153041254622165100167400ustar00rootroot00000000000000"""Script for plotting the results of the 'suite' benchmark. Invoke without parameters for usage hints. :Author: Francesc Alted :Date: 2010-06-01 """ import matplotlib as mpl from pylab import * KB_ = 1024 MB_ = 1024*KB_ GB_ = 1024*MB_ NCHUNKS = 128 # keep in sync with bench.c linewidth=2 #markers= ['+', ',', 'o', '.', 's', 'v', 'x', '>', '<', '^'] #markers= [ 'x', '+', 'o', 's', 'v', '^', '>', '<', ] markers= [ 's', 'o', 'v', '^', '+', 'x', '>', '<', '.', ',' ] markersize = 8 def get_values(filename): f = open(filename) values = {"memcpyw": [], "memcpyr": []} for line in f: if line.startswith('-->'): tmp = line.split('-->')[1] nthreads, size, elsize, sbits, codec = [i for i in tmp.split(', ')] nthreads, size, elsize, sbits = map(int, (nthreads, size, elsize, sbits)) values["size"] = size * NCHUNKS / MB_; values["elsize"] = elsize; values["sbits"] = sbits; values["codec"] = codec # New run for nthreads (ratios, speedsw, speedsr) = ([], [], []) # Add a new entry for (ratios, speedw, speedr) values[nthreads] = (ratios, speedsw, speedsr) #print "-->", nthreads, size, elsize, sbits elif line.startswith('memcpy(write):'): tmp = line.split(',')[1] memcpyw = float(tmp.split(' ')[1]) values["memcpyw"].append(memcpyw) elif line.startswith('memcpy(read):'): tmp = line.split(',')[1] memcpyr = float(tmp.split(' ')[1]) values["memcpyr"].append(memcpyr) elif line.startswith('comp(write):'): tmp = line.split(',')[1] speedw = float(tmp.split(' ')[1]) ratio = float(line.split(':')[-1]) speedsw.append(speedw) ratios.append(ratio) elif line.startswith('decomp(read):'): tmp = line.split(',')[1] speedr = float(tmp.split(' ')[1]) speedsr.append(speedr) if "OK" not in line: print "WARNING! OK not found in decomp line!" f.close() return nthreads, values def show_plot(plots, yaxis, legends, gtitle, xmax=None): xlabel('Compresssion ratio') ylabel('Speed (MB/s)') title(gtitle) xlim(0, xmax) #ylim(0, 10000) ylim(0, None) grid(True) # legends = [f[f.find('-'):f.index('.out')] for f in filenames] # legends = [l.replace('-', ' ') for l in legends] #legend([p[0] for p in plots], legends, loc = "upper left") legend([p[0] for p in plots if not isinstance(p, mpl.lines.Line2D)], legends, loc = "best") #subplots_adjust(bottom=0.2, top=None, wspace=0.2, hspace=0.2) if outfile: print "Saving plot to:", outfile savefig(outfile, dpi=64) else: show() if __name__ == '__main__': from optparse import OptionParser usage = "usage: %prog [-r] [-o outfile] [-t title ] [-d|-c] filename" compress_title = 'Compression speed' decompress_title = 'Decompression speed' yaxis = 'No axis name' parser = OptionParser(usage=usage) parser.add_option('-o', '--outfile', dest='outfile', help=('filename for output (many extensions ' 'supported, e.g. .png, .jpg, .pdf)')) parser.add_option('-t', '--title', dest='title', help='title of the plot',) parser.add_option('-l', '--limit', dest='limit', help='expression to limit number of threads shown',) parser.add_option('-x', '--xmax', dest='xmax', help='limit the x-axis', default=None) parser.add_option('-r', '--report', action='store_true', dest='report', help='generate file for reporting ', default=False) parser.add_option('-d', '--decompress', action='store_true', dest='dspeed', help='plot decompression data', default=False) parser.add_option('-c', '--compress', action='store_true', dest='cspeed', help='plot compression data', default=False) (options, args) = parser.parse_args() if len(args) == 0: parser.error("No input arguments") elif len(args) > 1: parser.error("Too many input arguments") else: pass if options.report and options.outfile: parser.error("Can only select one of [-r, -o]") if options.dspeed and options.cspeed: parser.error("Can only select one of [-d, -c]") elif options.cspeed: options.dspeed = False plot_title = compress_title else: # either neither or dspeed options.dspeed = True plot_title = decompress_title filename = args[0] cspeed = options.cspeed dspeed = options.dspeed if options.outfile: outfile = options.outfile elif options.report: if cspeed: outfile = filename[:filename.rindex('.')] + '-compr.png' else: outfile = filename[:filename.rindex('.')] + '-decompr.png' else: outfile = None plots = [] legends = [] nthreads, values = get_values(filename) #print "Values:", values if options.limit: thread_range = eval(options.limit) else: thread_range = range(1, nthreads+1) if options.title: plot_title = options.title else: plot_title += " (%(size).1f MB, %(elsize)d bytes, %(sbits)d bits), %(codec)s" % values gtitle = plot_title for nt in thread_range: #print "Values for %s threads --> %s" % (nt, values[nt]) (ratios, speedw, speedr) = values[nt] if cspeed: speed = speedw else: speed = speedr #plot_ = semilogx(ratios, speed, linewidth=2) plot_ = plot(ratios, speed, linewidth=2) plots.append(plot_) nmarker = nt if nt >= len(markers): nmarker = nt%len(markers) setp(plot_, marker=markers[nmarker], markersize=markersize, linewidth=linewidth) legends.append("%d threads" % nt) # Add memcpy lines if cspeed: mean = np.mean(values["memcpyw"]) message = "memcpy (write to memory)" else: mean = np.mean(values["memcpyr"]) message = "memcpy (read from memory)" plot_ = axhline(mean, linewidth=3, linestyle='-.', color='black') text(1.0, mean+50, message) plots.append(plot_) show_plot(plots, yaxis, legends, gtitle, xmax=int(options.xmax) if options.xmax else None) c-blosc-1.7.0/blosc/000077500000000000000000000000001254622165100141475ustar00rootroot00000000000000c-blosc-1.7.0/blosc/CMakeLists.txt000066400000000000000000000145531254622165100167170ustar00rootroot00000000000000# a simple way to detect that we are using CMAKE add_definitions(-DUSING_CMAKE) set(INTERNAL_LIBS ${CMAKE_SOURCE_DIR}/internal-complibs) # Hide symbols by default unless they're specifically exported. # This makes it easier to keep the set of exported symbols the # same across all compilers/platforms. set(CMAKE_C_VISIBILITY_PRESET hidden) # includes if(NOT DEACTIVATE_LZ4) if (LZ4_FOUND) include_directories( ${LZ4_INCLUDE_DIR} ) else(LZ4_FOUND) set(LZ4_LOCAL_DIR ${INTERNAL_LIBS}/lz4-1.7.0) include_directories( ${LZ4_LOCAL_DIR} ) endif(LZ4_FOUND) endif(NOT DEACTIVATE_LZ4) if(NOT DEACTIVATE_SNAPPY) if (SNAPPY_FOUND) include_directories( ${SNAPPY_INCLUDE_DIR} ) else(SNAPPY_FOUND) set(SNAPPY_LOCAL_DIR ${INTERNAL_LIBS}/snappy-1.1.1) include_directories( ${SNAPPY_LOCAL_DIR} ) endif(SNAPPY_FOUND) endif(NOT DEACTIVATE_SNAPPY) if(NOT DEACTIVATE_ZLIB) if (ZLIB_FOUND) include_directories( ${ZLIB_INCLUDE_DIR} ) else(ZLIB_FOUND) set(ZLIB_LOCAL_DIR ${INTERNAL_LIBS}/zlib-1.2.8) include_directories( ${ZLIB_LOCAL_DIR} ) endif(ZLIB_FOUND) endif(NOT DEACTIVATE_ZLIB) # library sources set(SOURCES blosc.c blosclz.c shuffle-generic.c bitshuffle-generic.c) if(COMPILER_SUPPORT_SSE2) message(STATUS "Adding run-time support for SSE2") set(SOURCES ${SOURCES} shuffle-sse2.c bitshuffle-sse2.c) endif(COMPILER_SUPPORT_SSE2) if(COMPILER_SUPPORT_AVX2) message(STATUS "Adding run-time support for AVX2") set(SOURCES ${SOURCES} shuffle-avx2.c bitshuffle-avx2.c) endif(COMPILER_SUPPORT_AVX2) set(SOURCES ${SOURCES} shuffle.c) # library install directory set(lib_dir lib${LIB_SUFFIX}) set(version_string ${BLOSC_VERSION_MAJOR}.${BLOSC_VERSION_MINOR}.${BLOSC_VERSION_PATCH}) set(CMAKE_THREAD_PREFER_PTHREAD TRUE) if(WIN32) # try to use the system library find_package(Threads) if(NOT Threads_FOUND) message(STATUS "using the internal pthread library for win32 systems.") set(SOURCES ${SOURCES} win32/pthread.c) else(NOT Threads_FOUND) set(LIBS ${LIBS} ${CMAKE_THREAD_LIBS_INIT}) endif(NOT Threads_FOUND) else(WIN32) find_package(Threads REQUIRED) set(LIBS ${LIBS} ${CMAKE_THREAD_LIBS_INIT}) endif(WIN32) if(NOT DEACTIVATE_LZ4) if(LZ4_FOUND) set(LIBS ${LIBS} ${LZ4_LIBRARY}) else(LZ4_FOUND) file(GLOB LZ4_FILES ${LZ4_LOCAL_DIR}/*.c) set(SOURCES ${SOURCES} ${LZ4_FILES}) endif(LZ4_FOUND) endif(NOT DEACTIVATE_LZ4) if(NOT DEACTIVATE_SNAPPY) if(SNAPPY_FOUND) set(LIBS ${LIBS} ${SNAPPY_LIBRARY}) else(SNAPPY_FOUND) file(GLOB SNAPPY_FILES ${SNAPPY_LOCAL_DIR}/*.cc) set(SOURCES ${SOURCES} ${SNAPPY_FILES}) endif(SNAPPY_FOUND) endif(NOT DEACTIVATE_SNAPPY) if(NOT DEACTIVATE_ZLIB) if(ZLIB_FOUND) set(LIBS ${LIBS} ${ZLIB_LIBRARY}) else(ZLIB_FOUND) file(GLOB ZLIB_FILES ${ZLIB_LOCAL_DIR}/*.c) set(SOURCES ${SOURCES} ${ZLIB_FILES}) endif(ZLIB_FOUND) endif(NOT DEACTIVATE_ZLIB) # targets add_library(blosc_shared SHARED ${SOURCES}) set_target_properties(blosc_shared PROPERTIES OUTPUT_NAME blosc) set_target_properties(blosc_shared PROPERTIES VERSION ${version_string} SOVERSION 1 # Change this when an ABI change happens ) set_property( TARGET blosc_shared APPEND PROPERTY COMPILE_DEFINITIONS BLOSC_SHARED_LIBRARY) # Based on the target architecture and hardware features supported # by the C compiler, set hardware architecture optimization flags # for specific shuffle implementations. if(COMPILER_SUPPORT_SSE2) if (MSVC) # MSVC targets SSE2 by default on 64-bit configurations, but not 32-bit configurations. if (${CMAKE_SIZEOF_VOID_P} EQUAL 4) set_source_files_properties(shuffle-sse2.c bitshuffle-sse2.c PROPERTIES COMPILE_FLAGS "/arch:SSE2") endif (${CMAKE_SIZEOF_VOID_P} EQUAL 4) else (MSVC) set_source_files_properties(shuffle-sse2.c bitshuffle-sse2.c PROPERTIES COMPILE_FLAGS -msse2) endif (MSVC) # Define a symbol for the shuffle-dispatch implementation # so it knows SSE2 is supported even though that file is # compiled without SSE2 support (for portability). set_property( SOURCE shuffle.c APPEND PROPERTY COMPILE_DEFINITIONS SHUFFLE_SSE2_ENABLED) endif(COMPILER_SUPPORT_SSE2) if(COMPILER_SUPPORT_AVX2) if (MSVC) set_source_files_properties(shuffle-avx2.c bitshuffle-avx2.c PROPERTIES COMPILE_FLAGS "/arch:AVX2") else (MSVC) set_source_files_properties(shuffle-avx2.c bitshuffle-avx2.c PROPERTIES COMPILE_FLAGS -mavx2) endif (MSVC) # Define a symbol for the shuffle-dispatch implementation # so it knows AVX2 is supported even though that file is # compiled without AVX2 support (for portability). set_property( SOURCE shuffle.c APPEND PROPERTY COMPILE_DEFINITIONS SHUFFLE_AVX2_ENABLED) endif(COMPILER_SUPPORT_AVX2) # When the option has been selected to compile the test suite, # compile an additional version of blosc_shared which exports # some normally-hidden symbols (to facilitate unit testing). if (BUILD_TESTS) add_library(blosc_shared_testing SHARED ${SOURCES}) set_target_properties(blosc_shared_testing PROPERTIES OUTPUT_NAME blosc_testing) set_property( TARGET blosc_shared_testing APPEND PROPERTY COMPILE_DEFINITIONS BLOSC_SHARED_LIBRARY) set_property( TARGET blosc_shared_testing APPEND PROPERTY COMPILE_DEFINITIONS BLOSC_TESTING) # TEMP : CMake doesn't automatically add -lpthread here like it does # for the blosc_shared target. Force it for now. if(UNIX) set_property( TARGET blosc_shared_testing APPEND PROPERTY LINK_FLAGS "-lpthread") endif() endif() target_link_libraries(blosc_shared ${LIBS}) if (BUILD_TESTS) target_link_libraries(blosc_shared_testing ${LIBS}) endif() if(BUILD_STATIC) add_library(blosc_static STATIC ${SOURCES}) set_target_properties(blosc_static PROPERTIES OUTPUT_NAME blosc) if (MSVC) set_target_properties(blosc_static PROPERTIES PREFIX lib) endif() target_link_libraries(blosc_static ${LIBS}) endif(BUILD_STATIC) # install install(FILES blosc.h blosc-export.h DESTINATION include COMPONENT DEV) install(TARGETS blosc_shared DESTINATION ${lib_dir} COMPONENT LIB) if(BUILD_STATIC) install(TARGETS blosc_static DESTINATION ${lib_dir} COMPONENT DEV) endif(BUILD_STATIC) c-blosc-1.7.0/blosc/bitshuffle-avx2.c000066400000000000000000000176711254622165100173400ustar00rootroot00000000000000/* * Bitshuffle - Filter for improving compression of typed binary data. * * Author: Kiyoshi Masui * Website: http://www.github.com/kiyo-masui/bitshuffle * Created: 2014 * * Note: Adapted for c-blosc by Francesc Alted. * * See LICENSES/BITSHUFFLE.txt file for details about copyright and * rights to use. * */ #include "bitshuffle-generic.h" #include "bitshuffle-sse2.h" #include "bitshuffle-avx2.h" /* Make sure AVX2 is available for the compilation target and compiler. */ #if !defined(__AVX2__) #error AVX2 is not supported by the target architecture/platform and/or this compiler. #endif #include /* The next is useful for debugging purposes */ #if 0 #include #include static void printymm(__m256i ymm0) { uint8_t buf[32]; ((__m256i *)buf)[0] = ymm0; printf("%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x\n", buf[0], buf[1], buf[2], buf[3], buf[4], buf[5], buf[6], buf[7], buf[8], buf[9], buf[10], buf[11], buf[12], buf[13], buf[14], buf[15], buf[16], buf[17], buf[18], buf[19], buf[20], buf[21], buf[22], buf[23], buf[24], buf[25], buf[26], buf[27], buf[28], buf[29], buf[30], buf[31]); } #endif /* ---- Code that requires AVX2. Intel Haswell (2013) and later. ---- */ /* Transpose bits within bytes. */ int64_t bshuf_trans_bit_byte_avx2(void* in, void* out, const size_t size, const size_t elem_size) { char* in_b = (char*) in; char* out_b = (char*) out; int32_t* out_i32; size_t nbyte = elem_size * size; int64_t count; __m256i ymm; int32_t bt; size_t ii, kk; for (ii = 0; ii + 31 < nbyte; ii += 32) { ymm = _mm256_loadu_si256((__m256i *) &in_b[ii]); for (kk = 0; kk < 8; kk++) { bt = _mm256_movemask_epi8(ymm); ymm = _mm256_slli_epi16(ymm, 1); out_i32 = (int32_t*) &out_b[((7 - kk) * nbyte + ii) / 8]; *out_i32 = bt; } } count = bshuf_trans_bit_byte_remainder(in, out, size, elem_size, nbyte - nbyte % 32); return count; } /* Transpose bits within elements. */ int64_t bshuf_trans_bit_elem_avx2(void* in, void* out, const size_t size, const size_t elem_size, void* tmp_buf) { int64_t count; CHECK_MULT_EIGHT(size); count = bshuf_trans_byte_elem_sse2(in, out, size, elem_size, tmp_buf); CHECK_ERR(count); count = bshuf_trans_bit_byte_avx2(out, tmp_buf, size, elem_size); CHECK_ERR(count); count = bshuf_trans_bitrow_eight(tmp_buf, out, size, elem_size); return count; } /* For data organized into a row for each bit (8 * elem_size rows), transpose * the bytes. */ int64_t bshuf_trans_byte_bitrow_avx2(void* in, void* out, const size_t size, const size_t elem_size) { char* in_b = (char*) in; char* out_b = (char*) out; size_t nrows = 8 * elem_size; size_t nbyte_row = size / 8; size_t ii, jj, kk, hh, mm; CHECK_MULT_EIGHT(size); if (elem_size % 4) return bshuf_trans_byte_bitrow_sse2(in, out, size, elem_size); __m256i ymm_0[8]; __m256i ymm_1[8]; __m256i ymm_storeage[8][4]; for (jj = 0; jj + 31 < nbyte_row; jj += 32) { for (ii = 0; ii + 3 < elem_size; ii += 4) { for (hh = 0; hh < 4; hh ++) { for (kk = 0; kk < 8; kk ++){ ymm_0[kk] = _mm256_loadu_si256((__m256i *) &in_b[ (ii * 8 + hh * 8 + kk) * nbyte_row + jj]); } for (kk = 0; kk < 4; kk ++){ ymm_1[kk] = _mm256_unpacklo_epi8(ymm_0[kk * 2], ymm_0[kk * 2 + 1]); ymm_1[kk + 4] = _mm256_unpackhi_epi8(ymm_0[kk * 2], ymm_0[kk * 2 + 1]); } for (kk = 0; kk < 2; kk ++){ for (mm = 0; mm < 2; mm ++){ ymm_0[kk * 4 + mm] = _mm256_unpacklo_epi16( ymm_1[kk * 4 + mm * 2], ymm_1[kk * 4 + mm * 2 + 1]); ymm_0[kk * 4 + mm + 2] = _mm256_unpackhi_epi16( ymm_1[kk * 4 + mm * 2], ymm_1[kk * 4 + mm * 2 + 1]); } } for (kk = 0; kk < 4; kk ++){ ymm_1[kk * 2] = _mm256_unpacklo_epi32(ymm_0[kk * 2], ymm_0[kk * 2 + 1]); ymm_1[kk * 2 + 1] = _mm256_unpackhi_epi32(ymm_0[kk * 2], ymm_0[kk * 2 + 1]); } for (kk = 0; kk < 8; kk ++){ ymm_storeage[kk][hh] = ymm_1[kk]; } } for (mm = 0; mm < 8; mm ++) { for (kk = 0; kk < 4; kk ++){ ymm_0[kk] = ymm_storeage[mm][kk]; } ymm_1[0] = _mm256_unpacklo_epi64(ymm_0[0], ymm_0[1]); ymm_1[1] = _mm256_unpacklo_epi64(ymm_0[2], ymm_0[3]); ymm_1[2] = _mm256_unpackhi_epi64(ymm_0[0], ymm_0[1]); ymm_1[3] = _mm256_unpackhi_epi64(ymm_0[2], ymm_0[3]); ymm_0[0] = _mm256_permute2x128_si256(ymm_1[0], ymm_1[1], 32); ymm_0[1] = _mm256_permute2x128_si256(ymm_1[2], ymm_1[3], 32); ymm_0[2] = _mm256_permute2x128_si256(ymm_1[0], ymm_1[1], 49); ymm_0[3] = _mm256_permute2x128_si256(ymm_1[2], ymm_1[3], 49); _mm256_storeu_si256((__m256i *) &out_b[ (jj + mm * 2 + 0 * 16) * nrows + ii * 8], ymm_0[0]); _mm256_storeu_si256((__m256i *) &out_b[ (jj + mm * 2 + 0 * 16 + 1) * nrows + ii * 8], ymm_0[1]); _mm256_storeu_si256((__m256i *) &out_b[ (jj + mm * 2 + 1 * 16) * nrows + ii * 8], ymm_0[2]); _mm256_storeu_si256((__m256i *) &out_b[ (jj + mm * 2 + 1 * 16 + 1) * nrows + ii * 8], ymm_0[3]); } } } for (ii = 0; ii < nrows; ii ++ ) { for (jj = nbyte_row - nbyte_row % 32; jj < nbyte_row; jj ++) { out_b[jj * nrows + ii] = in_b[ii * nbyte_row + jj]; } } return size * elem_size; } /* Shuffle bits within the bytes of eight element blocks. */ int64_t bshuf_shuffle_bit_eightelem_avx2(void* in, void* out, const size_t size, const size_t elem_size) { CHECK_MULT_EIGHT(size); /* With a bit of care, this could be written such that such that it is */ /* in_buf = out_buf safe. */ char* in_b = (char*) in; char* out_b = (char*) out; size_t nbyte = elem_size * size; size_t ii, jj, kk, ind; __m256i ymm; int32_t bt; if (elem_size % 4) { return bshuf_shuffle_bit_eightelem_sse2(in, out, size, elem_size); } else { for (jj = 0; jj + 31 < 8 * elem_size; jj += 32) { for (ii = 0; ii + 8 * elem_size - 1 < nbyte; ii += 8 * elem_size) { ymm = _mm256_loadu_si256((__m256i *) &in_b[ii + jj]); for (kk = 0; kk < 8; kk++) { bt = _mm256_movemask_epi8(ymm); ymm = _mm256_slli_epi16(ymm, 1); ind = (ii + jj / 8 + (7 - kk) * elem_size); * (int32_t *) &out_b[ind] = bt; } } } } return size * elem_size; } /* Untranspose bits within elements. */ int64_t bshuf_untrans_bit_elem_avx2(void* in, void* out, const size_t size, const size_t elem_size, void* tmp_buf) { int64_t count; CHECK_MULT_EIGHT(size); count = bshuf_trans_byte_bitrow_avx2(in, tmp_buf, size, elem_size); CHECK_ERR(count); count = bshuf_shuffle_bit_eightelem_avx2(tmp_buf, out, size, elem_size); return count; } c-blosc-1.7.0/blosc/bitshuffle-avx2.h000066400000000000000000000016301254622165100173310ustar00rootroot00000000000000/********************************************************************* Blosc - Blocked Shuffling and Compression Library Author: Francesc Alted See LICENSES/BLOSC.txt for details about copyright and rights to use. **********************************************************************/ /* AVX2-accelerated shuffle/unshuffle routines. */ #ifndef BITSHUFFLE_AVX2_H #define BITSHUFFLE_AVX2_H #include "shuffle-common.h" #ifdef __cplusplus extern "C" { #endif /** AVX2-accelerated bitshuffle routine. */ BLOSC_NO_EXPORT int64_t bshuf_trans_bit_elem_avx2(void* in, void* out, const size_t size, const size_t elem_size, void* tmp_buf); /** AVX2-accelerated bitunshuffle routine. */ BLOSC_NO_EXPORT int64_t bshuf_untrans_bit_elem_avx2(void* in, void* out, const size_t size, const size_t elem_size, void* tmp_buf); #ifdef __cplusplus } #endif #endif /* BITSHUFFLE_AVX2_H */ c-blosc-1.7.0/blosc/bitshuffle-generic.c000066400000000000000000000130301254622165100200550ustar00rootroot00000000000000/********************************************************************* Blosc - Blocked Shuffling and Compression Library Author: Francesc Alted See LICENSES/BLOSC.txt for details about copyright and rights to use. **********************************************************************/ #include "bitshuffle-generic.h" /* Transpose bytes within elements, starting partway through input. */ int64_t bshuf_trans_byte_elem_remainder(void* in, void* out, const size_t size, const size_t elem_size, const size_t start) { char* in_b = (char*) in; char* out_b = (char*) out; size_t ii, jj, kk; CHECK_MULT_EIGHT(start); if (size > start) { /* ii loop separated into 2 loops so the compiler can unroll */ /* the inner one. */ for (ii = start; ii + 7 < size; ii += 8) { for (jj = 0; jj < elem_size; jj++) { for (kk = 0; kk < 8; kk++) { out_b[jj * size + ii + kk] = in_b[ii * elem_size + kk * elem_size + jj]; } } } for (ii = size - size % 8; ii < size; ii ++) { for (jj = 0; jj < elem_size; jj++) { out_b[jj * size + ii] = in_b[ii * elem_size + jj]; } } } return size * elem_size; } /* Transpose bytes within elements. */ int64_t bshuf_trans_byte_elem_scal(void* in, void* out, const size_t size, const size_t elem_size) { return bshuf_trans_byte_elem_remainder(in, out, size, elem_size, 0); } /* Transpose bits within bytes. */ int64_t bshuf_trans_bit_byte_remainder(void* in, void* out, const size_t size, const size_t elem_size, const size_t start_byte) { int64_t* in_b = in; int8_t* out_b = out; int64_t x, t; size_t nbyte = elem_size * size; size_t nbyte_bitrow = nbyte / 8; size_t ii; int kk; CHECK_MULT_EIGHT(nbyte); CHECK_MULT_EIGHT(start_byte); for (ii = start_byte / 8; ii < nbyte_bitrow; ii ++) { x = in_b[ii]; TRANS_BIT_8X8(x, t); for (kk = 0; kk < 8; kk ++) { out_b[kk * nbyte_bitrow + ii] = x; x = x >> 8; } } return size * elem_size; } /* Transpose bits within bytes. */ int64_t bshuf_trans_bit_byte_scal(void* in, void* out, const size_t size, const size_t elem_size) { return bshuf_trans_bit_byte_remainder(in, out, size, elem_size, 0); } /* General transpose of an array, optimized for large element sizes. */ int64_t bshuf_trans_elem(void* in, void* out, const size_t lda, const size_t ldb, const size_t elem_size) { char* in_b = (char*) in; char* out_b = (char*) out; size_t ii, jj; for (ii = 0; ii < lda; ii++) { for (jj = 0; jj < ldb; jj++) { memcpy(&out_b[(jj*lda + ii) * elem_size], &in_b[(ii*ldb + jj) * elem_size], elem_size); } } return lda * ldb * elem_size; } /* Transpose rows of shuffled bits (size / 8 bytes) within groups of 8. */ int64_t bshuf_trans_bitrow_eight(void* in, void* out, const size_t size, const size_t elem_size) { size_t nbyte_bitrow = size / 8; CHECK_MULT_EIGHT(size); return bshuf_trans_elem(in, out, 8, elem_size, nbyte_bitrow); } /* Transpose bits within elements. */ int64_t bshuf_trans_bit_elem_scal(void* in, void* out, const size_t size, const size_t elem_size, void* tmp_buf) { int64_t count; CHECK_MULT_EIGHT(size); count = bshuf_trans_byte_elem_scal(in, out, size, elem_size); CHECK_ERR(count); count = bshuf_trans_bit_byte_scal(out, tmp_buf, size, elem_size); CHECK_ERR(count); count = bshuf_trans_bitrow_eight(tmp_buf, out, size, elem_size); return count; } /* For data organized into a row for each bit (8 * elem_size rows), transpose * the bytes. */ int64_t bshuf_trans_byte_bitrow_scal(void* in, void* out, const size_t size, const size_t elem_size) { char* in_b = (char*) in; char* out_b = (char*) out; size_t nbyte_row = size / 8; size_t ii, jj, kk; CHECK_MULT_EIGHT(size); for (jj = 0; jj < elem_size; jj++) { for (ii = 0; ii < nbyte_row; ii++) { for (kk = 0; kk < 8; kk++) { out_b[ii * 8 * elem_size + jj * 8 + kk] = \ in_b[(jj * 8 + kk) * nbyte_row + ii]; } } } return size * elem_size; } /* Shuffle bits within the bytes of eight element blocks. */ int64_t bshuf_shuffle_bit_eightelem_scal(void* in, void* out, const size_t size, const size_t elem_size) { char* in_b = (char*) in; char* out_b = (char*) out; size_t nbyte = elem_size * size; int64_t x, t; size_t jj, ii, kk; CHECK_MULT_EIGHT(size); for (jj = 0; jj < 8 * elem_size; jj += 8) { for (ii = 0; ii + 8 * elem_size - 1 < nbyte; ii += 8 * elem_size) { x = *((int64_t*) &in_b[ii + jj]); TRANS_BIT_8X8(x, t); for (kk = 0; kk < 8; kk++) { *((uint8_t*) &out_b[ii + jj / 8 + kk * elem_size]) = x; x = x >> 8; } } } return size * elem_size; } /* Untranspose bits within elements. */ int64_t bshuf_untrans_bit_elem_scal(void* in, void* out, const size_t size, const size_t elem_size, void* tmp_buf) { int64_t count; CHECK_MULT_EIGHT(size); count = bshuf_trans_byte_bitrow_scal(in, tmp_buf, size, elem_size); CHECK_ERR(count); count = bshuf_shuffle_bit_eightelem_scal(tmp_buf, out, size, elem_size); return count; } c-blosc-1.7.0/blosc/bitshuffle-generic.h000066400000000000000000000127021254622165100200670ustar00rootroot00000000000000/********************************************************************* Blosc - Blocked Shuffling and Compression Library Author: Francesc Alted See LICENSES/BLOSC.txt for details about copyright and rights to use. **********************************************************************/ /* Generic (non-hardware-accelerated) shuffle/unshuffle routines. These are used when hardware-accelerated functions aren't available for a particular platform; they are also used by the hardware- accelerated functions to handle any remaining elements in a block which isn't a multiple of the hardware's vector size. */ #ifndef BITSHUFFLE_GENERIC_H #define BITSHUFFLE_GENERIC_H #include "shuffle-common.h" #include #ifdef __cplusplus extern "C" { #endif /* Macros. */ #define CHECK_MULT_EIGHT(n) if (n % 8) return -80; #define MIN(X,Y) ((X) < (Y) ? (X) : (Y)) #define MAX(X,Y) ((X) > (Y) ? (X) : (Y)) #define CHECK_ERR(count) if (count < 0) { return count; } /* ---- Worker code not requiring special instruction sets. ---- * * The following code does not use any x86 specific vectorized instructions * and should compile on any machine * */ /* Transpose 8x8 bit array packed into a single quadword *x*. * *t* is workspace. */ #define TRANS_BIT_8X8(x, t) { \ t = (x ^ (x >> 7)) & 0x00AA00AA00AA00AALL; \ x = x ^ t ^ (t << 7); \ t = (x ^ (x >> 14)) & 0x0000CCCC0000CCCCLL; \ x = x ^ t ^ (t << 14); \ t = (x ^ (x >> 28)) & 0x00000000F0F0F0F0LL; \ x = x ^ t ^ (t << 28); \ } /* Transpose of an array of arbitrarily typed elements. */ #define TRANS_ELEM_TYPE(in, out, lda, ldb, type_t) { \ type_t* in_type = (type_t*) in; \ type_t* out_type = (type_t*) out; \ size_t ii, jj, kk; \ for (ii = 0; ii + 7 < lda; ii += 8) { \ for (jj = 0; jj < ldb; jj++) { \ for (kk = 0; kk < 8; kk++) { \ out_type[jj*lda + ii + kk] = \ in_type[ii*ldb + kk * ldb + jj]; \ } \ } \ } \ for (ii = lda - lda % 8; ii < lda; ii ++) { \ for (jj = 0; jj < ldb; jj++) { \ out_type[jj*lda + ii] = in_type[ii*ldb + jj]; \ } \ } \ } /* Private functions */ BLOSC_NO_EXPORT int64_t bshuf_trans_byte_elem_remainder(void* in, void* out, const size_t size, const size_t elem_size, const size_t start); BLOSC_NO_EXPORT int64_t bshuf_trans_byte_elem_scal(void* in, void* out, const size_t size, const size_t elem_size); BLOSC_NO_EXPORT int64_t bshuf_trans_bit_byte_remainder(void* in, void* out, const size_t size, const size_t elem_size, const size_t start_byte); BLOSC_NO_EXPORT int64_t bshuf_trans_elem(void* in, void* out, const size_t lda, const size_t ldb, const size_t elem_size); BLOSC_NO_EXPORT int64_t bshuf_trans_bitrow_eight(void* in, void* out, const size_t size, const size_t elem_size); BLOSC_NO_EXPORT int64_t bshuf_shuffle_bit_eightelem_scal(void* in, void* out, const size_t size, const size_t elem_size); /* Bitshuffle the data. * * Transpose the bits within elements. * * Parameters * ---------- * in : input buffer, must be of size * elem_size bytes * out : output buffer, must be of size * elem_size bytes * size : number of elements in input * elem_size : element size of typed data * tmp_buffer : temporary buffer with the same `size` than `in` and `out` * * Returns * ------- * nothing -- this cannot fail * */ BLOSC_NO_EXPORT int64_t bshuf_trans_bit_elem_scal(void* in, void* out, const size_t size, const size_t elem_size, void* tmp_buf); /* Unshuffle bitshuffled data. * * Untranspose the bits within elements. * * To properly unshuffle bitshuffled data, *size* and *elem_size* must * match the parameters used to shuffle the data. * * Parameters * ---------- * in : input buffer, must be of size * elem_size bytes * out : output buffer, must be of size * elem_size bytes * size : number of elements in input * elem_size : element size of typed data * tmp_buffer : temporary buffer with the same `size` than `in` and `out` * * Returns * ------- * nothing -- this cannot fail * */ BLOSC_NO_EXPORT int64_t bshuf_untrans_bit_elem_scal(void* in, void* out, const size_t size, const size_t elem_size, void* tmp_buf); #ifdef __cplusplus } #endif #endif /* BITSHUFFLE_GENERIC_H */ c-blosc-1.7.0/blosc/bitshuffle-sse2.c000066400000000000000000000377011254622165100173300ustar00rootroot00000000000000/* * Bitshuffle - Filter for improving compression of typed binary data. * * Author: Kiyoshi Masui * Website: http://www.github.com/kiyo-masui/bitshuffle * Created: 2014 * * Note: Adapted for c-blosc by Francesc Alted. * * See LICENSES/BITSHUFFLE.txt file for details about copyright and * rights to use. * */ #include "bitshuffle-generic.h" #include "bitshuffle-sse2.h" /* Make sure SSE2 is available for the compilation target and compiler. */ #if !defined(__SSE2__) #error SSE2 is not supported by the target architecture/platform and/or this compiler. #endif #include /* The next is useful for debugging purposes */ #if 0 #include #include static void printxmm(__m128i xmm0) { uint8_t buf[32]; ((__m128i *)buf)[0] = xmm0; printf("%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x\n", buf[0], buf[1], buf[2], buf[3], buf[4], buf[5], buf[6], buf[7], buf[8], buf[9], buf[10], buf[11], buf[12], buf[13], buf[14], buf[15]); } #endif /* ---- Worker code that requires SSE2. Intel Petium 4 (2000) and later. ---- */ /* Transpose bytes within elements for 16 bit elements. */ int64_t bshuf_trans_byte_elem_SSE_16(void* in, void* out, const size_t size) { char* in_b = (char*) in; char* out_b = (char*) out; __m128i a0, b0, a1, b1; size_t ii; for (ii=0; ii + 15 < size; ii += 16) { a0 = _mm_loadu_si128((__m128i *) &in_b[2*ii + 0*16]); b0 = _mm_loadu_si128((__m128i *) &in_b[2*ii + 1*16]); a1 = _mm_unpacklo_epi8(a0, b0); b1 = _mm_unpackhi_epi8(a0, b0); a0 = _mm_unpacklo_epi8(a1, b1); b0 = _mm_unpackhi_epi8(a1, b1); a1 = _mm_unpacklo_epi8(a0, b0); b1 = _mm_unpackhi_epi8(a0, b0); a0 = _mm_unpacklo_epi8(a1, b1); b0 = _mm_unpackhi_epi8(a1, b1); _mm_storeu_si128((__m128i *) &out_b[0*size + ii], a0); _mm_storeu_si128((__m128i *) &out_b[1*size + ii], b0); } return bshuf_trans_byte_elem_remainder(in, out, size, 2, size - size % 16); } /* Transpose bytes within elements for 32 bit elements. */ int64_t bshuf_trans_byte_elem_SSE_32(void* in, void* out, const size_t size) { char* in_b = (char*) in; char* out_b = (char*) out; __m128i a0, b0, c0, d0, a1, b1, c1, d1; size_t ii; for (ii=0; ii + 15 < size; ii += 16) { a0 = _mm_loadu_si128((__m128i *) &in_b[4*ii + 0*16]); b0 = _mm_loadu_si128((__m128i *) &in_b[4*ii + 1*16]); c0 = _mm_loadu_si128((__m128i *) &in_b[4*ii + 2*16]); d0 = _mm_loadu_si128((__m128i *) &in_b[4*ii + 3*16]); a1 = _mm_unpacklo_epi8(a0, b0); b1 = _mm_unpackhi_epi8(a0, b0); c1 = _mm_unpacklo_epi8(c0, d0); d1 = _mm_unpackhi_epi8(c0, d0); a0 = _mm_unpacklo_epi8(a1, b1); b0 = _mm_unpackhi_epi8(a1, b1); c0 = _mm_unpacklo_epi8(c1, d1); d0 = _mm_unpackhi_epi8(c1, d1); a1 = _mm_unpacklo_epi8(a0, b0); b1 = _mm_unpackhi_epi8(a0, b0); c1 = _mm_unpacklo_epi8(c0, d0); d1 = _mm_unpackhi_epi8(c0, d0); a0 = _mm_unpacklo_epi64(a1, c1); b0 = _mm_unpackhi_epi64(a1, c1); c0 = _mm_unpacklo_epi64(b1, d1); d0 = _mm_unpackhi_epi64(b1, d1); _mm_storeu_si128((__m128i *) &out_b[0*size + ii], a0); _mm_storeu_si128((__m128i *) &out_b[1*size + ii], b0); _mm_storeu_si128((__m128i *) &out_b[2*size + ii], c0); _mm_storeu_si128((__m128i *) &out_b[3*size + ii], d0); } return bshuf_trans_byte_elem_remainder(in, out, size, 4, size - size % 16); } /* Transpose bytes within elements for 64 bit elements. */ int64_t bshuf_trans_byte_elem_SSE_64(void* in, void* out, const size_t size) { char* in_b = (char*) in; char* out_b = (char*) out; __m128i a0, b0, c0, d0, e0, f0, g0, h0; __m128i a1, b1, c1, d1, e1, f1, g1, h1; size_t ii; for (ii=0; ii + 15 < size; ii += 16) { a0 = _mm_loadu_si128((__m128i *) &in_b[8*ii + 0*16]); b0 = _mm_loadu_si128((__m128i *) &in_b[8*ii + 1*16]); c0 = _mm_loadu_si128((__m128i *) &in_b[8*ii + 2*16]); d0 = _mm_loadu_si128((__m128i *) &in_b[8*ii + 3*16]); e0 = _mm_loadu_si128((__m128i *) &in_b[8*ii + 4*16]); f0 = _mm_loadu_si128((__m128i *) &in_b[8*ii + 5*16]); g0 = _mm_loadu_si128((__m128i *) &in_b[8*ii + 6*16]); h0 = _mm_loadu_si128((__m128i *) &in_b[8*ii + 7*16]); a1 = _mm_unpacklo_epi8(a0, b0); b1 = _mm_unpackhi_epi8(a0, b0); c1 = _mm_unpacklo_epi8(c0, d0); d1 = _mm_unpackhi_epi8(c0, d0); e1 = _mm_unpacklo_epi8(e0, f0); f1 = _mm_unpackhi_epi8(e0, f0); g1 = _mm_unpacklo_epi8(g0, h0); h1 = _mm_unpackhi_epi8(g0, h0); a0 = _mm_unpacklo_epi8(a1, b1); b0 = _mm_unpackhi_epi8(a1, b1); c0 = _mm_unpacklo_epi8(c1, d1); d0 = _mm_unpackhi_epi8(c1, d1); e0 = _mm_unpacklo_epi8(e1, f1); f0 = _mm_unpackhi_epi8(e1, f1); g0 = _mm_unpacklo_epi8(g1, h1); h0 = _mm_unpackhi_epi8(g1, h1); a1 = _mm_unpacklo_epi32(a0, c0); b1 = _mm_unpackhi_epi32(a0, c0); c1 = _mm_unpacklo_epi32(b0, d0); d1 = _mm_unpackhi_epi32(b0, d0); e1 = _mm_unpacklo_epi32(e0, g0); f1 = _mm_unpackhi_epi32(e0, g0); g1 = _mm_unpacklo_epi32(f0, h0); h1 = _mm_unpackhi_epi32(f0, h0); a0 = _mm_unpacklo_epi64(a1, e1); b0 = _mm_unpackhi_epi64(a1, e1); c0 = _mm_unpacklo_epi64(b1, f1); d0 = _mm_unpackhi_epi64(b1, f1); e0 = _mm_unpacklo_epi64(c1, g1); f0 = _mm_unpackhi_epi64(c1, g1); g0 = _mm_unpacklo_epi64(d1, h1); h0 = _mm_unpackhi_epi64(d1, h1); _mm_storeu_si128((__m128i *) &out_b[0*size + ii], a0); _mm_storeu_si128((__m128i *) &out_b[1*size + ii], b0); _mm_storeu_si128((__m128i *) &out_b[2*size + ii], c0); _mm_storeu_si128((__m128i *) &out_b[3*size + ii], d0); _mm_storeu_si128((__m128i *) &out_b[4*size + ii], e0); _mm_storeu_si128((__m128i *) &out_b[5*size + ii], f0); _mm_storeu_si128((__m128i *) &out_b[6*size + ii], g0); _mm_storeu_si128((__m128i *) &out_b[7*size + ii], h0); } return bshuf_trans_byte_elem_remainder(in, out, size, 8, size - size % 16); } /* Memory copy with bshuf call signature. */ int64_t bshuf_copy(void* in, void* out, const size_t size, const size_t elem_size) { char* in_b = (char*) in; char* out_b = (char*) out; memcpy(out_b, in_b, size * elem_size); return size * elem_size; } /* Transpose bytes within elements using best SSE algorithm available. */ int64_t bshuf_trans_byte_elem_sse2(void* in, void* out, const size_t size, const size_t elem_size, void* tmp_buf) { int64_t count; /* Trivial cases: power of 2 bytes. */ switch (elem_size) { case 1: count = bshuf_copy(in, out, size, elem_size); return count; case 2: count = bshuf_trans_byte_elem_SSE_16(in, out, size); return count; case 4: count = bshuf_trans_byte_elem_SSE_32(in, out, size); return count; case 8: count = bshuf_trans_byte_elem_SSE_64(in, out, size); return count; } /* Worst case: odd number of bytes. Turns out that this is faster for */ /* (odd * 2) byte elements as well (hence % 4). */ if (elem_size % 4) { count = bshuf_trans_byte_elem_scal(in, out, size, elem_size); return count; } /* Multiple of power of 2: transpose hierarchically. */ { size_t nchunk_elem; if ((elem_size % 8) == 0) { nchunk_elem = elem_size / 8; TRANS_ELEM_TYPE(in, out, size, nchunk_elem, int64_t); count = bshuf_trans_byte_elem_SSE_64(out, tmp_buf, size * nchunk_elem); bshuf_trans_elem(tmp_buf, out, 8, nchunk_elem, size); } else if ((elem_size % 4) == 0) { nchunk_elem = elem_size / 4; TRANS_ELEM_TYPE(in, out, size, nchunk_elem, int32_t); count = bshuf_trans_byte_elem_SSE_32(out, tmp_buf, size * nchunk_elem); bshuf_trans_elem(tmp_buf, out, 4, nchunk_elem, size); } else { /* Not used since scalar algorithm is faster. */ nchunk_elem = elem_size / 2; TRANS_ELEM_TYPE(in, out, size, nchunk_elem, int16_t); count = bshuf_trans_byte_elem_SSE_16(out, tmp_buf, size * nchunk_elem); bshuf_trans_elem(tmp_buf, out, 2, nchunk_elem, size); } return count; } } /* Transpose bits within bytes. */ int64_t bshuf_trans_bit_byte_sse2(void* in, void* out, const size_t size, const size_t elem_size) { char* in_b = (char*) in; char* out_b = (char*) out; uint16_t* out_ui16; int64_t count; size_t nbyte = elem_size * size; __m128i xmm; int32_t bt; size_t ii, kk; CHECK_MULT_EIGHT(nbyte); for (ii = 0; ii + 15 < nbyte; ii += 16) { xmm = _mm_loadu_si128((__m128i *) &in_b[ii]); for (kk = 0; kk < 8; kk++) { bt = _mm_movemask_epi8(xmm); xmm = _mm_slli_epi16(xmm, 1); out_ui16 = (uint16_t*) &out_b[((7 - kk) * nbyte + ii) / 8]; *out_ui16 = bt; } } count = bshuf_trans_bit_byte_remainder(in, out, size, elem_size, nbyte - nbyte % 16); return count; } /* Transpose bits within elements. */ int64_t bshuf_trans_bit_elem_sse2(void* in, void* out, const size_t size, const size_t elem_size, void* tmp_buf) { int64_t count; CHECK_MULT_EIGHT(size); count = bshuf_trans_byte_elem_sse2(in, out, size, elem_size, tmp_buf); CHECK_ERR(count); count = bshuf_trans_bit_byte_sse2(out, tmp_buf, size, elem_size); CHECK_ERR(count); count = bshuf_trans_bitrow_eight(tmp_buf, out, size, elem_size); return count; } /* For data organized into a row for each bit (8 * elem_size rows), transpose * the bytes. */ int64_t bshuf_trans_byte_bitrow_sse2(void* in, void* out, const size_t size, const size_t elem_size) { char* in_b = (char*) in; char* out_b = (char*) out; size_t nrows = 8 * elem_size; size_t nbyte_row = size / 8; size_t ii, jj; __m128i a0, b0, c0, d0, e0, f0, g0, h0; __m128i a1, b1, c1, d1, e1, f1, g1, h1; __m128 *as, *bs, *cs, *ds, *es, *fs, *gs, *hs; CHECK_MULT_EIGHT(size); for (ii = 0; ii + 7 < nrows; ii += 8) { for (jj = 0; jj + 15 < nbyte_row; jj += 16) { a0 = _mm_loadu_si128((__m128i *) &in_b[(ii + 0)*nbyte_row + jj]); b0 = _mm_loadu_si128((__m128i *) &in_b[(ii + 1)*nbyte_row + jj]); c0 = _mm_loadu_si128((__m128i *) &in_b[(ii + 2)*nbyte_row + jj]); d0 = _mm_loadu_si128((__m128i *) &in_b[(ii + 3)*nbyte_row + jj]); e0 = _mm_loadu_si128((__m128i *) &in_b[(ii + 4)*nbyte_row + jj]); f0 = _mm_loadu_si128((__m128i *) &in_b[(ii + 5)*nbyte_row + jj]); g0 = _mm_loadu_si128((__m128i *) &in_b[(ii + 6)*nbyte_row + jj]); h0 = _mm_loadu_si128((__m128i *) &in_b[(ii + 7)*nbyte_row + jj]); a1 = _mm_unpacklo_epi8(a0, b0); b1 = _mm_unpacklo_epi8(c0, d0); c1 = _mm_unpacklo_epi8(e0, f0); d1 = _mm_unpacklo_epi8(g0, h0); e1 = _mm_unpackhi_epi8(a0, b0); f1 = _mm_unpackhi_epi8(c0, d0); g1 = _mm_unpackhi_epi8(e0, f0); h1 = _mm_unpackhi_epi8(g0, h0); a0 = _mm_unpacklo_epi16(a1, b1); b0 = _mm_unpacklo_epi16(c1, d1); c0 = _mm_unpackhi_epi16(a1, b1); d0 = _mm_unpackhi_epi16(c1, d1); e0 = _mm_unpacklo_epi16(e1, f1); f0 = _mm_unpacklo_epi16(g1, h1); g0 = _mm_unpackhi_epi16(e1, f1); h0 = _mm_unpackhi_epi16(g1, h1); a1 = _mm_unpacklo_epi32(a0, b0); b1 = _mm_unpackhi_epi32(a0, b0); c1 = _mm_unpacklo_epi32(c0, d0); d1 = _mm_unpackhi_epi32(c0, d0); e1 = _mm_unpacklo_epi32(e0, f0); f1 = _mm_unpackhi_epi32(e0, f0); g1 = _mm_unpacklo_epi32(g0, h0); h1 = _mm_unpackhi_epi32(g0, h0); /* We don't have a storeh instruction for integers, so interpret */ /* as a float. Have a storel (_mm_storel_epi64). */ as = (__m128 *) &a1; bs = (__m128 *) &b1; cs = (__m128 *) &c1; ds = (__m128 *) &d1; es = (__m128 *) &e1; fs = (__m128 *) &f1; gs = (__m128 *) &g1; hs = (__m128 *) &h1; _mm_storel_pi((__m64 *) &out_b[(jj + 0) * nrows + ii], *as); _mm_storel_pi((__m64 *) &out_b[(jj + 2) * nrows + ii], *bs); _mm_storel_pi((__m64 *) &out_b[(jj + 4) * nrows + ii], *cs); _mm_storel_pi((__m64 *) &out_b[(jj + 6) * nrows + ii], *ds); _mm_storel_pi((__m64 *) &out_b[(jj + 8) * nrows + ii], *es); _mm_storel_pi((__m64 *) &out_b[(jj + 10) * nrows + ii], *fs); _mm_storel_pi((__m64 *) &out_b[(jj + 12) * nrows + ii], *gs); _mm_storel_pi((__m64 *) &out_b[(jj + 14) * nrows + ii], *hs); _mm_storeh_pi((__m64 *) &out_b[(jj + 1) * nrows + ii], *as); _mm_storeh_pi((__m64 *) &out_b[(jj + 3) * nrows + ii], *bs); _mm_storeh_pi((__m64 *) &out_b[(jj + 5) * nrows + ii], *cs); _mm_storeh_pi((__m64 *) &out_b[(jj + 7) * nrows + ii], *ds); _mm_storeh_pi((__m64 *) &out_b[(jj + 9) * nrows + ii], *es); _mm_storeh_pi((__m64 *) &out_b[(jj + 11) * nrows + ii], *fs); _mm_storeh_pi((__m64 *) &out_b[(jj + 13) * nrows + ii], *gs); _mm_storeh_pi((__m64 *) &out_b[(jj + 15) * nrows + ii], *hs); } for (jj = nbyte_row - nbyte_row % 16; jj < nbyte_row; jj ++) { out_b[jj * nrows + ii + 0] = in_b[(ii + 0)*nbyte_row + jj]; out_b[jj * nrows + ii + 1] = in_b[(ii + 1)*nbyte_row + jj]; out_b[jj * nrows + ii + 2] = in_b[(ii + 2)*nbyte_row + jj]; out_b[jj * nrows + ii + 3] = in_b[(ii + 3)*nbyte_row + jj]; out_b[jj * nrows + ii + 4] = in_b[(ii + 4)*nbyte_row + jj]; out_b[jj * nrows + ii + 5] = in_b[(ii + 5)*nbyte_row + jj]; out_b[jj * nrows + ii + 6] = in_b[(ii + 6)*nbyte_row + jj]; out_b[jj * nrows + ii + 7] = in_b[(ii + 7)*nbyte_row + jj]; } } return size * elem_size; } /* Shuffle bits within the bytes of eight element blocks. */ int64_t bshuf_shuffle_bit_eightelem_sse2(void* in, void* out, const size_t size, const size_t elem_size) { /* With a bit of care, this could be written such that such that it is */ /* in_buf = out_buf safe. */ char* in_b = (char*) in; uint16_t* out_ui16 = (uint16_t*) out; size_t nbyte = elem_size * size; __m128i xmm; int32_t bt; size_t ii, jj, kk; size_t ind; CHECK_MULT_EIGHT(size); if (elem_size % 2) { bshuf_shuffle_bit_eightelem_scal(in, out, size, elem_size); } else { for (ii = 0; ii + 8 * elem_size - 1 < nbyte; ii += 8 * elem_size) { for (jj = 0; jj + 15 < 8 * elem_size; jj += 16) { xmm = _mm_loadu_si128((__m128i *) &in_b[ii + jj]); for (kk = 0; kk < 8; kk++) { bt = _mm_movemask_epi8(xmm); xmm = _mm_slli_epi16(xmm, 1); ind = (ii + jj / 8 + (7 - kk) * elem_size); out_ui16[ind / 2] = bt; } } } } return size * elem_size; } /* Untranspose bits within elements. */ int64_t bshuf_untrans_bit_elem_sse2(void* in, void* out, const size_t size, const size_t elem_size, void* tmp_buf) { int64_t count; CHECK_MULT_EIGHT(size); count = bshuf_trans_byte_bitrow_sse2(in, tmp_buf, size, elem_size); CHECK_ERR(count); count = bshuf_shuffle_bit_eightelem_sse2(tmp_buf, out, size, elem_size); return count; } c-blosc-1.7.0/blosc/bitshuffle-sse2.h000066400000000000000000000026231254622165100173300ustar00rootroot00000000000000/********************************************************************* Blosc - Blocked Shuffling and Compression Library Author: Francesc Alted See LICENSES/BLOSC.txt for details about copyright and rights to use. **********************************************************************/ /* SSE2-accelerated shuffle/unshuffle routines. */ #ifndef BITSHUFFLE_SSE2_H #define BITSHUFFLE_SSE2_H #include "shuffle-common.h" #ifdef __cplusplus extern "C" { #endif BLOSC_NO_EXPORT int64_t bshuf_trans_byte_elem_sse2(void* in, void* out, const size_t size, const size_t elem_size, void* tmp_buf); BLOSC_NO_EXPORT int64_t bshuf_trans_byte_bitrow_sse2(void* in, void* out, const size_t size, const size_t elem_size); BLOSC_NO_EXPORT int64_t bshuf_shuffle_bit_eightelem_sse2(void* in, void* out, const size_t size, const size_t elem_size); /** SSE2-accelerated bitshuffle routine. */ BLOSC_NO_EXPORT int64_t bshuf_trans_bit_elem_sse2(void* in, void* out, const size_t size, const size_t elem_size, void* tmp_buf); /** SSE2-accelerated bitunshuffle routine. */ BLOSC_NO_EXPORT int64_t bshuf_untrans_bit_elem_sse2(void* in, void* out, const size_t size, const size_t elem_size, void* tmp_buf); #ifdef __cplusplus } #endif #endif /* BITSHUFFLE_SSE2_H */ c-blosc-1.7.0/blosc/blosc-export.h000066400000000000000000000031471254622165100167460ustar00rootroot00000000000000/********************************************************************* Blosc - Blocked Shuffling and Compression Library Author: Francesc Alted See LICENSES/BLOSC.txt for details about copyright and rights to use. **********************************************************************/ #ifndef BLOSC_EXPORT_H #define BLOSC_EXPORT_H /* Macros for specifying exported symbols. BLOSC_EXPORT is used to decorate symbols that should be exported by the blosc shared library. BLOSC_NO_EXPORT is used to decorate symbols that should NOT be exported by the blosc shared library. */ #if defined(BLOSC_SHARED_LIBRARY) #if defined(_MSC_VER) #define BLOSC_EXPORT __declspec(dllexport) #elif (defined(__GNUC__) && __GNUC__ >= 4) || defined(__clang__) #if defined(_WIN32) || defined(__CYGWIN__) || defined(__MINGW32__) #define BLOSC_EXPORT __attribute__((dllexport)) #else #define BLOSC_EXPORT __attribute__((visibility("default"))) #endif /* defined(_WIN32) || defined(__CYGWIN__) */ #else #error Cannot determine how to define BLOSC_EXPORT for this compiler. #endif #else #define BLOSC_EXPORT #endif /* defined(BLOSC_SHARED_LIBRARY) */ #if defined(__GNUC__) || defined(__clang__) #define BLOSC_NO_EXPORT __attribute__((visibility("hidden"))) #else #define BLOSC_NO_EXPORT #endif /* defined(__GNUC__) || defined(__clang__) */ /* When testing, export everything to make it easier to implement tests. */ #if defined(BLOSC_TESTING) #undef BLOSC_NO_EXPORT #define BLOSC_NO_EXPORT BLOSC_EXPORT #endif /* defined(BLOSC_TESTING) */ #endif /* BLOSC_EXPORT_H */ c-blosc-1.7.0/blosc/blosc.c000066400000000000000000001623311254622165100154230ustar00rootroot00000000000000/********************************************************************* Blosc - Blocked Shuffling and Compression Library Author: Francesc Alted Creation date: 2009-05-20 See LICENSES/BLOSC.txt for details about copyright and rights to use. **********************************************************************/ #include #include #include #include #include #if defined(USING_CMAKE) #include "config.h" #endif /* USING_CMAKE */ #include "blosc.h" #include "shuffle.h" #include "blosclz.h" #if defined(HAVE_LZ4) #include "lz4.h" #include "lz4hc.h" #endif /* HAVE_LZ4 */ #if defined(HAVE_SNAPPY) #include "snappy-c.h" #endif /* HAVE_SNAPPY */ #if defined(HAVE_ZLIB) #include "zlib.h" #endif /* HAVE_ZLIB */ #if defined(_WIN32) && !defined(__MINGW32__) #include #include /* stdint.h only available in VS2010 (VC++ 16.0) and newer */ #if defined(_MSC_VER) && _MSC_VER < 1600 #include "win32/stdint-windows.h" #else #include #endif #include #define getpid _getpid #else #include #include #include #endif /* _WIN32 */ #if defined(_WIN32) && !defined(__GNUC__) #include "win32/pthread.h" #include "win32/pthread.c" #else #include #endif /* If C11 is supported, use it's built-in aligned allocation. */ #if __STDC_VERSION__ >= 201112L #include #endif /* Some useful units */ #define KB 1024 #define MB (1024*KB) /* Minimum buffer size to be compressed */ #define MIN_BUFFERSIZE 128 /* Cannot be smaller than 66 */ /* The maximum number of splits in a block for compression */ #define MAX_SPLITS 16 /* Cannot be larger than 128 */ /* The size of L1 cache. 32 KB is quite common nowadays. */ #define L1 (32*KB) /* Have problems using posix barriers when symbol value is 200112L */ /* This requires more investigation, but will work for the moment */ #if defined(_POSIX_BARRIERS) && ( (_POSIX_BARRIERS - 20012L) >= 0 && _POSIX_BARRIERS != 200112L) #define _POSIX_BARRIERS_MINE #endif /* Synchronization variables */ struct blosc_context { int32_t compress; /* 1 if we are doing compression 0 if decompress */ const uint8_t* src; uint8_t* dest; /* The current pos in the destination buffer */ uint8_t* header_flags; /* Flags for header. Currently booked: - 0: byte-shuffled? - 1: memcpy'ed? - 2: bit-shuffled? */ int32_t sourcesize; /* Number of bytes in source buffer (or uncompressed bytes in compressed file) */ int32_t nblocks; /* Number of total blocks in buffer */ int32_t leftover; /* Extra bytes at end of buffer */ int32_t blocksize; /* Length of the block in bytes */ int32_t typesize; /* Type size */ int32_t num_output_bytes; /* Counter for the number of output bytes */ int32_t destsize; /* Maximum size for destination buffer */ uint8_t* bstarts; /* Start of the buffer past header info */ int32_t compcode; /* Compressor code to use */ int clevel; /* Compression level (1-9) */ /* Threading */ int32_t numthreads; int32_t threads_started; int32_t end_threads; pthread_t threads[BLOSC_MAX_THREADS]; int32_t tids[BLOSC_MAX_THREADS]; pthread_mutex_t count_mutex; #ifdef _POSIX_BARRIERS_MINE pthread_barrier_t barr_init; pthread_barrier_t barr_finish; #else int32_t count_threads; pthread_mutex_t count_threads_mutex; pthread_cond_t count_threads_cv; #endif #if !defined(_WIN32) pthread_attr_t ct_attr; /* creation time attrs for threads */ #endif int32_t thread_giveup_code; /* error code when give up */ int32_t thread_nblock; /* block counter */ }; struct thread_context { struct blosc_context* parent_context; int32_t tid; uint8_t* tmp; uint8_t* tmp2; int32_t tmpblocksize; /* Used to keep track of how big the temporary buffers are */ }; /* Global context for non-contextual API */ static struct blosc_context* g_global_context; static pthread_mutex_t global_comp_mutex; static int32_t g_compressor = BLOSC_BLOSCLZ; /* the compressor to use by default */ static int32_t g_threads = 1; static int32_t g_force_blocksize = 0; static int32_t g_initlib = 0; /* Wrapped function to adjust the number of threads used by blosc */ int blosc_set_nthreads_(struct blosc_context*); /* Releases the global threadpool */ int blosc_release_threadpool(struct blosc_context* context); /* Macros for synchronization */ /* Wait until all threads are initialized */ #ifdef _POSIX_BARRIERS_MINE #define WAIT_INIT(RET_VAL, CONTEXT_PTR) \ rc = pthread_barrier_wait(&CONTEXT_PTR->barr_init); \ if (rc != 0 && rc != PTHREAD_BARRIER_SERIAL_THREAD) { \ printf("Could not wait on barrier (init): %d\n", rc); \ return((RET_VAL)); \ } #else #define WAIT_INIT(RET_VAL, CONTEXT_PTR) \ pthread_mutex_lock(&CONTEXT_PTR->count_threads_mutex); \ if (CONTEXT_PTR->count_threads < CONTEXT_PTR->numthreads) { \ CONTEXT_PTR->count_threads++; \ pthread_cond_wait(&CONTEXT_PTR->count_threads_cv, &CONTEXT_PTR->count_threads_mutex); \ } \ else { \ pthread_cond_broadcast(&CONTEXT_PTR->count_threads_cv); \ } \ pthread_mutex_unlock(&CONTEXT_PTR->count_threads_mutex); #endif /* Wait for all threads to finish */ #ifdef _POSIX_BARRIERS_MINE #define WAIT_FINISH(RET_VAL, CONTEXT_PTR) \ rc = pthread_barrier_wait(&CONTEXT_PTR->barr_finish); \ if (rc != 0 && rc != PTHREAD_BARRIER_SERIAL_THREAD) { \ printf("Could not wait on barrier (finish)\n"); \ return((RET_VAL)); \ } #else #define WAIT_FINISH(RET_VAL, CONTEXT_PTR) \ pthread_mutex_lock(&CONTEXT_PTR->count_threads_mutex); \ if (CONTEXT_PTR->count_threads > 0) { \ CONTEXT_PTR->count_threads--; \ pthread_cond_wait(&CONTEXT_PTR->count_threads_cv, &CONTEXT_PTR->count_threads_mutex); \ } \ else { \ pthread_cond_broadcast(&CONTEXT_PTR->count_threads_cv); \ } \ pthread_mutex_unlock(&CONTEXT_PTR->count_threads_mutex); #endif /* A function for aligned malloc that is portable */ static uint8_t *my_malloc(size_t size) { void *block = NULL; int res = 0; /* Do an alignment to 32 bytes because AVX2 is supported */ #if _ISOC11_SOURCE /* C11 aligned allocation. 'size' must be a multiple of the alignment. */ block = aligned_alloc(32, size); #elif defined(_WIN32) /* A (void *) cast needed for avoiding a warning with MINGW :-/ */ block = (void *)_aligned_malloc(size, 32); #elif defined __APPLE__ /* Mac OS X guarantees 16-byte alignment in small allocs */ block = malloc(size); #elif _POSIX_C_SOURCE >= 200112L || _XOPEN_SOURCE >= 600 /* Platform does have an implementation of posix_memalign */ res = posix_memalign(&block, 32, size); #else block = malloc(size); #endif /* _WIN32 */ if (block == NULL || res != 0) { printf("Error allocating memory!"); return NULL; } return (uint8_t *)block; } /* Release memory booked by my_malloc */ static void my_free(void *block) { #if defined(_WIN32) _aligned_free(block); #else free(block); #endif /* _WIN32 */ } /* Copy 4 bytes from `*pa` to int32_t, changing endianness if necessary. */ static int32_t sw32_(const uint8_t *pa) { int32_t idest; uint8_t *dest = (uint8_t *)&idest; int i = 1; /* for big/little endian detection */ char *p = (char *)&i; if (p[0] != 1) { /* big endian */ dest[0] = pa[3]; dest[1] = pa[2]; dest[2] = pa[1]; dest[3] = pa[0]; } else { /* little endian */ dest[0] = pa[0]; dest[1] = pa[1]; dest[2] = pa[2]; dest[3] = pa[3]; } return idest; } /* Copy 4 bytes from `*pa` to `*dest`, changing endianness if necessary. */ static void _sw32(uint8_t* dest, int32_t a) { uint8_t *pa = (uint8_t *)&a; int i = 1; /* for big/little endian detection */ char *p = (char *)&i; if (p[0] != 1) { /* big endian */ dest[0] = pa[3]; dest[1] = pa[2]; dest[2] = pa[1]; dest[3] = pa[0]; } else { /* little endian */ dest[0] = pa[0]; dest[1] = pa[1]; dest[2] = pa[2]; dest[3] = pa[3]; } } /* * Conversion routines between compressor and compression libraries */ /* Return the library code associated with the compressor name */ static int compname_to_clibcode(const char *compname) { if (strcmp(compname, BLOSC_BLOSCLZ_COMPNAME) == 0) return BLOSC_BLOSCLZ_LIB; if (strcmp(compname, BLOSC_LZ4_COMPNAME) == 0) return BLOSC_LZ4_LIB; if (strcmp(compname, BLOSC_LZ4HC_COMPNAME) == 0) return BLOSC_LZ4_LIB; if (strcmp(compname, BLOSC_SNAPPY_COMPNAME) == 0) return BLOSC_SNAPPY_LIB; if (strcmp(compname, BLOSC_ZLIB_COMPNAME) == 0) return BLOSC_ZLIB_LIB; return -1; } /* Return the library name associated with the compressor code */ static char *clibcode_to_clibname(int clibcode) { if (clibcode == BLOSC_BLOSCLZ_LIB) return BLOSC_BLOSCLZ_LIBNAME; if (clibcode == BLOSC_LZ4_LIB) return BLOSC_LZ4_LIBNAME; if (clibcode == BLOSC_SNAPPY_LIB) return BLOSC_SNAPPY_LIBNAME; if (clibcode == BLOSC_ZLIB_LIB) return BLOSC_ZLIB_LIBNAME; return NULL; /* should never happen */ } /* * Conversion routines between compressor names and compressor codes */ /* Get the compressor name associated with the compressor code */ int blosc_compcode_to_compname(int compcode, char **compname) { int code = -1; /* -1 means non-existent compressor code */ char *name = NULL; /* Map the compressor code */ if (compcode == BLOSC_BLOSCLZ) name = BLOSC_BLOSCLZ_COMPNAME; else if (compcode == BLOSC_LZ4) name = BLOSC_LZ4_COMPNAME; else if (compcode == BLOSC_LZ4HC) name = BLOSC_LZ4HC_COMPNAME; else if (compcode == BLOSC_SNAPPY) name = BLOSC_SNAPPY_COMPNAME; else if (compcode == BLOSC_ZLIB) name = BLOSC_ZLIB_COMPNAME; *compname = name; /* Guess if there is support for this code */ if (compcode == BLOSC_BLOSCLZ) code = BLOSC_BLOSCLZ; #if defined(HAVE_LZ4) else if (compcode == BLOSC_LZ4) code = BLOSC_LZ4; else if (compcode == BLOSC_LZ4HC) code = BLOSC_LZ4HC; #endif /* HAVE_LZ4 */ #if defined(HAVE_SNAPPY) else if (compcode == BLOSC_SNAPPY) code = BLOSC_SNAPPY; #endif /* HAVE_SNAPPY */ #if defined(HAVE_ZLIB) else if (compcode == BLOSC_ZLIB) code = BLOSC_ZLIB; #endif /* HAVE_ZLIB */ return code; } /* Get the compressor code for the compressor name. -1 if it is not available */ int blosc_compname_to_compcode(const char *compname) { int code = -1; /* -1 means non-existent compressor code */ if (strcmp(compname, BLOSC_BLOSCLZ_COMPNAME) == 0) { code = BLOSC_BLOSCLZ; } #if defined(HAVE_LZ4) else if (strcmp(compname, BLOSC_LZ4_COMPNAME) == 0) { code = BLOSC_LZ4; } else if (strcmp(compname, BLOSC_LZ4HC_COMPNAME) == 0) { code = BLOSC_LZ4HC; } #endif /* HAVE_LZ4 */ #if defined(HAVE_SNAPPY) else if (strcmp(compname, BLOSC_SNAPPY_COMPNAME) == 0) { code = BLOSC_SNAPPY; } #endif /* HAVE_SNAPPY */ #if defined(HAVE_ZLIB) else if (strcmp(compname, BLOSC_ZLIB_COMPNAME) == 0) { code = BLOSC_ZLIB; } #endif /* HAVE_ZLIB */ return code; } #if defined(HAVE_LZ4) static int lz4_wrap_compress(const char* input, size_t input_length, char* output, size_t maxout, int accel) { int cbytes; cbytes = LZ4_compress_fast(input, output, (int)input_length, (int)maxout, accel); return cbytes; } static int lz4hc_wrap_compress(const char* input, size_t input_length, char* output, size_t maxout, int clevel) { int cbytes; if (input_length > (size_t)(2<<30)) return -1; /* input larger than 1 GB is not supported */ /* clevel for lz4hc goes up to 16, at least in LZ4 1.1.3 */ cbytes = LZ4_compressHC2_limitedOutput(input, output, (int)input_length, (int)maxout, clevel*2-1); return cbytes; } static int lz4_wrap_decompress(const char* input, size_t compressed_length, char* output, size_t maxout) { size_t cbytes; cbytes = LZ4_decompress_fast(input, output, (int)maxout); if (cbytes != compressed_length) { return 0; } return (int)maxout; } #endif /* HAVE_LZ4 */ #if defined(HAVE_SNAPPY) static int snappy_wrap_compress(const char* input, size_t input_length, char* output, size_t maxout) { snappy_status status; size_t cl = maxout; status = snappy_compress(input, input_length, output, &cl); if (status != SNAPPY_OK){ return 0; } return (int)cl; } static int snappy_wrap_decompress(const char* input, size_t compressed_length, char* output, size_t maxout) { snappy_status status; size_t ul = maxout; status = snappy_uncompress(input, compressed_length, output, &ul); if (status != SNAPPY_OK){ return 0; } return (int)ul; } #endif /* HAVE_SNAPPY */ #if defined(HAVE_ZLIB) /* zlib is not very respectful with sharing name space with others. Fortunately, its names do not collide with those already in blosc. */ static int zlib_wrap_compress(const char* input, size_t input_length, char* output, size_t maxout, int clevel) { int status; uLongf cl = maxout; status = compress2( (Bytef*)output, &cl, (Bytef*)input, (uLong)input_length, clevel); if (status != Z_OK){ return 0; } return (int)cl; } static int zlib_wrap_decompress(const char* input, size_t compressed_length, char* output, size_t maxout) { int status; uLongf ul = maxout; status = uncompress( (Bytef*)output, &ul, (Bytef*)input, (uLong)compressed_length); if (status != Z_OK){ return 0; } return (int)ul; } #endif /* HAVE_ZLIB */ /* Compute acceleration for blosclz */ static int get_accel(const struct blosc_context* context) { int32_t clevel = context->clevel; int32_t typesize = context->typesize; if (clevel == 9) { return 1; } if (context->compcode == BLOSC_BLOSCLZ) { /* Compute the power of 2. See: * http://www.exploringbinary.com/ten-ways-to-check-if-an-integer-is-a-power-of-two-in-c/ */ int32_t tspow2 = ((typesize != 0) && !(typesize & (typesize - 1))); if (tspow2 && typesize < 32) { return 32; } } else if (context->compcode == BLOSC_LZ4) { /* This acceleration setting based on discussions held in: * https://groups.google.com/forum/#!topic/lz4c/zosy90P8MQw */ return (10 - clevel); } return 1; } /* Shuffle & compress a single block */ static int blosc_c(const struct blosc_context* context, int32_t blocksize, int32_t leftoverblock, int32_t ntbytes, int32_t maxbytes, const uint8_t *src, uint8_t *dest, uint8_t *tmp) { int32_t j, neblock, nsplits; int32_t cbytes; /* number of compressed bytes in split */ int32_t ctbytes = 0; /* number of compressed bytes in block */ int32_t maxout; int32_t typesize = context->typesize; const uint8_t *_tmp = src; char *compname; int accel; int bscount; if (typesize > 1) { /* Shuffling only makes sense if typesize > 1 */ if (*(context->header_flags) & BLOSC_DOSHUFFLE) { shuffle(typesize, blocksize, src, tmp); _tmp = tmp; } /* We don't allow more than 1 filter at the same time (yet) */ else if (*(context->header_flags) & BLOSC_DOBITSHUFFLE) { bscount = bitshuffle(typesize, blocksize, src, tmp, dest); if (bscount < 0) return bscount; _tmp = tmp; } } /* Calculate acceleration for different compressors */ accel = get_accel(context); /* Compress for each shuffled slice split for this block. */ /* If typesize is too large, neblock is too small or we are in a leftover block, do not split at all. */ if ((typesize <= MAX_SPLITS) && (blocksize/typesize) >= MIN_BUFFERSIZE && (!leftoverblock)) { nsplits = typesize; } else { nsplits = 1; } neblock = blocksize / nsplits; for (j = 0; j < nsplits; j++) { dest += sizeof(int32_t); ntbytes += (int32_t)sizeof(int32_t); ctbytes += (int32_t)sizeof(int32_t); maxout = neblock; #if defined(HAVE_SNAPPY) if (context->compcode == BLOSC_SNAPPY) { /* TODO perhaps refactor this to keep the value stashed somewhere */ maxout = snappy_max_compressed_length(neblock); } #endif /* HAVE_SNAPPY */ if (ntbytes+maxout > maxbytes) { maxout = maxbytes - ntbytes; /* avoid buffer overrun */ if (maxout <= 0) { return 0; /* non-compressible block */ } } if (context->compcode == BLOSC_BLOSCLZ) { cbytes = blosclz_compress(context->clevel, _tmp+j*neblock, neblock, dest, maxout, accel); } #if defined(HAVE_LZ4) else if (context->compcode == BLOSC_LZ4) { cbytes = lz4_wrap_compress((char *)_tmp+j*neblock, (size_t)neblock, (char *)dest, (size_t)maxout, accel); } else if (context->compcode == BLOSC_LZ4HC) { cbytes = lz4hc_wrap_compress((char *)_tmp+j*neblock, (size_t)neblock, (char *)dest, (size_t)maxout, context->clevel); } #endif /* HAVE_LZ4 */ #if defined(HAVE_SNAPPY) else if (context->compcode == BLOSC_SNAPPY) { cbytes = snappy_wrap_compress((char *)_tmp+j*neblock, (size_t)neblock, (char *)dest, (size_t)maxout); } #endif /* HAVE_SNAPPY */ #if defined(HAVE_ZLIB) else if (context->compcode == BLOSC_ZLIB) { cbytes = zlib_wrap_compress((char *)_tmp+j*neblock, (size_t)neblock, (char *)dest, (size_t)maxout, context->clevel); } #endif /* HAVE_ZLIB */ else { blosc_compcode_to_compname(context->compcode, &compname); fprintf(stderr, "Blosc has not been compiled with '%s' ", compname); fprintf(stderr, "compression support. Please use one having it."); return -5; /* signals no compression support */ } if (cbytes > maxout) { /* Buffer overrun caused by compression (should never happen) */ return -1; } else if (cbytes < 0) { /* cbytes should never be negative */ return -2; } else if (cbytes == 0 || cbytes == neblock) { /* The compressor has been unable to compress data at all. */ /* Before doing the copy, check that we are not running into a buffer overflow. */ if ((ntbytes+neblock) > maxbytes) { return 0; /* Non-compressible data */ } memcpy(dest, _tmp+j*neblock, neblock); cbytes = neblock; } _sw32(dest - 4, cbytes); dest += cbytes; ntbytes += cbytes; ctbytes += cbytes; } /* Closes j < nsplits */ return ctbytes; } /* Decompress & unshuffle a single block */ static int blosc_d(struct blosc_context* context, int32_t blocksize, int32_t leftoverblock, const uint8_t *src, uint8_t *dest, uint8_t *tmp, uint8_t *tmp2) { int32_t j, neblock, nsplits; int32_t nbytes; /* number of decompressed bytes in split */ int32_t cbytes; /* number of compressed bytes in split */ int32_t ctbytes = 0; /* number of compressed bytes in block */ int32_t ntbytes = 0; /* number of uncompressed bytes in block */ uint8_t *_tmp = dest; int32_t typesize = context->typesize; int32_t compcode; char *compname; int bscount; if ((typesize > 1) && ((*(context->header_flags) & BLOSC_DOSHUFFLE) || \ (*(context->header_flags) & BLOSC_DOBITSHUFFLE))) { _tmp = tmp; } compcode = (*(context->header_flags) & 0xe0) >> 5; /* Compress for each shuffled slice split for this block. */ if ((typesize <= MAX_SPLITS) && (blocksize/typesize) >= MIN_BUFFERSIZE && (!leftoverblock)) { nsplits = typesize; } else { nsplits = 1; } neblock = blocksize / nsplits; for (j = 0; j < nsplits; j++) { cbytes = sw32_(src); /* amount of compressed bytes */ src += sizeof(int32_t); ctbytes += (int32_t)sizeof(int32_t); /* Uncompress */ if (cbytes == neblock) { memcpy(_tmp, src, neblock); nbytes = neblock; } else { if (compcode == BLOSC_BLOSCLZ_FORMAT) { nbytes = blosclz_decompress(src, cbytes, _tmp, neblock); } #if defined(HAVE_LZ4) else if (compcode == BLOSC_LZ4_FORMAT) { nbytes = lz4_wrap_decompress((char *)src, (size_t)cbytes, (char*)_tmp, (size_t)neblock); } #endif /* HAVE_LZ4 */ #if defined(HAVE_SNAPPY) else if (compcode == BLOSC_SNAPPY_FORMAT) { nbytes = snappy_wrap_decompress((char *)src, (size_t)cbytes, (char*)_tmp, (size_t)neblock); } #endif /* HAVE_SNAPPY */ #if defined(HAVE_ZLIB) else if (compcode == BLOSC_ZLIB_FORMAT) { nbytes = zlib_wrap_decompress((char *)src, (size_t)cbytes, (char*)_tmp, (size_t)neblock); } #endif /* HAVE_ZLIB */ else { blosc_compcode_to_compname(compcode, &compname); fprintf(stderr, "Blosc has not been compiled with decompression " "support for '%s' format. ", compname); fprintf(stderr, "Please recompile for adding this support.\n"); return -5; /* signals no decompression support */ } /* Check that decompressed bytes number is correct */ if (nbytes != neblock) { return -2; } } src += cbytes; ctbytes += cbytes; _tmp += nbytes; ntbytes += nbytes; } /* Closes j < nsplits */ if (typesize > 1) { if (*(context->header_flags) & BLOSC_DOSHUFFLE) { unshuffle(typesize, blocksize, tmp, dest); } else if (*(context->header_flags) & BLOSC_DOBITSHUFFLE) { bscount = bitunshuffle(typesize, blocksize, tmp, dest, tmp2); if (bscount < 0) return bscount; } } /* Return the number of uncompressed bytes */ return ntbytes; } /* Serial version for compression/decompression */ static int serial_blosc(struct blosc_context* context) { int32_t j, bsize, leftoverblock; int32_t cbytes; int32_t ebsize = context->blocksize + context->typesize * (int32_t)sizeof(int32_t); int32_t ntbytes = context->num_output_bytes; uint8_t *tmp = my_malloc(context->blocksize); uint8_t *tmp2 = my_malloc(ebsize); for (j = 0; j < context->nblocks; j++) { if (context->compress && !(*(context->header_flags) & BLOSC_MEMCPYED)) { _sw32(context->bstarts + j * 4, ntbytes); } bsize = context->blocksize; leftoverblock = 0; if ((j == context->nblocks - 1) && (context->leftover > 0)) { bsize = context->leftover; leftoverblock = 1; } if (context->compress) { if (*(context->header_flags) & BLOSC_MEMCPYED) { /* We want to memcpy only */ memcpy(context->dest+BLOSC_MAX_OVERHEAD+j*context->blocksize, context->src+j*context->blocksize, bsize); cbytes = bsize; } else { /* Regular compression */ cbytes = blosc_c(context, bsize, leftoverblock, ntbytes, context->destsize, context->src+j*context->blocksize, context->dest+ntbytes, tmp); if (cbytes == 0) { ntbytes = 0; /* uncompressible data */ break; } } } else { if (*(context->header_flags) & BLOSC_MEMCPYED) { /* We want to memcpy only */ memcpy(context->dest+j*context->blocksize, context->src+BLOSC_MAX_OVERHEAD+j*context->blocksize, bsize); cbytes = bsize; } else { /* Regular decompression */ cbytes = blosc_d(context, bsize, leftoverblock, context->src + sw32_(context->bstarts + j * 4), context->dest+j*context->blocksize, tmp, tmp2); } } if (cbytes < 0) { ntbytes = cbytes; /* error in blosc_c or blosc_d */ break; } ntbytes += cbytes; } // Free temporaries my_free(tmp); my_free(tmp2); return ntbytes; } /* Threaded version for compression/decompression */ static int parallel_blosc(struct blosc_context* context) { int rc; /* Check whether we need to restart threads */ blosc_set_nthreads_(context); /* Set sentinels */ context->thread_giveup_code = 1; context->thread_nblock = -1; /* Synchronization point for all threads (wait for initialization) */ WAIT_INIT(-1, context); /* Synchronization point for all threads (wait for finalization) */ WAIT_FINISH(-1, context); if (context->thread_giveup_code > 0) { /* Return the total bytes (de-)compressed in threads */ return context->num_output_bytes; } else { /* Compression/decompression gave up. Return error code. */ return context->thread_giveup_code; } } /* Do the compression or decompression of the buffer depending on the global params. */ static int do_job(struct blosc_context* context) { int32_t ntbytes; /* Run the serial version when nthreads is 1 or when the buffers are not much larger than blocksize */ if (context->numthreads == 1 || (context->sourcesize / context->blocksize) <= 1) { ntbytes = serial_blosc(context); } else { ntbytes = parallel_blosc(context); } return ntbytes; } static int32_t compute_blocksize(struct blosc_context* context, int32_t clevel, int32_t typesize, int32_t nbytes, int32_t forced_blocksize) { int32_t blocksize; /* Protection against very small buffers */ if (nbytes < (int32_t)typesize) { return 1; } blocksize = nbytes; /* Start by a whole buffer as blocksize */ if (forced_blocksize) { blocksize = forced_blocksize; /* Check that forced blocksize is not too small */ if (blocksize < MIN_BUFFERSIZE) { blocksize = MIN_BUFFERSIZE; } } else if (nbytes >= L1 * typesize) { blocksize = L1 * typesize; /* For Zlib, increase the block sizes in a factor of 8 because it is meant for compression large blocks (it shows a big overhead in compressing small ones). */ if (context->compcode == BLOSC_ZLIB) { blocksize *= 8; } /* For LZ4HC, increase the block sizes in a factor of 8 because it is meant for compression large blocks (it shows a big overhead in compressing small ones). */ if (context->compcode == BLOSC_LZ4HC) { blocksize *= 8; } if (clevel == 0) { blocksize /= 16; } else if (clevel <= 3) { blocksize /= 8; } else if (clevel <= 5) { blocksize /= 4; } else if (clevel <= 6) { blocksize /= 2; } else if (clevel < 9) { blocksize *= 1; } else { blocksize *= 2; } } /* Check that blocksize is not too large */ if (blocksize > (int32_t)nbytes) { blocksize = nbytes; } /* blocksize must be a multiple of the typesize */ if (blocksize > typesize) { blocksize = blocksize / typesize * typesize; } return blocksize; } static int initialize_context_compression(struct blosc_context* context, int clevel, int doshuffle, size_t typesize, size_t sourcesize, const void* src, void* dest, size_t destsize, int32_t compressor, int32_t blocksize, int32_t numthreads) { /* Set parameters */ context->compress = 1; context->src = (const uint8_t*)src; context->dest = (uint8_t *)(dest); context->num_output_bytes = 0; context->destsize = (int32_t)destsize; context->sourcesize = sourcesize; context->typesize = typesize; context->compcode = compressor; context->numthreads = numthreads; context->end_threads = 0; context->clevel = clevel; /* Check buffer size limits */ if (sourcesize > BLOSC_MAX_BUFFERSIZE) { /* If buffer is too large, give up. */ fprintf(stderr, "Input buffer size cannot exceed %d bytes\n", BLOSC_MAX_BUFFERSIZE); return -1; } /* Compression level */ if (clevel < 0 || clevel > 9) { /* If clevel not in 0..9, print an error */ fprintf(stderr, "`clevel` parameter must be between 0 and 9!\n"); return -10; } /* Shuffle */ if (doshuffle != 0 && doshuffle != 1 && doshuffle != 2) { fprintf(stderr, "`shuffle` parameter must be either 0, 1 or 2!\n"); return -10; } /* Check typesize limits */ if (context->typesize > BLOSC_MAX_TYPESIZE) { /* If typesize is too large, treat buffer as an 1-byte stream. */ context->typesize = 1; } /* Get the blocksize */ context->blocksize = compute_blocksize(context, clevel, (int32_t)context->typesize, context->sourcesize, blocksize); /* Compute number of blocks in buffer */ context->nblocks = context->sourcesize / context->blocksize; context->leftover = context->sourcesize % context->blocksize; context->nblocks = (context->leftover > 0) ? (context->nblocks + 1) : context->nblocks; return 1; } static int write_compression_header(struct blosc_context* context, int clevel, int doshuffle) { int32_t compcode; /* Write version header for this block */ context->dest[0] = BLOSC_VERSION_FORMAT; /* blosc format version */ /* Write compressor format */ compcode = -1; switch (context->compcode) { case BLOSC_BLOSCLZ: compcode = BLOSC_BLOSCLZ_FORMAT; context->dest[1] = BLOSC_BLOSCLZ_VERSION_FORMAT; /* blosclz format version */ break; #if defined(HAVE_LZ4) case BLOSC_LZ4: compcode = BLOSC_LZ4_FORMAT; context->dest[1] = BLOSC_LZ4_VERSION_FORMAT; /* lz4 format version */ break; case BLOSC_LZ4HC: compcode = BLOSC_LZ4HC_FORMAT; context->dest[1] = BLOSC_LZ4HC_VERSION_FORMAT; /* lz4hc is the same as lz4 */ break; #endif /* HAVE_LZ4 */ #if defined(HAVE_SNAPPY) case BLOSC_SNAPPY: compcode = BLOSC_SNAPPY_FORMAT; context->dest[1] = BLOSC_SNAPPY_VERSION_FORMAT; /* snappy format version */ break; #endif /* HAVE_SNAPPY */ #if defined(HAVE_ZLIB) case BLOSC_ZLIB: compcode = BLOSC_ZLIB_FORMAT; context->dest[1] = BLOSC_ZLIB_VERSION_FORMAT; /* zlib format version */ break; #endif /* HAVE_ZLIB */ default: { char *compname; blosc_compcode_to_compname(compcode, &compname); fprintf(stderr, "Blosc has not been compiled with '%s' ", compname); fprintf(stderr, "compression support. Please use one having it."); return -5; /* signals no compression support */ break; } } context->header_flags = context->dest+2; /* flags */ context->dest[2] = 0; /* zeroes flags */ context->dest[3] = (uint8_t)context->typesize; /* type size */ _sw32(context->dest + 4, context->sourcesize); /* size of the buffer */ _sw32(context->dest + 8, context->blocksize); /* block size */ context->bstarts = context->dest + 16; /* starts for every block */ context->num_output_bytes = 16 + sizeof(int32_t)*context->nblocks; /* space for header and pointers */ if (context->clevel == 0) { /* Compression level 0 means buffer to be memcpy'ed */ *(context->header_flags) |= BLOSC_MEMCPYED; } if (context->sourcesize < MIN_BUFFERSIZE) { /* Buffer is too small. Try memcpy'ing. */ *(context->header_flags) |= BLOSC_MEMCPYED; } if (doshuffle == BLOSC_SHUFFLE) { /* Byte-shuffle is active */ *(context->header_flags) |= BLOSC_DOSHUFFLE; /* bit 0 set to one in flags */ } if (doshuffle == BLOSC_BITSHUFFLE) { /* Bit-shuffle is active */ *(context->header_flags) |= BLOSC_DOBITSHUFFLE; /* bit 2 set to one in flags */ } *(context->header_flags) |= compcode << 5; /* compressor format start at bit 5 */ return 1; } int blosc_compress_context(struct blosc_context* context) { int32_t ntbytes = 0; if (!(*(context->header_flags) & BLOSC_MEMCPYED)) { /* Do the actual compression */ ntbytes = do_job(context); if (ntbytes < 0) { return -1; } if ((ntbytes == 0) && (context->sourcesize+BLOSC_MAX_OVERHEAD <= context->destsize)) { /* Last chance for fitting `src` buffer in `dest`. Update flags and do a memcpy later on. */ *(context->header_flags) |= BLOSC_MEMCPYED; } } if (*(context->header_flags) & BLOSC_MEMCPYED) { if (context->sourcesize + BLOSC_MAX_OVERHEAD > context->destsize) { /* We are exceeding maximum output size */ ntbytes = 0; } else if (((context->sourcesize % L1) == 0) || (context->numthreads > 1)) { /* More effective with large buffers that are multiples of the cache size or multi-cores */ context->num_output_bytes = BLOSC_MAX_OVERHEAD; ntbytes = do_job(context); if (ntbytes < 0) { return -1; } } else { memcpy(context->dest+BLOSC_MAX_OVERHEAD, context->src, context->sourcesize); ntbytes = context->sourcesize + BLOSC_MAX_OVERHEAD; } } /* Set the number of compressed bytes in header */ _sw32(context->dest + 12, ntbytes); assert(ntbytes <= context->destsize); return ntbytes; } /* The public routine for compression with context. */ int blosc_compress_ctx(int clevel, int doshuffle, size_t typesize, size_t nbytes, const void* src, void* dest, size_t destsize, const char* compressor, size_t blocksize, int numinternalthreads) { int error, result; struct blosc_context context; context.threads_started = 0; error = initialize_context_compression(&context, clevel, doshuffle, typesize, nbytes, src, dest, destsize, blosc_compname_to_compcode(compressor), blocksize, numinternalthreads); if (error < 0) { return error; } error = write_compression_header(&context, clevel, doshuffle); if (error < 0) { return error; } result = blosc_compress_context(&context); if (numinternalthreads > 1) { blosc_release_threadpool(&context); } return result; } /* The public routine for compression. See blosc.h for docstrings. */ int blosc_compress(int clevel, int doshuffle, size_t typesize, size_t nbytes, const void *src, void *dest, size_t destsize) { int error; int result; pthread_mutex_lock(&global_comp_mutex); error = initialize_context_compression(g_global_context, clevel, doshuffle, typesize, nbytes, src, dest, destsize, g_compressor, g_force_blocksize, g_threads); if (error < 0) { return error; } error = write_compression_header(g_global_context, clevel, doshuffle); if (error < 0) { return error; } result = blosc_compress_context(g_global_context); pthread_mutex_unlock(&global_comp_mutex); return result; } int blosc_run_decompression_with_context(struct blosc_context* context, const void* src, void* dest, size_t destsize, int numinternalthreads) { uint8_t version; uint8_t versionlz; uint32_t ctbytes; int32_t ntbytes; context->compress = 0; context->src = (const uint8_t*)src; context->dest = (uint8_t*)dest; context->destsize = destsize; context->num_output_bytes = 0; context->numthreads = numinternalthreads; context->end_threads = 0; /* Read the header block */ version = context->src[0]; /* blosc format version */ versionlz = context->src[1]; /* blosclz format version */ context->header_flags = (uint8_t*)(context->src + 2); /* flags */ context->typesize = (int32_t)context->src[3]; /* typesize */ context->sourcesize = sw32_(context->src + 4); /* buffer size */ context->blocksize = sw32_(context->src + 8); /* block size */ ctbytes = sw32_(context->src + 12); /* compressed buffer size */ /* Unused values */ version += 0; /* shut up compiler warning */ versionlz += 0; /* shut up compiler warning */ ctbytes += 0; /* shut up compiler warning */ context->bstarts = (uint8_t*)(context->src + 16); /* Compute some params */ /* Total blocks */ context->nblocks = context->sourcesize / context->blocksize; context->leftover = context->sourcesize % context->blocksize; context->nblocks = (context->leftover>0)? context->nblocks+1: context->nblocks; /* Check that we have enough space to decompress */ if (context->sourcesize > (int32_t)destsize) { return -1; } /* Check whether this buffer is memcpy'ed */ if (*(context->header_flags) & BLOSC_MEMCPYED) { if (((context->sourcesize % L1) == 0) || (context->numthreads > 1)) { /* More effective with large buffers that are multiples of the cache size or multi-cores */ ntbytes = do_job(context); if (ntbytes < 0) { return -1; } } else { memcpy(dest, (uint8_t *)src+BLOSC_MAX_OVERHEAD, context->sourcesize); ntbytes = context->sourcesize; } } else { /* Do the actual decompression */ ntbytes = do_job(context); if (ntbytes < 0) { return -1; } } assert(ntbytes <= (int32_t)destsize); return ntbytes; } /* The public routine for decompression with context. */ int blosc_decompress_ctx(const void *src, void *dest, size_t destsize, int numinternalthreads) { struct blosc_context context; int result = blosc_run_decompression_with_context(&context, src, dest, destsize, numinternalthreads); context.threads_started = 0; if (numinternalthreads > 1) { blosc_release_threadpool(&context); } return result; } /* The public routine for decompression. See blosc.h for docstrings. */ int blosc_decompress(const void *src, void *dest, size_t destsize) { return blosc_run_decompression_with_context(g_global_context, src, dest, destsize, g_threads); } /* Specific routine optimized for decompression a small number of items out of a compressed chunk. This does not use threads because it would affect negatively to performance. */ int blosc_getitem(const void *src, int start, int nitems, void *dest) { uint8_t *_src=NULL; /* current pos for source buffer */ uint8_t version, versionlz; /* versions for compressed header */ uint8_t flags; /* flags for header */ int32_t ntbytes = 0; /* the number of uncompressed bytes */ int32_t nblocks; /* number of total blocks in buffer */ int32_t leftover; /* extra bytes at end of buffer */ uint8_t *bstarts; /* start pointers for each block */ int tmp_init = 0; int32_t typesize, blocksize, nbytes, ctbytes; int32_t j, bsize, bsize2, leftoverblock; int32_t cbytes, startb, stopb; int stop = start + nitems; uint8_t *tmp; uint8_t *tmp2; int32_t ebsize; _src = (uint8_t *)(src); /* Read the header block */ version = _src[0]; /* blosc format version */ versionlz = _src[1]; /* blosclz format version */ flags = _src[2]; /* flags */ typesize = (int32_t)_src[3]; /* typesize */ nbytes = sw32_(_src + 4); /* buffer size */ blocksize = sw32_(_src + 8); /* block size */ ctbytes = sw32_(_src + 12); /* compressed buffer size */ ebsize = blocksize + typesize * (int32_t)sizeof(int32_t); tmp = my_malloc(blocksize); /* tmp for thread 0 */ tmp2 = my_malloc(ebsize); /* tmp2 for thread 0 */ version += 0; /* shut up compiler warning */ versionlz += 0; /* shut up compiler warning */ ctbytes += 0; /* shut up compiler warning */ _src += 16; bstarts = _src; /* Compute some params */ /* Total blocks */ nblocks = nbytes / blocksize; leftover = nbytes % blocksize; nblocks = (leftover>0)? nblocks+1: nblocks; _src += sizeof(int32_t)*nblocks; /* Check region boundaries */ if ((start < 0) || (start*typesize > nbytes)) { fprintf(stderr, "`start` out of bounds"); return -1; } if ((stop < 0) || (stop*typesize > nbytes)) { fprintf(stderr, "`start`+`nitems` out of bounds"); return -1; } for (j = 0; j < nblocks; j++) { bsize = blocksize; leftoverblock = 0; if ((j == nblocks - 1) && (leftover > 0)) { bsize = leftover; leftoverblock = 1; } /* Compute start & stop for each block */ startb = start * typesize - j * blocksize; stopb = stop * typesize - j * blocksize; if ((startb >= (int)blocksize) || (stopb <= 0)) { continue; } if (startb < 0) { startb = 0; } if (stopb > (int)blocksize) { stopb = blocksize; } bsize2 = stopb - startb; /* Do the actual data copy */ if (flags & BLOSC_MEMCPYED) { /* We want to memcpy only */ memcpy((uint8_t *)dest + ntbytes, (uint8_t *)src + BLOSC_MAX_OVERHEAD + j*blocksize + startb, bsize2); cbytes = bsize2; } else { struct blosc_context context; /* blosc_d only uses typesize and flags */ context.typesize = typesize; context.header_flags = &flags; /* Regular decompression. Put results in tmp2. */ cbytes = blosc_d(&context, bsize, leftoverblock, (uint8_t *)src + sw32_(bstarts + j * 4), tmp2, tmp, tmp2); if (cbytes < 0) { ntbytes = cbytes; break; } /* Copy to destination */ memcpy((uint8_t *)dest + ntbytes, tmp2 + startb, bsize2); cbytes = bsize2; } ntbytes += cbytes; } my_free(tmp); my_free(tmp2); return ntbytes; } /* Decompress & unshuffle several blocks in a single thread */ static void *t_blosc(void *ctxt) { struct thread_context* context = (struct thread_context*)ctxt; int32_t cbytes, ntdest; int32_t tblocks; /* number of blocks per thread */ int32_t leftover2; int32_t tblock; /* limit block on a thread */ int32_t nblock_; /* private copy of nblock */ int32_t bsize, leftoverblock; /* Parameters for threads */ int32_t blocksize; int32_t ebsize; int32_t compress; int32_t maxbytes; int32_t ntbytes; int32_t flags; int32_t nblocks; int32_t leftover; uint8_t *bstarts; const uint8_t *src; uint8_t *dest; uint8_t *tmp; uint8_t *tmp2; int rc; while(1) { /* Synchronization point for all threads (wait for initialization) */ WAIT_INIT(NULL, context->parent_context); if(context->parent_context->end_threads) { break; } /* Get parameters for this thread before entering the main loop */ blocksize = context->parent_context->blocksize; ebsize = blocksize + context->parent_context->typesize * (int32_t)sizeof(int32_t); compress = context->parent_context->compress; flags = *(context->parent_context->header_flags); maxbytes = context->parent_context->destsize; nblocks = context->parent_context->nblocks; leftover = context->parent_context->leftover; bstarts = context->parent_context->bstarts; src = context->parent_context->src; dest = context->parent_context->dest; if (blocksize > context->tmpblocksize) { my_free(context->tmp); my_free(context->tmp2); context->tmp = my_malloc(blocksize); context->tmp2 = my_malloc(ebsize); } tmp = context->tmp; tmp2 = context->tmp2; ntbytes = 0; /* only useful for decompression */ if (compress && !(flags & BLOSC_MEMCPYED)) { /* Compression always has to follow the block order */ pthread_mutex_lock(&context->parent_context->count_mutex); context->parent_context->thread_nblock++; nblock_ = context->parent_context->thread_nblock; pthread_mutex_unlock(&context->parent_context->count_mutex); tblock = nblocks; } else { /* Decompression can happen using any order. We choose sequential block order on each thread */ /* Blocks per thread */ tblocks = nblocks / context->parent_context->numthreads; leftover2 = nblocks % context->parent_context->numthreads; tblocks = (leftover2>0)? tblocks+1: tblocks; nblock_ = context->tid*tblocks; tblock = nblock_ + tblocks; if (tblock > nblocks) { tblock = nblocks; } } /* Loop over blocks */ leftoverblock = 0; while ((nblock_ < tblock) && context->parent_context->thread_giveup_code > 0) { bsize = blocksize; if (nblock_ == (nblocks - 1) && (leftover > 0)) { bsize = leftover; leftoverblock = 1; } if (compress) { if (flags & BLOSC_MEMCPYED) { /* We want to memcpy only */ memcpy(dest+BLOSC_MAX_OVERHEAD+nblock_*blocksize, src+nblock_*blocksize, bsize); cbytes = bsize; } else { /* Regular compression */ cbytes = blosc_c(context->parent_context, bsize, leftoverblock, 0, ebsize, src+nblock_*blocksize, tmp2, tmp); } } else { if (flags & BLOSC_MEMCPYED) { /* We want to memcpy only */ memcpy(dest+nblock_*blocksize, src+BLOSC_MAX_OVERHEAD+nblock_*blocksize, bsize); cbytes = bsize; } else { cbytes = blosc_d(context->parent_context, bsize, leftoverblock, src + sw32_(bstarts + nblock_ * 4), dest+nblock_*blocksize, tmp, tmp2); } } /* Check whether current thread has to giveup */ if (context->parent_context->thread_giveup_code <= 0) { break; } /* Check results for the compressed/decompressed block */ if (cbytes < 0) { /* compr/decompr failure */ /* Set giveup_code error */ pthread_mutex_lock(&context->parent_context->count_mutex); context->parent_context->thread_giveup_code = cbytes; pthread_mutex_unlock(&context->parent_context->count_mutex); break; } if (compress && !(flags & BLOSC_MEMCPYED)) { /* Start critical section */ pthread_mutex_lock(&context->parent_context->count_mutex); ntdest = context->parent_context->num_output_bytes; _sw32(bstarts + nblock_ * 4, ntdest); /* update block start counter */ if ( (cbytes == 0) || (ntdest+cbytes > maxbytes) ) { context->parent_context->thread_giveup_code = 0; /* uncompressible buffer */ pthread_mutex_unlock(&context->parent_context->count_mutex); break; } context->parent_context->thread_nblock++; nblock_ = context->parent_context->thread_nblock; context->parent_context->num_output_bytes += cbytes; /* update return bytes counter */ pthread_mutex_unlock(&context->parent_context->count_mutex); /* End of critical section */ /* Copy the compressed buffer to destination */ memcpy(dest+ntdest, tmp2, cbytes); } else { nblock_++; /* Update counter for this thread */ ntbytes += cbytes; } } /* closes while (nblock_) */ /* Sum up all the bytes decompressed */ if ((!compress || (flags & BLOSC_MEMCPYED)) && context->parent_context->thread_giveup_code > 0) { /* Update global counter for all threads (decompression only) */ pthread_mutex_lock(&context->parent_context->count_mutex); context->parent_context->num_output_bytes += ntbytes; pthread_mutex_unlock(&context->parent_context->count_mutex); } /* Meeting point for all threads (wait for finalization) */ WAIT_FINISH(NULL, context->parent_context); } /* Cleanup our working space and context */ my_free(context->tmp); my_free(context->tmp2); my_free(context); return(NULL); } static int init_threads(struct blosc_context* context) { int32_t tid; int rc2; int32_t ebsize; struct thread_context* thread_context; /* Initialize mutex and condition variable objects */ pthread_mutex_init(&context->count_mutex, NULL); /* Set context thread sentinels */ context->thread_giveup_code = 1; context->thread_nblock = -1; /* Barrier initialization */ #ifdef _POSIX_BARRIERS_MINE pthread_barrier_init(&context->barr_init, NULL, context->numthreads+1); pthread_barrier_init(&context->barr_finish, NULL, context->numthreads+1); #else pthread_mutex_init(&context->count_threads_mutex, NULL); pthread_cond_init(&context->count_threads_cv, NULL); context->count_threads = 0; /* Reset threads counter */ #endif #if !defined(_WIN32) /* Initialize and set thread detached attribute */ pthread_attr_init(&context->ct_attr); pthread_attr_setdetachstate(&context->ct_attr, PTHREAD_CREATE_JOINABLE); #endif /* Finally, create the threads in detached state */ for (tid = 0; tid < context->numthreads; tid++) { context->tids[tid] = tid; /* Create a thread context thread owns context (will destroy when finished) */ thread_context = (struct thread_context*)my_malloc(sizeof(struct thread_context)); thread_context->parent_context = context; thread_context->tid = tid; ebsize = context->blocksize + context->typesize * (int32_t)sizeof(int32_t); thread_context->tmp = my_malloc(context->blocksize); thread_context->tmp2 = my_malloc(ebsize); thread_context->tmpblocksize = context->blocksize; #if !defined(_WIN32) rc2 = pthread_create(&context->threads[tid], &context->ct_attr, t_blosc, (void *)thread_context); #else rc2 = pthread_create(&context->threads[tid], NULL, t_blosc, (void *)thread_context); #endif if (rc2) { fprintf(stderr, "ERROR; return code from pthread_create() is %d\n", rc2); fprintf(stderr, "\tError detail: %s\n", strerror(rc2)); return(-1); } } return(0); } int blosc_set_nthreads(int nthreads_new) { int ret = g_threads; /* Re-initialize Blosc */ blosc_destroy(); blosc_init(); g_threads = nthreads_new; return ret; } int blosc_set_nthreads_(struct blosc_context* context) { if (context->numthreads > BLOSC_MAX_THREADS) { fprintf(stderr, "Error. nthreads cannot be larger than BLOSC_MAX_THREADS (%d)", BLOSC_MAX_THREADS); return -1; } else if (context->numthreads <= 0) { fprintf(stderr, "Error. nthreads must be a positive integer"); return -1; } /* Launch a new pool of threads */ if (context->numthreads > 1 && context->numthreads != context->threads_started) { blosc_release_threadpool(context); init_threads(context); } /* We have now started the threads */ context->threads_started = context->numthreads; return context->numthreads; } int blosc_set_compressor(const char *compname) { int code = blosc_compname_to_compcode(compname); g_compressor = code; /* Check if should initialize (implementing previous 1.2.3 behaviour, where calling blosc_set_nthreads was enough) */ if (!g_initlib) blosc_init(); return code; } char* blosc_list_compressors(void) { static int compressors_list_done = 0; static char ret[256]; if (compressors_list_done) return ret; ret[0] = '\0'; strcat(ret, BLOSC_BLOSCLZ_COMPNAME); #if defined(HAVE_LZ4) strcat(ret, ","); strcat(ret, BLOSC_LZ4_COMPNAME); strcat(ret, ","); strcat(ret, BLOSC_LZ4HC_COMPNAME); #endif /* HAVE_LZ4 */ #if defined(HAVE_SNAPPY) strcat(ret, ","); strcat(ret, BLOSC_SNAPPY_COMPNAME); #endif /* HAVE_SNAPPY */ #if defined(HAVE_ZLIB) strcat(ret, ","); strcat(ret, BLOSC_ZLIB_COMPNAME); #endif /* HAVE_ZLIB */ compressors_list_done = 1; return ret; } char* blosc_get_version_string(void) { static char ret[256]; strcpy(ret, BLOSC_VERSION_STRING); return ret; } int blosc_get_complib_info(char *compname, char **complib, char **version) { int clibcode; char *clibname; char *clibversion = "unknown"; #if (defined(HAVE_LZ4) && defined(LZ4_VERSION_MAJOR)) || (defined(HAVE_SNAPPY) && defined(SNAPPY_VERSION)) char sbuffer[256]; #endif clibcode = compname_to_clibcode(compname); clibname = clibcode_to_clibname(clibcode); /* complib version */ if (clibcode == BLOSC_BLOSCLZ_LIB) { clibversion = BLOSCLZ_VERSION_STRING; } #if defined(HAVE_LZ4) else if (clibcode == BLOSC_LZ4_LIB) { #if defined(LZ4_VERSION_MAJOR) sprintf(sbuffer, "%d.%d.%d", LZ4_VERSION_MAJOR, LZ4_VERSION_MINOR, LZ4_VERSION_RELEASE); clibversion = sbuffer; #endif /* LZ4_VERSION_MAJOR */ } #endif /* HAVE_LZ4 */ #if defined(HAVE_SNAPPY) else if (clibcode == BLOSC_SNAPPY_LIB) { #if defined(SNAPPY_VERSION) sprintf(sbuffer, "%d.%d.%d", SNAPPY_MAJOR, SNAPPY_MINOR, SNAPPY_PATCHLEVEL); clibversion = sbuffer; #endif /* SNAPPY_VERSION */ } #endif /* HAVE_SNAPPY */ #if defined(HAVE_ZLIB) else if (clibcode == BLOSC_ZLIB_LIB) { clibversion = ZLIB_VERSION; } #endif /* HAVE_ZLIB */ *complib = strdup(clibname); *version = strdup(clibversion); return clibcode; } /* Return `nbytes`, `cbytes` and `blocksize` from a compressed buffer. */ void blosc_cbuffer_sizes(const void *cbuffer, size_t *nbytes, size_t *cbytes, size_t *blocksize) { uint8_t *_src = (uint8_t *)(cbuffer); /* current pos for source buffer */ uint8_t version, versionlz; /* versions for compressed header */ /* Read the version info (could be useful in the future) */ version = _src[0]; /* blosc format version */ versionlz = _src[1]; /* blosclz format version */ version += 0; /* shut up compiler warning */ versionlz += 0; /* shut up compiler warning */ /* Read the interesting values */ *nbytes = (size_t)sw32_(_src + 4); /* uncompressed buffer size */ *blocksize = (size_t)sw32_(_src + 8); /* block size */ *cbytes = (size_t)sw32_(_src + 12); /* compressed buffer size */ } /* Return `typesize` and `flags` from a compressed buffer. */ void blosc_cbuffer_metainfo(const void *cbuffer, size_t *typesize, int *flags) { uint8_t *_src = (uint8_t *)(cbuffer); /* current pos for source buffer */ uint8_t version, versionlz; /* versions for compressed header */ /* Read the version info (could be useful in the future) */ version = _src[0]; /* blosc format version */ versionlz = _src[1]; /* blosclz format version */ version += 0; /* shut up compiler warning */ versionlz += 0; /* shut up compiler warning */ /* Read the interesting values */ *flags = (int)_src[2]; /* flags */ *typesize = (size_t)_src[3]; /* typesize */ } /* Return version information from a compressed buffer. */ void blosc_cbuffer_versions(const void *cbuffer, int *version, int *versionlz) { uint8_t *_src = (uint8_t *)(cbuffer); /* current pos for source buffer */ /* Read the version info */ *version = (int)_src[0]; /* blosc format version */ *versionlz = (int)_src[1]; /* Lempel-Ziv compressor format version */ } /* Return the compressor library/format used in a compressed buffer. */ char *blosc_cbuffer_complib(const void *cbuffer) { uint8_t *_src = (uint8_t *)(cbuffer); /* current pos for source buffer */ int clibcode; char *complib; /* Read the compressor format/library info */ clibcode = (_src[2] & 0xe0) >> 5; complib = clibcode_to_clibname(clibcode); return complib; } /* Force the use of a specific blocksize. If 0, an automatic blocksize will be used (the default). */ void blosc_set_blocksize(size_t size) { g_force_blocksize = (int32_t)size; } void blosc_init(void) { pthread_mutex_init(&global_comp_mutex, NULL); g_global_context = (struct blosc_context*)my_malloc(sizeof(struct blosc_context)); g_global_context->threads_started = 0; g_initlib = 1; } void blosc_destroy(void) { g_initlib = 0; blosc_release_threadpool(g_global_context); my_free(g_global_context); pthread_mutex_destroy(&global_comp_mutex); } int blosc_release_threadpool(struct blosc_context* context) { int32_t t; void* status; int rc; int rc2; if (context->threads_started > 0) { /* Tell all existing threads to finish */ context->end_threads = 1; /* Sync threads */ WAIT_INIT(-1, context); /* Join exiting threads */ for (t=0; tthreads_started; t++) { rc2 = pthread_join(context->threads[t], &status); if (rc2) { fprintf(stderr, "ERROR; return code from pthread_join() is %d\n", rc2); fprintf(stderr, "\tError detail: %s\n", strerror(rc2)); } } /* Release mutex and condition variable objects */ pthread_mutex_destroy(&context->count_mutex); /* Barriers */ #ifdef _POSIX_BARRIERS_MINE pthread_barrier_destroy(&context->barr_init); pthread_barrier_destroy(&context->barr_finish); #else pthread_mutex_destroy(&context->count_threads_mutex); pthread_cond_destroy(&context->count_threads_cv); #endif /* Thread attributes */ #if !defined(_WIN32) pthread_attr_destroy(&context->ct_attr); #endif } context->threads_started = 0; return 0; } int blosc_free_resources(void) { return blosc_release_threadpool(g_global_context); } c-blosc-1.7.0/blosc/blosc.h000066400000000000000000000330721254622165100154270ustar00rootroot00000000000000/********************************************************************* Blosc - Blocked Shuffling and Compression Library Author: Francesc Alted See LICENSES/BLOSC.txt for details about copyright and rights to use. **********************************************************************/ #ifndef BLOSC_H #define BLOSC_H #include #include #include "blosc-export.h" #ifdef __cplusplus extern "C" { #endif /* Version numbers */ #define BLOSC_VERSION_MAJOR 1 /* for major interface/format changes */ #define BLOSC_VERSION_MINOR 7 /* for minor interface/format changes */ #define BLOSC_VERSION_RELEASE 0 /* for tweaks, bug-fixes, or development */ #define BLOSC_VERSION_STRING "1.7.0" /* string version. Sync with above! */ #define BLOSC_VERSION_REVISION "$Rev$" /* revision version */ #define BLOSC_VERSION_DATE "$Date:: 2015-07-05 #$" /* date version */ #define BLOSCLZ_VERSION_STRING "1.0.5" /* the internal compressor version */ /* The *_FORMAT symbols should be just 1-byte long */ #define BLOSC_VERSION_FORMAT 2 /* Blosc format version, starting at 1 */ /* Minimum header length */ #define BLOSC_MIN_HEADER_LENGTH 16 /* The maximum overhead during compression in bytes. This equals to BLOSC_MIN_HEADER_LENGTH now, but can be higher in future implementations */ #define BLOSC_MAX_OVERHEAD BLOSC_MIN_HEADER_LENGTH /* Maximum source buffer size to be compressed */ #define BLOSC_MAX_BUFFERSIZE (INT_MAX - BLOSC_MAX_OVERHEAD) /* Maximum typesize before considering source buffer as a stream of bytes */ #define BLOSC_MAX_TYPESIZE 255 /* Cannot be larger than 255 */ /* The maximum number of threads (for some static arrays) */ #define BLOSC_MAX_THREADS 256 /* Codes for shuffling (see blosc_compress) */ #define BLOSC_NOSHUFFLE 0 /* no shuffle */ #define BLOSC_SHUFFLE 1 /* byte-wise shuffle */ #define BLOSC_BITSHUFFLE 2 /* bit-wise shuffle */ /* Codes for internal flags (see blosc_cbuffer_metainfo) */ #define BLOSC_DOSHUFFLE 0x1 /* byte-wise shuffle */ #define BLOSC_MEMCPYED 0x2 /* plain copy */ #define BLOSC_DOBITSHUFFLE 0x4 /* bit-wise shuffle */ /* Codes for the different compressors shipped with Blosc */ #define BLOSC_BLOSCLZ 0 #define BLOSC_LZ4 1 #define BLOSC_LZ4HC 2 #define BLOSC_SNAPPY 3 #define BLOSC_ZLIB 4 /* Names for the different compressors shipped with Blosc */ #define BLOSC_BLOSCLZ_COMPNAME "blosclz" #define BLOSC_LZ4_COMPNAME "lz4" #define BLOSC_LZ4HC_COMPNAME "lz4hc" #define BLOSC_SNAPPY_COMPNAME "snappy" #define BLOSC_ZLIB_COMPNAME "zlib" /* Codes for the different compression libraries shipped with Blosc */ #define BLOSC_BLOSCLZ_LIB 0 #define BLOSC_LZ4_LIB 1 #define BLOSC_SNAPPY_LIB 2 #define BLOSC_ZLIB_LIB 3 /* Names for the different compression libraries shipped with Blosc */ #define BLOSC_BLOSCLZ_LIBNAME "BloscLZ" #define BLOSC_LZ4_LIBNAME "LZ4" #define BLOSC_SNAPPY_LIBNAME "Snappy" #define BLOSC_ZLIB_LIBNAME "Zlib" /* The codes for compressor formats shipped with Blosc (code must be < 8) */ #define BLOSC_BLOSCLZ_FORMAT BLOSC_BLOSCLZ_LIB #define BLOSC_LZ4_FORMAT BLOSC_LZ4_LIB /* LZ4HC and LZ4 share the same format */ #define BLOSC_LZ4HC_FORMAT BLOSC_LZ4_LIB #define BLOSC_SNAPPY_FORMAT BLOSC_SNAPPY_LIB #define BLOSC_ZLIB_FORMAT BLOSC_ZLIB_LIB /* The version formats for compressors shipped with Blosc */ /* All versions here starts at 1 */ #define BLOSC_BLOSCLZ_VERSION_FORMAT 1 #define BLOSC_LZ4_VERSION_FORMAT 1 #define BLOSC_LZ4HC_VERSION_FORMAT 1 /* LZ4HC and LZ4 share the same format */ #define BLOSC_SNAPPY_VERSION_FORMAT 1 #define BLOSC_ZLIB_VERSION_FORMAT 1 /** Initialize the Blosc library environment. You must call this previous to any other Blosc call, unless you want Blosc to be used simultaneously in a multi-threaded environment, in which case you should *exclusively* use the blosc_compress_ctx()/blosc_decompress_ctx() pair (see below). */ BLOSC_EXPORT void blosc_init(void); /** Destroy the Blosc library environment. You must call this after to you are done with all the Blosc calls, unless you have not used blosc_init() before (see blosc_init() above). */ BLOSC_EXPORT void blosc_destroy(void); /** Compress a block of data in the `src` buffer and returns the size of compressed block. The size of `src` buffer is specified by `nbytes`. There is not a minimum for `src` buffer size (`nbytes`). `clevel` is the desired compression level and must be a number between 0 (no compression) and 9 (maximum compression). `doshuffle` specifies whether the shuffle compression preconditioner should be applied or not. BLOSC_NOSHUFFLE means not applying it, BLOSC_SHUFFLE means applying it at a byte level and BLOSC_BITSHUFFLE at a bit level (slower but may achieve better entropy alignment). `typesize` is the number of bytes for the atomic type in binary `src` buffer. This is mainly useful for the shuffle preconditioner. For implementation reasons, only a 1 < typesize < 256 will allow the shuffle filter to work. When typesize is not in this range, shuffle will be silently disabled. The `dest` buffer must have at least the size of `destsize`. Blosc guarantees that if you set `destsize` to, at least, (`nbytes`+BLOSC_MAX_OVERHEAD), the compression will always succeed. The `src` buffer and the `dest` buffer can not overlap. Compression is memory safe and guaranteed not to write the `dest` buffer more than what is specified in `destsize`. If `src` buffer cannot be compressed into `destsize`, the return value is zero and you should discard the contents of the `dest` buffer. A negative return value means that an internal error happened. This should never happen. If you see this, please report it back together with the buffer data causing this and compression settings. */ BLOSC_EXPORT int blosc_compress(int clevel, int doshuffle, size_t typesize, size_t nbytes, const void *src, void *dest, size_t destsize); /** Context interface to blosc compression. This does not require a call to blosc_init() and can be called from multithreaded applications without the global lock being used, so allowing Blosc be executed simultaneously in those scenarios. It uses the same parameters than the blosc_compress() function plus: `compressor`: the string representing the type of compressor to use. `blocksize`: the requested size of the compressed blocks. If 0, an automatic blocksize will be used. `numinternalthreads`: the number of threads to use internally. A negative return value means that an internal error happened. This should never happen. If you see this, please report it back together with the buffer data causing this and compression settings. */ BLOSC_EXPORT int blosc_compress_ctx(int clevel, int doshuffle, size_t typesize, size_t nbytes, const void* src, void* dest, size_t destsize, const char* compressor, size_t blocksize, int numinternalthreads); /** Decompress a block of compressed data in `src`, put the result in `dest` and returns the size of the decompressed block. The `src` buffer and the `dest` buffer can not overlap. Decompression is memory safe and guaranteed not to write the `dest` buffer more than what is specified in `destsize`. If an error occurs, e.g. the compressed data is corrupted or the output buffer is not large enough, then 0 (zero) or a negative value will be returned instead. */ BLOSC_EXPORT int blosc_decompress(const void *src, void *dest, size_t destsize); /** Context interface to blosc decompression. This does not require a call to blosc_init() and can be called from multithreaded applications without the global lock being used, so allowing Blosc be executed simultaneously in those scenarios. It uses the same parameters than the blosc_decompress() function plus: `numinternalthreads`: number of threads to use internally. Decompression is memory safe and guaranteed not to write the `dest` buffer more than what is specified in `destsize`. If an error occurs, e.g. the compressed data is corrupted or the output buffer is not large enough, then 0 (zero) or a negative value will be returned instead. */ BLOSC_EXPORT int blosc_decompress_ctx(const void *src, void *dest, size_t destsize, int numinternalthreads); /** Get `nitems` (of typesize size) in `src` buffer starting in `start`. The items are returned in `dest` buffer, which has to have enough space for storing all items. Returns the number of bytes copied to `dest` or a negative value if some error happens. */ BLOSC_EXPORT int blosc_getitem(const void *src, int start, int nitems, void *dest); /** Initialize a pool of threads for compression/decompression. If `nthreads` is 1, then the serial version is chosen and a possible previous existing pool is ended. If this is not called, `nthreads` is set to 1 internally. Returns the previous number of threads. */ BLOSC_EXPORT int blosc_set_nthreads(int nthreads); /** Select the compressor to be used. The supported ones are "blosclz", "lz4", "lz4hc", "snappy" and "zlib". If this function is not called, then "blosclz" will be used. In case the compressor is not recognized, or there is not support for it in this build, it returns a -1. Else it returns the code for the compressor (>=0). */ BLOSC_EXPORT int blosc_set_compressor(const char* compname); /** Get the `compname` associated with the `compcode`. If the compressor code is not recognized, or there is not support for it in this build, -1 is returned. Else, the compressor code is returned. */ BLOSC_EXPORT int blosc_compcode_to_compname(int compcode, char **compname); /** Return the compressor code associated with the compressor name. If the compressor name is not recognized, or there is not support for it in this build, -1 is returned instead. */ BLOSC_EXPORT int blosc_compname_to_compcode(const char *compname); /** Get a list of compressors supported in the current build. The returned value is a string with a concatenation of "blosclz", "lz4", "lz4hc", "snappy" or "zlib" separated by commas, depending on which ones are present in the build. This function does not leak, so you should not free() the returned list. This function should always succeed. */ BLOSC_EXPORT char* blosc_list_compressors(void); /** Return the version of blosc in string format. Useful for dynamic libraries. */ BLOSC_EXPORT char* blosc_get_version_string(void); /** Get info from compression libraries included in the current build. In `compname` you pass the compressor name that you want info from. In `complib` and `version` you get the compression library name and version (if available) as output. In `complib` and `version` you get a pointer to the compressor library name and the version in string format respectively. After using the name and version, you should free() them so as to avoid leaks. If the compressor is supported, it returns the code for the library (>=0). If it is not supported, this function returns -1. */ BLOSC_EXPORT int blosc_get_complib_info(char *compname, char **complib, char **version); /** Free possible memory temporaries and thread resources. Use this when you are not going to use Blosc for a long while. In case of problems releasing the resources, it returns a negative number, else it returns 0. */ BLOSC_EXPORT int blosc_free_resources(void); /** Return information about a compressed buffer, namely the number of uncompressed bytes (`nbytes`) and compressed (`cbytes`). It also returns the `blocksize` (which is used internally for doing the compression by blocks). You only need to pass the first BLOSC_MIN_HEADER_LENGTH bytes of a compressed buffer for this call to work. This function should always succeed. */ BLOSC_EXPORT void blosc_cbuffer_sizes(const void *cbuffer, size_t *nbytes, size_t *cbytes, size_t *blocksize); /** Return information about a compressed buffer, namely the type size (`typesize`), as well as some internal `flags`. The `flags` is a set of bits, where the currently used ones are: * bit 0: whether the shuffle filter has been applied or not * bit 1: whether the internal buffer is a pure memcpy or not You can use the `BLOSC_DOSHUFFLE`, `BLOSC_DOBITSHUFFLE` and `BLOSC_MEMCPYED` symbols for extracting the interesting bits (e.g. ``flags & BLOSC_DOSHUFFLE`` says whether the buffer is byte-shuffled or not). This function should always succeed. */ BLOSC_EXPORT void blosc_cbuffer_metainfo(const void *cbuffer, size_t *typesize, int *flags); /** Return information about a compressed buffer, namely the internal Blosc format version (`version`) and the format for the internal Lempel-Ziv compressor used (`versionlz`). This function should always succeed. */ BLOSC_EXPORT void blosc_cbuffer_versions(const void *cbuffer, int *version, int *versionlz); /** Return the compressor library/format used in a compressed buffer. This function should always succeed. */ BLOSC_EXPORT char *blosc_cbuffer_complib(const void *cbuffer); /********************************************************************* Low-level functions follows. Use them only if you are an expert! *********************************************************************/ /** Force the use of a specific blocksize. If 0, an automatic blocksize will be used (the default). */ BLOSC_EXPORT void blosc_set_blocksize(size_t blocksize); #ifdef __cplusplus } #endif #endif c-blosc-1.7.0/blosc/blosclz.c000066400000000000000000000350731254622165100157730ustar00rootroot00000000000000/********************************************************************* Blosc - Blocked Shuffling and Compression Library Author: Francesc Alted Creation date: 2009-05-20 See LICENSES/BLOSC.txt for details about copyright and rights to use. **********************************************************************/ /********************************************************************* The code in this file is heavily based on FastLZ, a lightning-fast lossless compression library. See LICENSES/FASTLZ.txt for details. **********************************************************************/ #include #include #include #include "blosclz.h" #if defined(_WIN32) && !defined(__MINGW32__) #include /* stdint.h only available in VS2010 (VC++ 16.0) and newer */ #if defined(_MSC_VER) && _MSC_VER < 1600 #include "win32/stdint-windows.h" #else #include #endif /* llabs only available in VS2013 (VC++ 18.0) and newer */ #if defined(_MSC_VER) && _MSC_VER < 1800 #define llabs(v) abs(v) #endif #else #include #endif /* _WIN32 */ /* * Prevent accessing more than 8-bit at once, except on x86 architectures. */ #if !defined(BLOSCLZ_STRICT_ALIGN) #define BLOSCLZ_STRICT_ALIGN #if defined(__i386__) || defined(__386) || defined (__amd64) /* GNU C, Sun Studio */ #undef BLOSCLZ_STRICT_ALIGN #elif defined(__i486__) || defined(__i586__) || defined(__i686__) /* GNU C */ #undef BLOSCLZ_STRICT_ALIGN #elif defined(_M_IX86) || defined(_M_X64) /* Intel, MSVC */ #undef BLOSCLZ_STRICT_ALIGN #elif defined(__386) #undef BLOSCLZ_STRICT_ALIGN #elif defined(_X86_) /* MinGW */ #undef BLOSCLZ_STRICT_ALIGN #elif defined(__I86__) /* Digital Mars */ #undef BLOSCLZ_STRICT_ALIGN /* Seems like unaligned access in ARM (at least ARMv6) is pretty expensive, so we are going to always enforce strict aligment in ARM. If anybody suggest that newer ARMs are better, we can revisit this. */ /* #elif defined(__ARM_FEATURE_UNALIGNED) */ /* ARM, GNU C */ /* #undef BLOSCLZ_STRICT_ALIGN */ #endif #endif /* * Always check for bound when decompressing. * Generally it is best to leave it defined. */ #define BLOSCLZ_SAFE /* * Give hints to the compiler for branch prediction optimization. */ #if defined(__GNUC__) && (__GNUC__ > 2) #define BLOSCLZ_EXPECT_CONDITIONAL(c) (__builtin_expect((c), 1)) #define BLOSCLZ_UNEXPECT_CONDITIONAL(c) (__builtin_expect((c), 0)) #else #define BLOSCLZ_EXPECT_CONDITIONAL(c) (c) #define BLOSCLZ_UNEXPECT_CONDITIONAL(c) (c) #endif /* * Use inlined functions for supported systems. */ #if defined(_MSC_VER) && !defined(__cplusplus) /* Visual Studio */ #define inline __inline /* Visual C is not C99, but supports some kind of inline */ #endif #define MAX_COPY 32 #define MAX_DISTANCE 8191 #define MAX_FARDISTANCE (65535+MAX_DISTANCE-1) #ifdef BLOSCLZ_STRICT_ALIGN #define BLOSCLZ_READU16(p) ((p)[0] | (p)[1]<<8) #else #define BLOSCLZ_READU16(p) *((const uint16_t*)(p)) #endif /* * Fast copy macros */ #if defined(_WIN32) #define CPYSIZE 32 #else #define CPYSIZE 8 #endif #define MCPY(d,s) { memcpy(d, s, CPYSIZE); d+=CPYSIZE; s+=CPYSIZE; } #define FASTCOPY(d,s,e) { do { MCPY(d,s) } while (d 32) || (llabs(op-ref) < CPYSIZE)) { \ for(; len; --len) \ *op++ = *ref++; \ } \ else BLOCK_COPY(op, ref, len, op_limit); /* Simple, but pretty effective hash function for 3-byte sequence */ #define HASH_FUNCTION(v, p, l) { \ v = BLOSCLZ_READU16(p); \ v ^= BLOSCLZ_READU16(p + 1) ^ ( v >> (16 - l)); \ v &= (1 << l) - 1; \ } /* Another version which seems to be a bit more effective than the above, * but a bit slower. Could be interesting for high opt_level. */ #define MINMATCH 3 #define HASH_FUNCTION2(v, p, l) { \ v = BLOSCLZ_READU16(p); \ v = (v * 2654435761U) >> ((MINMATCH * 8) - (l + 1)); \ v &= (1 << l) - 1; \ } #define LITERAL(ip, op, op_limit, anchor, copy) { \ if (BLOSCLZ_UNEXPECT_CONDITIONAL(op+2 > op_limit)) \ goto out; \ *op++ = *anchor++; \ ip = anchor; \ copy++; \ if(BLOSCLZ_UNEXPECT_CONDITIONAL(copy == MAX_COPY)) { \ copy = 0; \ *op++ = MAX_COPY-1; \ } \ continue; \ } #define IP_BOUNDARY 2 int blosclz_compress(const int opt_level, const void* input, int length, void* output, int maxout, int accel) { uint8_t* ip = (uint8_t*) input; uint8_t* ibase = (uint8_t*) input; uint8_t* ip_bound = ip + length - IP_BOUNDARY; uint8_t* ip_limit = ip + length - 12; uint8_t* op = (uint8_t*) output; /* Hash table depends on the opt level. Hash_log cannot be larger than 15. */ /* The parametrization below is made from playing with the bench suite, like: $ bench/bench blosclz single 4 $ bench/bench blosclz single 4 4194280 12 25 and taking the minimum times on a i5-3380M @ 2.90GHz. Curiously enough, values >= 14 does not always get maximum compression, even with large blocksizes. */ int8_t hash_log_[10] = {-1, 11, 11, 11, 12, 13, 13, 13, 13, 13}; uint8_t hash_log = hash_log_[opt_level]; uint16_t hash_size = 1 << hash_log; uint16_t *htab; uint8_t* op_limit; int32_t hval; uint8_t copy; double maxlength_[10] = {-1, .1, .15, .2, .3, .45, .6, .75, .9, 1.0}; int32_t maxlength = (int32_t) (length * maxlength_[opt_level]); if (maxlength > (int32_t) maxout) { maxlength = (int32_t) maxout; } op_limit = op + maxlength; /* output buffer cannot be less than 66 bytes or we can get into trouble */ if (BLOSCLZ_UNEXPECT_CONDITIONAL(maxlength < 66 || length < 4)) { return 0; } /* prepare the acceleration to be used in condition */ accel = accel < 1 ? 1 : accel; accel -= 1; htab = (uint16_t *) calloc(hash_size, sizeof(uint16_t)); /* we start with literal copy */ copy = 2; *op++ = MAX_COPY-1; *op++ = *ip++; *op++ = *ip++; /* main loop */ while(BLOSCLZ_EXPECT_CONDITIONAL(ip < ip_limit)) { const uint8_t* ref; int32_t distance; int32_t len = 3; /* minimum match length */ uint8_t* anchor = ip; /* comparison starting-point */ /* check for a run */ if(ip[0] == ip[-1] && BLOSCLZ_READU16(ip-1)==BLOSCLZ_READU16(ip+1)) { distance = 1; ip += 3; ref = anchor - 1 + 3; goto match; } /* find potential match */ HASH_FUNCTION(hval, ip, hash_log); ref = ibase + htab[hval]; /* calculate distance to the match */ distance = (int32_t)(anchor - ref); /* update hash table if necessary */ if ((distance & accel) == 0) htab[hval] = (uint16_t)(anchor - ibase); /* is this a match? check the first 3 bytes */ if (distance==0 || (distance >= MAX_FARDISTANCE) || *ref++ != *ip++ || *ref++!=*ip++ || *ref++!=*ip++) LITERAL(ip, op, op_limit, anchor, copy); /* far, needs at least 5-byte match */ if (opt_level >= 5 && distance >= MAX_DISTANCE) { if (*ip++ != *ref++ || *ip++ != *ref++) LITERAL(ip, op, op_limit, anchor, copy); len += 2; } match: /* last matched byte */ ip = anchor + len; /* distance is biased */ distance--; if(!distance) { /* zero distance means a run */ uint8_t x = ip[-1]; int64_t value, value2; /* Broadcast the value for every byte in a 64-bit register */ memset(&value, x, 8); /* safe because the outer check against ip limit */ while (ip < (ip_bound - (sizeof(int64_t) - IP_BOUNDARY))) { #if !defined(BLOSCLZ_STRICT_ALIGN) value2 = ((int64_t *)ref)[0]; #else memcpy(&value2, ref, 8); #endif if (value != value2) { /* Find the byte that starts to differ */ while (ip < ip_bound) { if (*ref++ != x) break; else ip++; } break; } else { ip += 8; ref += 8; } } if (ip > ip_bound) { long l = (long)(ip - ip_bound); ip -= l; ref -= l; } /* End of optimization */ } else { for(;;) { /* safe because the outer check against ip limit */ while (ip < (ip_bound - (sizeof(int64_t) - IP_BOUNDARY))) { #if !defined(BLOSCLZ_STRICT_ALIGN) if (((int64_t *)ref)[0] != ((int64_t *)ip)[0]) { #endif /* Find the byte that starts to differ */ while (ip < ip_bound) { if (*ref++ != *ip++) break; } break; #if !defined(BLOSCLZ_STRICT_ALIGN) } else { ip += 8; ref += 8; } #endif } /* Last correction before exiting loop */ if (ip > ip_bound) { int32_t l = (int32_t)(ip - ip_bound); ip -= l; ref -= l; } /* End of optimization */ break; } } /* if we have copied something, adjust the copy count */ if (copy) /* copy is biased, '0' means 1 byte copy */ *(op-copy-1) = copy-1; else /* back, to overwrite the copy count */ op--; /* reset literal counter */ copy = 0; /* length is biased, '1' means a match of 3 bytes */ ip -= 3; len = (int32_t)(ip - anchor); /* check that we have space enough to encode the match for all the cases */ if (BLOSCLZ_UNEXPECT_CONDITIONAL(op+(len/255)+6 > op_limit)) goto out; /* encode the match */ if(distance < MAX_DISTANCE) { if(len < 7) { *op++ = (len << 5) + (distance >> 8); *op++ = (distance & 255); } else { *op++ = (uint8_t)((7 << 5) + (distance >> 8)); for(len-=7; len >= 255; len-= 255) *op++ = 255; *op++ = len; *op++ = (distance & 255); } } else { /* far away, but not yet in the another galaxy... */ if(len < 7) { distance -= MAX_DISTANCE; *op++ = (uint8_t)((len << 5) + 31); *op++ = 255; *op++ = (uint8_t)(distance >> 8); *op++ = distance & 255; } else { distance -= MAX_DISTANCE; *op++ = (7 << 5) + 31; for(len-=7; len >= 255; len-= 255) *op++ = 255; *op++ = len; *op++ = 255; *op++ = (uint8_t)(distance >> 8); *op++ = distance & 255; } } /* update the hash at match boundary */ HASH_FUNCTION(hval, ip, hash_log); htab[hval] = (uint16_t)(ip++ - ibase); HASH_FUNCTION(hval, ip, hash_log); htab[hval] = (uint16_t)(ip++ - ibase); /* assuming literal copy */ *op++ = MAX_COPY-1; } /* left-over as literal copy */ ip_bound++; while(ip <= ip_bound) { if (BLOSCLZ_UNEXPECT_CONDITIONAL(op+2 > op_limit)) goto out; *op++ = *ip++; copy++; if(copy == MAX_COPY) { copy = 0; *op++ = MAX_COPY-1; } } /* if we have copied something, adjust the copy length */ if(copy) *(op-copy-1) = copy-1; else op--; /* marker for blosclz */ *(uint8_t*)output |= (1 << 5); free(htab); return (int)(op - (uint8_t*)output); out: free(htab); return 0; } int blosclz_decompress(const void* input, int length, void* output, int maxout) { const uint8_t* ip = (const uint8_t*) input; const uint8_t* ip_limit = ip + length; uint8_t* op = (uint8_t*) output; uint8_t* op_limit = op + maxout; int32_t ctrl = (*ip++) & 31; int32_t loop = 1; do { uint8_t* ref = op; int32_t len = ctrl >> 5; int32_t ofs = (ctrl & 31) << 8; if(ctrl >= 32) { uint8_t code; len--; ref -= ofs; if (len == 7-1) do { code = *ip++; len += code; } while (code==255); code = *ip++; ref -= code; /* match from 16-bit distance */ if(BLOSCLZ_UNEXPECT_CONDITIONAL(code==255)) if(BLOSCLZ_EXPECT_CONDITIONAL(ofs==(31 << 8))) { ofs = (*ip++) << 8; ofs += *ip++; ref = op - ofs - MAX_DISTANCE; } #ifdef BLOSCLZ_SAFE if (BLOSCLZ_UNEXPECT_CONDITIONAL(op + len + 3 > op_limit)) { return 0; } if (BLOSCLZ_UNEXPECT_CONDITIONAL(ref-1 < (uint8_t *)output)) { return 0; } #endif if(BLOSCLZ_EXPECT_CONDITIONAL(ip < ip_limit)) ctrl = *ip++; else loop = 0; if(ref == op) { /* optimize copy for a run */ uint8_t b = ref[-1]; memset(op, b, len+3); op += len+3; } else { /* copy from reference */ ref--; len += 3; #if !defined(_WIN32) && ((defined(__GNUC__) || defined(__INTEL_COMPILER) || !defined(__clang__))) GCC_SAFE_COPY(op, ref, len, op_limit); #else SAFE_COPY(op, ref, len, op_limit); #endif } } else { ctrl++; #ifdef BLOSCLZ_SAFE if (BLOSCLZ_UNEXPECT_CONDITIONAL(op + ctrl > op_limit)) { return 0; } if (BLOSCLZ_UNEXPECT_CONDITIONAL(ip + ctrl > ip_limit)) { return 0; } #endif BLOCK_COPY(op, ip, ctrl, op_limit); loop = (int32_t)BLOSCLZ_EXPECT_CONDITIONAL(ip < ip_limit); if(loop) ctrl = *ip++; } } while(BLOSCLZ_EXPECT_CONDITIONAL(loop)); return (int)(op - (uint8_t*)output); } c-blosc-1.7.0/blosc/blosclz.h000066400000000000000000000041551254622165100157750ustar00rootroot00000000000000/********************************************************************* Blosc - Blocked Shuffling and Compression Library Author: Francesc Alted See LICENSES/BLOSC.txt for details about copyright and rights to use. **********************************************************************/ /********************************************************************* The code in this file is heavily based on FastLZ, a lightning-fast lossless compression library. See LICENSES/FASTLZ.txt for details about copyright and rights to use. **********************************************************************/ #ifndef BLOSCLZ_H #define BLOSCLZ_H #if defined (__cplusplus) extern "C" { #endif /** Compress a block of data in the input buffer and returns the size of compressed block. The size of input buffer is specified by length. The minimum input buffer size is 16. The output buffer must be at least 5% larger than the input buffer and can not be smaller than 66 bytes. If the input is not compressible, or output does not fit in maxout bytes, the return value will be 0 and you will have to discard the output buffer. The acceleration parameter is related with the frequency for updating the internal hash. An acceleration of 1 means that the internal hash is updated at full rate. A value < 1 is not allowed and will be silently set to 1. The input buffer and the output buffer can not overlap. */ int blosclz_compress(const int opt_level, const void* input, int length, void* output, int maxout, int accel); /** Decompress a block of compressed data and returns the size of the decompressed block. If error occurs, e.g. the compressed data is corrupted or the output buffer is not large enough, then 0 (zero) will be returned instead. The input buffer and the output buffer can not overlap. Decompression is memory safe and guaranteed not to write the output buffer more than what is specified in maxout. */ int blosclz_decompress(const void* input, int length, void* output, int maxout); #if defined (__cplusplus) } #endif #endif /* BLOSCLZ_H */ c-blosc-1.7.0/blosc/config.h.in000066400000000000000000000003561254622165100161760ustar00rootroot00000000000000#ifndef _CONFIGURATION_HEADER_GUARD_H_ #define _CONFIGURATION_HEADER_GUARD_H_ #cmakedefine HAVE_LZ4 @HAVE_LZ4@ #cmakedefine HAVE_SNAPPY @HAVE_SNAPPY@ #cmakedefine HAVE_ZLIB @HAVE_ZLIB@ #cmakedefine BLOSC_DLL_EXPORT @DLL_EXPORT@ #endif c-blosc-1.7.0/blosc/shuffle-avx2.c000066400000000000000000000740141254622165100166330ustar00rootroot00000000000000/********************************************************************* Blosc - Blocked Shuffling and Compression Library Author: Francesc Alted See LICENSES/BLOSC.txt for details about copyright and rights to use. **********************************************************************/ #include "shuffle-generic.h" #include "shuffle-avx2.h" /* Make sure AVX2 is available for the compilation target and compiler. */ #if !defined(__AVX2__) #error AVX2 is not supported by the target architecture/platform and/or this compiler. #endif #include /* The next is useful for debugging purposes */ #if 0 #include #include static void printymm(__m256i ymm0) { uint8_t buf[32]; ((__m256i *)buf)[0] = ymm0; printf("%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x\n", buf[0], buf[1], buf[2], buf[3], buf[4], buf[5], buf[6], buf[7], buf[8], buf[9], buf[10], buf[11], buf[12], buf[13], buf[14], buf[15], buf[16], buf[17], buf[18], buf[19], buf[20], buf[21], buf[22], buf[23], buf[24], buf[25], buf[26], buf[27], buf[28], buf[29], buf[30], buf[31]); } #endif /* GCC doesn't include the split load/store intrinsics needed for the tiled shuffle, so define them here. */ #if defined(__GNUC__) && !defined(__clang__) static inline __m256i __attribute__((__always_inline__)) _mm256_loadu2_m128i(const __m128i* const hiaddr, const __m128i* const loaddr) { return _mm256_inserti128_si256( _mm256_castsi128_si256(_mm_loadu_si128(loaddr)), _mm_loadu_si128(hiaddr), 1); } static inline void __attribute__((__always_inline__)) _mm256_storeu2_m128i(__m128i* const hiaddr, __m128i* const loaddr, const __m256i a) { _mm_storeu_si128(loaddr, _mm256_castsi256_si128(a)); _mm_storeu_si128(hiaddr, _mm256_extracti128_si256(a, 1)); } #endif /* defined(__GNUC__) */ /* Routine optimized for shuffling a buffer for a type size of 2 bytes. */ static void shuffle2_avx2(uint8_t* const dest, const uint8_t* const src, const size_t vectorizable_elements, const size_t total_elements) { static const size_t bytesoftype = 2; size_t j; int k; __m256i ymm0[2], ymm1[2]; /* Create the shuffle mask. NOTE: The XMM/YMM 'set' intrinsics require the arguments to be ordered from most to least significant (i.e., their order is reversed when compared to loading the mask from an array). */ const __m256i shmask = _mm256_set_epi8( 0x0f, 0x0d, 0x0b, 0x09, 0x07, 0x05, 0x03, 0x01, 0x0e, 0x0c, 0x0a, 0x08, 0x06, 0x04, 0x02, 0x00, 0x0f, 0x0d, 0x0b, 0x09, 0x07, 0x05, 0x03, 0x01, 0x0e, 0x0c, 0x0a, 0x08, 0x06, 0x04, 0x02, 0x00); for (j = 0; j < vectorizable_elements; j += sizeof(__m256i)) { /* Fetch 32 elements (64 bytes) then transpose bytes, words and double words. */ for (k = 0; k < 2; k++) { ymm0[k] = _mm256_loadu_si256((__m256i*)(src + (j * bytesoftype) + (k * sizeof(__m256i)))); ymm1[k] = _mm256_shuffle_epi8(ymm0[k], shmask); } ymm0[0] = _mm256_permute4x64_epi64(ymm1[0], 0xd8); ymm0[1] = _mm256_permute4x64_epi64(ymm1[1], 0x8d); ymm1[0] = _mm256_blend_epi32(ymm0[0], ymm0[1], 0xf0); ymm0[1] = _mm256_blend_epi32(ymm0[0], ymm0[1], 0x0f); ymm1[1] = _mm256_permute4x64_epi64(ymm0[1], 0x4e); /* Store the result vectors */ uint8_t* const dest_for_jth_element = dest + j; for (k = 0; k < 2; k++) { _mm256_storeu_si256((__m256i*)(dest_for_jth_element + (k * total_elements)), ymm1[k]); } } } /* Routine optimized for shuffling a buffer for a type size of 4 bytes. */ static void shuffle4_avx2(uint8_t* const dest, const uint8_t* const src, const size_t vectorizable_elements, const size_t total_elements) { static const size_t bytesoftype = 4; size_t i; int j; __m256i ymm0[4], ymm1[4]; /* Create the shuffle mask. NOTE: The XMM/YMM 'set' intrinsics require the arguments to be ordered from most to least significant (i.e., their order is reversed when compared to loading the mask from an array). */ const __m256i mask = _mm256_set_epi32( 0x07, 0x03, 0x06, 0x02, 0x05, 0x01, 0x04, 0x00); for (i = 0; i < vectorizable_elements; i += sizeof(__m256i)) { /* Fetch 32 elements (128 bytes) then transpose bytes and words. */ for (j = 0; j < 4; j++) { ymm0[j] = _mm256_loadu_si256((__m256i*)(src + (i * bytesoftype) + (j * sizeof(__m256i)))); ymm1[j] = _mm256_shuffle_epi32(ymm0[j], 0xd8); ymm0[j] = _mm256_shuffle_epi32(ymm0[j], 0x8d); ymm0[j] = _mm256_unpacklo_epi8(ymm1[j], ymm0[j]); ymm1[j] = _mm256_shuffle_epi32(ymm0[j], 0x04e); ymm0[j] = _mm256_unpacklo_epi16(ymm0[j], ymm1[j]); } /* Transpose double words */ for (j = 0; j < 2; j++) { ymm1[j*2] = _mm256_unpacklo_epi32(ymm0[j*2], ymm0[j*2+1]); ymm1[j*2+1] = _mm256_unpackhi_epi32(ymm0[j*2], ymm0[j*2+1]); } /* Transpose quad words */ for (j = 0; j < 2; j++) { ymm0[j*2] = _mm256_unpacklo_epi64(ymm1[j], ymm1[j+2]); ymm0[j*2+1] = _mm256_unpackhi_epi64(ymm1[j], ymm1[j+2]); } for (j = 0; j < 4; j++) { ymm0[j] = _mm256_permutevar8x32_epi32(ymm0[j], mask); } /* Store the result vectors */ uint8_t* const dest_for_ith_element = dest + i; for (j = 0; j < 4; j++) { _mm256_storeu_si256((__m256i*)(dest_for_ith_element + (j * total_elements)), ymm0[j]); } } } /* Routine optimized for shuffling a buffer for a type size of 8 bytes. */ static void shuffle8_avx2(uint8_t* const dest, const uint8_t* const src, const size_t vectorizable_elements, const size_t total_elements) { static const size_t bytesoftype = 8; size_t j; int k, l; __m256i ymm0[8], ymm1[8]; for (j = 0; j < vectorizable_elements; j += sizeof(__m256i)) { /* Fetch 32 elements (256 bytes) then transpose bytes. */ for (k = 0; k < 8; k++) { ymm0[k] = _mm256_loadu_si256((__m256i*)(src + (j * bytesoftype) + (k * sizeof(__m256i)))); ymm1[k] = _mm256_shuffle_epi32(ymm0[k], 0x4e); ymm1[k] = _mm256_unpacklo_epi8(ymm0[k], ymm1[k]); } /* Transpose words */ for (k = 0, l = 0; k < 4; k++, l +=2) { ymm0[k*2] = _mm256_unpacklo_epi16(ymm1[l], ymm1[l+1]); ymm0[k*2+1] = _mm256_unpackhi_epi16(ymm1[l], ymm1[l+1]); } /* Transpose double words */ for (k = 0, l = 0; k < 4; k++, l++) { if (k == 2) l += 2; ymm1[k*2] = _mm256_unpacklo_epi32(ymm0[l], ymm0[l+2]); ymm1[k*2+1] = _mm256_unpackhi_epi32(ymm0[l], ymm0[l+2]); } /* Transpose quad words */ for (k = 0; k < 4; k++) { ymm0[k*2] = _mm256_unpacklo_epi64(ymm1[k], ymm1[k+4]); ymm0[k*2+1] = _mm256_unpackhi_epi64(ymm1[k], ymm1[k+4]); } for(k = 0; k < 8; k++) { ymm1[k] = _mm256_permute4x64_epi64(ymm0[k], 0x72); ymm0[k] = _mm256_permute4x64_epi64(ymm0[k], 0xD8); ymm0[k] = _mm256_unpacklo_epi16(ymm0[k], ymm1[k]); } /* Store the result vectors */ uint8_t* const dest_for_jth_element = dest + j; for (k = 0; k < 8; k++) { _mm256_storeu_si256((__m256i*)(dest_for_jth_element + (k * total_elements)), ymm0[k]); } } } /* Routine optimized for shuffling a buffer for a type size of 16 bytes. */ static void shuffle16_avx2(uint8_t* const dest, const uint8_t* const src, const size_t vectorizable_elements, const size_t total_elements) { static const size_t bytesoftype = 16; size_t j; int k, l; __m256i ymm0[16], ymm1[16]; /* Create the shuffle mask. NOTE: The XMM/YMM 'set' intrinsics require the arguments to be ordered from most to least significant (i.e., their order is reversed when compared to loading the mask from an array). */ const __m256i shmask = _mm256_set_epi8( 0x0f, 0x07, 0x0e, 0x06, 0x0d, 0x05, 0x0c, 0x04, 0x0b, 0x03, 0x0a, 0x02, 0x09, 0x01, 0x08, 0x00, 0x0f, 0x07, 0x0e, 0x06, 0x0d, 0x05, 0x0c, 0x04, 0x0b, 0x03, 0x0a, 0x02, 0x09, 0x01, 0x08, 0x00); for (j = 0; j < vectorizable_elements; j += sizeof(__m256i)) { /* Fetch 32 elements (512 bytes) into 16 YMM registers. */ for (k = 0; k < 16; k++) { ymm0[k] = _mm256_loadu_si256((__m256i*)(src + (j * bytesoftype) + (k * sizeof(__m256i)))); } /* Transpose bytes */ for (k = 0, l = 0; k < 8; k++, l +=2) { ymm1[k*2] = _mm256_unpacklo_epi8(ymm0[l], ymm0[l+1]); ymm1[k*2+1] = _mm256_unpackhi_epi8(ymm0[l], ymm0[l+1]); } /* Transpose words */ for (k = 0, l = -2; k < 8; k++, l++) { if ((k%2) == 0) l += 2; ymm0[k*2] = _mm256_unpacklo_epi16(ymm1[l], ymm1[l+2]); ymm0[k*2+1] = _mm256_unpackhi_epi16(ymm1[l], ymm1[l+2]); } /* Transpose double words */ for (k = 0, l = -4; k < 8; k++, l++) { if ((k%4) == 0) l += 4; ymm1[k*2] = _mm256_unpacklo_epi32(ymm0[l], ymm0[l+4]); ymm1[k*2+1] = _mm256_unpackhi_epi32(ymm0[l], ymm0[l+4]); } /* Transpose quad words */ for (k = 0; k < 8; k++) { ymm0[k*2] = _mm256_unpacklo_epi64(ymm1[k], ymm1[k+8]); ymm0[k*2+1] = _mm256_unpackhi_epi64(ymm1[k], ymm1[k+8]); } for (k = 0; k < 16; k++) { ymm0[k] = _mm256_permute4x64_epi64(ymm0[k], 0xd8); ymm0[k] = _mm256_shuffle_epi8(ymm0[k], shmask); } /* Store the result vectors */ uint8_t* const dest_for_jth_element = dest + j; for (k = 0; k < 16; k++) { _mm256_storeu_si256((__m256i*)(dest_for_jth_element + (k * total_elements)), ymm0[k]); } } } /* Routine optimized for shuffling a buffer for a type size larger than 16 bytes. */ static void shuffle16_tiled_avx2(uint8_t* const dest, const uint8_t* const src, const size_t vectorizable_elements, const size_t total_elements, const size_t bytesoftype) { size_t j; int k, l; __m256i ymm0[16], ymm1[16]; const lldiv_t vecs_per_el = lldiv(bytesoftype, sizeof(__m128i)); /* Create the shuffle mask. NOTE: The XMM/YMM 'set' intrinsics require the arguments to be ordered from most to least significant (i.e., their order is reversed when compared to loading the mask from an array). */ const __m256i shmask = _mm256_set_epi8( 0x0f, 0x07, 0x0e, 0x06, 0x0d, 0x05, 0x0c, 0x04, 0x0b, 0x03, 0x0a, 0x02, 0x09, 0x01, 0x08, 0x00, 0x0f, 0x07, 0x0e, 0x06, 0x0d, 0x05, 0x0c, 0x04, 0x0b, 0x03, 0x0a, 0x02, 0x09, 0x01, 0x08, 0x00); for (j = 0; j < vectorizable_elements; j += sizeof(__m256i)) { /* Advance the offset into the type by the vector size (in bytes), unless this is the initial iteration and the type size is not a multiple of the vector size. In that case, only advance by the number of bytes necessary so that the number of remaining bytes in the type will be a multiple of the vector size. */ size_t offset_into_type; for (offset_into_type = 0; offset_into_type < bytesoftype; offset_into_type += (offset_into_type == 0 && vecs_per_el.rem > 0 ? vecs_per_el.rem : sizeof(__m128i))) { /* Fetch elements in groups of 512 bytes */ const uint8_t* const src_with_offset = src + offset_into_type; for (k = 0; k < 16; k++) { ymm0[k] = _mm256_loadu2_m128i( (__m128i*)(src_with_offset + (j + (2 * k) + 1) * bytesoftype), (__m128i*)(src_with_offset + (j + (2 * k)) * bytesoftype)); } /* Transpose bytes */ for (k = 0, l = 0; k < 8; k++, l +=2) { ymm1[k*2] = _mm256_unpacklo_epi8(ymm0[l], ymm0[l+1]); ymm1[k*2+1] = _mm256_unpackhi_epi8(ymm0[l], ymm0[l+1]); } /* Transpose words */ for (k = 0, l = -2; k < 8; k++, l++) { if ((k%2) == 0) l += 2; ymm0[k*2] = _mm256_unpacklo_epi16(ymm1[l], ymm1[l+2]); ymm0[k*2+1] = _mm256_unpackhi_epi16(ymm1[l], ymm1[l+2]); } /* Transpose double words */ for (k = 0, l = -4; k < 8; k++, l++) { if ((k%4) == 0) l += 4; ymm1[k*2] = _mm256_unpacklo_epi32(ymm0[l], ymm0[l+4]); ymm1[k*2+1] = _mm256_unpackhi_epi32(ymm0[l], ymm0[l+4]); } /* Transpose quad words */ for (k = 0; k < 8; k++) { ymm0[k*2] = _mm256_unpacklo_epi64(ymm1[k], ymm1[k+8]); ymm0[k*2+1] = _mm256_unpackhi_epi64(ymm1[k], ymm1[k+8]); } for (k = 0; k < 16; k++) { ymm0[k] = _mm256_permute4x64_epi64(ymm0[k], 0xd8); ymm0[k] = _mm256_shuffle_epi8(ymm0[k], shmask); } /* Store the result vectors */ uint8_t* const dest_for_jth_element = dest + j; for (k = 0; k < 16; k++) { _mm256_storeu_si256((__m256i*)(dest_for_jth_element + (total_elements * (offset_into_type + k))), ymm0[k]); } } } } /* Routine optimized for unshuffling a buffer for a type size of 2 bytes. */ static void unshuffle2_avx2(uint8_t* const dest, const uint8_t* const src, const size_t vectorizable_elements, const size_t total_elements) { static const size_t bytesoftype = 2; size_t i; int j; __m256i ymm0[2], ymm1[2]; for (i = 0; i < vectorizable_elements; i += sizeof(__m256i)) { /* Load 32 elements (64 bytes) into 2 YMM registers. */ const uint8_t* const src_for_ith_element = src + i; for (j = 0; j < 2; j++) { ymm0[j] = _mm256_loadu_si256((__m256i*)(src_for_ith_element + (j * total_elements))); } /* Shuffle bytes */ for (j = 0; j < 2; j++) { ymm0[j] = _mm256_permute4x64_epi64(ymm0[j], 0xd8); } /* Compute the low 64 bytes */ ymm1[0] = _mm256_unpacklo_epi8(ymm0[0], ymm0[1]); /* Compute the hi 64 bytes */ ymm1[1] = _mm256_unpackhi_epi8(ymm0[0], ymm0[1]); /* Store the result vectors in proper order */ _mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (0 * sizeof(__m256i))), ymm1[0]); _mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (1 * sizeof(__m256i))), ymm1[1]); } } /* Routine optimized for unshuffling a buffer for a type size of 4 bytes. */ static void unshuffle4_avx2(uint8_t* const dest, const uint8_t* const src, const size_t vectorizable_elements, const size_t total_elements) { static const size_t bytesoftype = 4; size_t i; int j; __m256i ymm0[4], ymm1[4]; for (i = 0; i < vectorizable_elements; i += sizeof(__m256i)) { /* Load 32 elements (128 bytes) into 4 YMM registers. */ const uint8_t* const src_for_ith_element = src + i; for (j = 0; j < 4; j++) { ymm0[j] = _mm256_loadu_si256((__m256i*)(src_for_ith_element + (j * total_elements))); } /* Shuffle bytes */ for (j = 0; j < 2; j++) { /* Compute the low 64 bytes */ ymm1[j] = _mm256_unpacklo_epi8(ymm0[j*2], ymm0[j*2+1]); /* Compute the hi 64 bytes */ ymm1[2+j] = _mm256_unpackhi_epi8(ymm0[j*2], ymm0[j*2+1]); } /* Shuffle 2-byte words */ for (j = 0; j < 2; j++) { /* Compute the low 64 bytes */ ymm0[j] = _mm256_unpacklo_epi16(ymm1[j*2], ymm1[j*2+1]); /* Compute the hi 64 bytes */ ymm0[2+j] = _mm256_unpackhi_epi16(ymm1[j*2], ymm1[j*2+1]); } ymm1[0] = _mm256_permute2x128_si256(ymm0[0], ymm0[2], 0x20); ymm1[1] = _mm256_permute2x128_si256(ymm0[1], ymm0[3], 0x20); ymm1[2] = _mm256_permute2x128_si256(ymm0[0], ymm0[2], 0x31); ymm1[3] = _mm256_permute2x128_si256(ymm0[1], ymm0[3], 0x31); /* Store the result vectors in proper order */ for (j = 0; j < 4; j++) { _mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (j * sizeof(__m256i))), ymm1[j]); } } } /* Routine optimized for unshuffling a buffer for a type size of 8 bytes. */ static void unshuffle8_avx2(uint8_t* const dest, const uint8_t* const src, const size_t vectorizable_elements, const size_t total_elements) { static const size_t bytesoftype = 8; size_t i; int j; __m256i ymm0[8], ymm1[8]; for (i = 0; i < vectorizable_elements; i += sizeof(__m256i)) { /* Fetch 32 elements (256 bytes) into 8 YMM registers. */ const uint8_t* const src_for_ith_element = src + i; for (j = 0; j < 8; j++) { ymm0[j] = _mm256_loadu_si256((__m256i*)(src_for_ith_element + (j * total_elements))); } /* Shuffle bytes */ for (j = 0; j < 4; j++) { /* Compute the low 32 bytes */ ymm1[j] = _mm256_unpacklo_epi8(ymm0[j*2], ymm0[j*2+1]); /* Compute the hi 32 bytes */ ymm1[4+j] = _mm256_unpackhi_epi8(ymm0[j*2], ymm0[j*2+1]); } /* Shuffle words */ for (j = 0; j < 4; j++) { /* Compute the low 32 bytes */ ymm0[j] = _mm256_unpacklo_epi16(ymm1[j*2], ymm1[j*2+1]); /* Compute the hi 32 bytes */ ymm0[4+j] = _mm256_unpackhi_epi16(ymm1[j*2], ymm1[j*2+1]); } for (j = 0; j < 8; j++) { ymm0[j] = _mm256_permute4x64_epi64(ymm0[j], 0xd8); } /* Shuffle 4-byte dwords */ for (j = 0; j < 4; j++) { /* Compute the low 32 bytes */ ymm1[j] = _mm256_unpacklo_epi32(ymm0[j*2], ymm0[j*2+1]); /* Compute the hi 32 bytes */ ymm1[4+j] = _mm256_unpackhi_epi32(ymm0[j*2], ymm0[j*2+1]); } /* Store the result vectors in proper order */ _mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (0 * sizeof(__m256i))), ymm1[0]); _mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (1 * sizeof(__m256i))), ymm1[2]); _mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (2 * sizeof(__m256i))), ymm1[1]); _mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (3 * sizeof(__m256i))), ymm1[3]); _mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (4 * sizeof(__m256i))), ymm1[4]); _mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (5 * sizeof(__m256i))), ymm1[6]); _mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (6 * sizeof(__m256i))), ymm1[5]); _mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (7 * sizeof(__m256i))), ymm1[7]); } } /* Routine optimized for unshuffling a buffer for a type size of 16 bytes. */ static void unshuffle16_avx2(uint8_t* const dest, const uint8_t* const src, const size_t vectorizable_elements, const size_t total_elements) { static const size_t bytesoftype = 16; size_t i; int j; __m256i ymm0[16], ymm1[16]; for (i = 0; i < vectorizable_elements; i += sizeof(__m256i)) { /* Fetch 32 elements (512 bytes) into 16 YMM registers. */ const uint8_t* const src_for_ith_element = src + i; for (j = 0; j < 16; j++) { ymm0[j] = _mm256_loadu_si256((__m256i*)(src_for_ith_element + (j * total_elements))); } /* Shuffle bytes */ for (j = 0; j < 8; j++) { /* Compute the low 32 bytes */ ymm1[j] = _mm256_unpacklo_epi8(ymm0[j*2], ymm0[j*2+1]); /* Compute the hi 32 bytes */ ymm1[8+j] = _mm256_unpackhi_epi8(ymm0[j*2], ymm0[j*2+1]); } /* Shuffle 2-byte words */ for (j = 0; j < 8; j++) { /* Compute the low 32 bytes */ ymm0[j] = _mm256_unpacklo_epi16(ymm1[j*2], ymm1[j*2+1]); /* Compute the hi 32 bytes */ ymm0[8+j] = _mm256_unpackhi_epi16(ymm1[j*2], ymm1[j*2+1]); } /* Shuffle 4-byte dwords */ for (j = 0; j < 8; j++) { /* Compute the low 32 bytes */ ymm1[j] = _mm256_unpacklo_epi32(ymm0[j*2], ymm0[j*2+1]); /* Compute the hi 32 bytes */ ymm1[8+j] = _mm256_unpackhi_epi32(ymm0[j*2], ymm0[j*2+1]); } /* Shuffle 8-byte qwords */ for (j = 0; j < 8; j++) { /* Compute the low 32 bytes */ ymm0[j] = _mm256_unpacklo_epi64(ymm1[j*2], ymm1[j*2+1]); /* Compute the hi 32 bytes */ ymm0[8+j] = _mm256_unpackhi_epi64(ymm1[j*2], ymm1[j*2+1]); } for (j = 0; j < 8; j++) { ymm1[j] = _mm256_permute2x128_si256(ymm0[j], ymm0[j+8], 0x20); ymm1[j+8] = _mm256_permute2x128_si256(ymm0[j], ymm0[j+8], 0x31); } /* Store the result vectors in proper order */ _mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (0 * sizeof(__m256i))), ymm1[0]); _mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (1 * sizeof(__m256i))), ymm1[4]); _mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (2 * sizeof(__m256i))), ymm1[2]); _mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (3 * sizeof(__m256i))), ymm1[6]); _mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (4 * sizeof(__m256i))), ymm1[1]); _mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (5 * sizeof(__m256i))), ymm1[5]); _mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (6 * sizeof(__m256i))), ymm1[3]); _mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (7 * sizeof(__m256i))), ymm1[7]); _mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (8 * sizeof(__m256i))), ymm1[8]); _mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (9 * sizeof(__m256i))), ymm1[12]); _mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (10 * sizeof(__m256i))), ymm1[10]); _mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (11 * sizeof(__m256i))), ymm1[14]); _mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (12 * sizeof(__m256i))), ymm1[9]); _mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (13 * sizeof(__m256i))), ymm1[13]); _mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (14 * sizeof(__m256i))), ymm1[11]); _mm256_storeu_si256((__m256i*)(dest + (i * bytesoftype) + (15 * sizeof(__m256i))), ymm1[15]); } } /* Routine optimized for unshuffling a buffer for a type size larger than 16 bytes. */ static void unshuffle16_tiled_avx2(uint8_t* const dest, const uint8_t* const src, const size_t vectorizable_elements, const size_t total_elements, const size_t bytesoftype) { size_t i; int j; __m256i ymm0[16], ymm1[16]; const lldiv_t vecs_per_el = lldiv(bytesoftype, sizeof(__m128i)); /* The unshuffle loops are inverted (compared to shuffle_tiled16_avx2) to optimize cache utilization. */ size_t offset_into_type; for (offset_into_type = 0; offset_into_type < bytesoftype; offset_into_type += (offset_into_type == 0 && vecs_per_el.rem > 0 ? vecs_per_el.rem : sizeof(__m128i))) { for (i = 0; i < vectorizable_elements; i += sizeof(__m256i)) { /* Load the first 16 bytes of 32 adjacent elements (512 bytes) into 16 YMM registers */ const uint8_t* const src_for_ith_element = src + i; for (j = 0; j < 16; j++) { ymm0[j] = _mm256_loadu_si256((__m256i*)(src_for_ith_element + (total_elements * (offset_into_type + j)))); } /* Shuffle bytes */ for (j = 0; j < 8; j++) { /* Compute the low 32 bytes */ ymm1[j] = _mm256_unpacklo_epi8(ymm0[j*2], ymm0[j*2+1]); /* Compute the hi 32 bytes */ ymm1[8+j] = _mm256_unpackhi_epi8(ymm0[j*2], ymm0[j*2+1]); } /* Shuffle 2-byte words */ for (j = 0; j < 8; j++) { /* Compute the low 32 bytes */ ymm0[j] = _mm256_unpacklo_epi16(ymm1[j*2], ymm1[j*2+1]); /* Compute the hi 32 bytes */ ymm0[8+j] = _mm256_unpackhi_epi16(ymm1[j*2], ymm1[j*2+1]); } /* Shuffle 4-byte dwords */ for (j = 0; j < 8; j++) { /* Compute the low 32 bytes */ ymm1[j] = _mm256_unpacklo_epi32(ymm0[j*2], ymm0[j*2+1]); /* Compute the hi 32 bytes */ ymm1[8+j] = _mm256_unpackhi_epi32(ymm0[j*2], ymm0[j*2+1]); } /* Shuffle 8-byte qwords */ for (j = 0; j < 8; j++) { /* Compute the low 32 bytes */ ymm0[j] = _mm256_unpacklo_epi64(ymm1[j*2], ymm1[j*2+1]); /* Compute the hi 32 bytes */ ymm0[8+j] = _mm256_unpackhi_epi64(ymm1[j*2], ymm1[j*2+1]); } for (j = 0; j < 8; j++) { ymm1[j] = _mm256_permute2x128_si256(ymm0[j], ymm0[j+8], 0x20); ymm1[j+8] = _mm256_permute2x128_si256(ymm0[j], ymm0[j+8], 0x31); } /* Store the result vectors in proper order */ const uint8_t* const dest_with_offset = dest + offset_into_type; _mm256_storeu2_m128i( (__m128i*)(dest_with_offset + (i + 0x01) * bytesoftype), (__m128i*)(dest_with_offset + (i + 0x00) * bytesoftype), ymm1[0]); _mm256_storeu2_m128i( (__m128i*)(dest_with_offset + (i + 0x03) * bytesoftype), (__m128i*)(dest_with_offset + (i + 0x02) * bytesoftype), ymm1[4]); _mm256_storeu2_m128i( (__m128i*)(dest_with_offset + (i + 0x05) * bytesoftype), (__m128i*)(dest_with_offset + (i + 0x04) * bytesoftype), ymm1[2]); _mm256_storeu2_m128i( (__m128i*)(dest_with_offset + (i + 0x07) * bytesoftype), (__m128i*)(dest_with_offset + (i + 0x06) * bytesoftype), ymm1[6]); _mm256_storeu2_m128i( (__m128i*)(dest_with_offset + (i + 0x09) * bytesoftype), (__m128i*)(dest_with_offset + (i + 0x08) * bytesoftype), ymm1[1]); _mm256_storeu2_m128i( (__m128i*)(dest_with_offset + (i + 0x0b) * bytesoftype), (__m128i*)(dest_with_offset + (i + 0x0a) * bytesoftype), ymm1[5]); _mm256_storeu2_m128i( (__m128i*)(dest_with_offset + (i + 0x0d) * bytesoftype), (__m128i*)(dest_with_offset + (i + 0x0c) * bytesoftype), ymm1[3]); _mm256_storeu2_m128i( (__m128i*)(dest_with_offset + (i + 0x0f) * bytesoftype), (__m128i*)(dest_with_offset + (i + 0x0e) * bytesoftype), ymm1[7]); _mm256_storeu2_m128i( (__m128i*)(dest_with_offset + (i + 0x11) * bytesoftype), (__m128i*)(dest_with_offset + (i + 0x10) * bytesoftype), ymm1[8]); _mm256_storeu2_m128i( (__m128i*)(dest_with_offset + (i + 0x13) * bytesoftype), (__m128i*)(dest_with_offset + (i + 0x12) * bytesoftype), ymm1[12]); _mm256_storeu2_m128i( (__m128i*)(dest_with_offset + (i + 0x15) * bytesoftype), (__m128i*)(dest_with_offset + (i + 0x14) * bytesoftype), ymm1[10]); _mm256_storeu2_m128i( (__m128i*)(dest_with_offset + (i + 0x17) * bytesoftype), (__m128i*)(dest_with_offset + (i + 0x16) * bytesoftype), ymm1[14]); _mm256_storeu2_m128i( (__m128i*)(dest_with_offset + (i + 0x19) * bytesoftype), (__m128i*)(dest_with_offset + (i + 0x18) * bytesoftype), ymm1[9]); _mm256_storeu2_m128i( (__m128i*)(dest_with_offset + (i + 0x1b) * bytesoftype), (__m128i*)(dest_with_offset + (i + 0x1a) * bytesoftype), ymm1[13]); _mm256_storeu2_m128i( (__m128i*)(dest_with_offset + (i + 0x1d) * bytesoftype), (__m128i*)(dest_with_offset + (i + 0x1c) * bytesoftype), ymm1[11]); _mm256_storeu2_m128i( (__m128i*)(dest_with_offset + (i + 0x1f) * bytesoftype), (__m128i*)(dest_with_offset + (i + 0x1e) * bytesoftype), ymm1[15]); } } } /* Shuffle a block. This can never fail. */ void shuffle_avx2(const size_t bytesoftype, const size_t blocksize, const uint8_t* const _src, uint8_t* const _dest) { const size_t vectorized_chunk_size = bytesoftype * sizeof(__m256i); /* If the block size is too small to be vectorized, use the generic implementation. */ if (blocksize < vectorized_chunk_size) { shuffle_generic(bytesoftype, blocksize, _src, _dest); return; } /* If the blocksize is not a multiple of both the typesize and the vector size, round the blocksize down to the next value which is a multiple of both. The vectorized shuffle can be used for that portion of the data, and the naive implementation can be used for the remaining portion. */ const size_t vectorizable_bytes = blocksize - (blocksize % vectorized_chunk_size); const size_t vectorizable_elements = vectorizable_bytes / bytesoftype; const size_t total_elements = blocksize / bytesoftype; /* Optimized shuffle implementations */ switch (bytesoftype) { case 2: shuffle2_avx2(_dest, _src, vectorizable_elements, total_elements); break; case 4: shuffle4_avx2(_dest, _src, vectorizable_elements, total_elements); break; case 8: shuffle8_avx2(_dest, _src, vectorizable_elements, total_elements); break; case 16: shuffle16_avx2(_dest, _src, vectorizable_elements, total_elements); break; default: /* For types larger than 16 bytes, use the AVX2 tiled shuffle. */ if (bytesoftype > sizeof(__m128i)) { shuffle16_tiled_avx2(_dest, _src, vectorizable_elements, total_elements, bytesoftype); } else { /* Non-optimized shuffle */ shuffle_generic(bytesoftype, blocksize, _src, _dest); /* The non-optimized function covers the whole buffer, so we're done processing here. */ return; } } /* If the buffer had any bytes at the end which couldn't be handled by the vectorized implementations, use the non-optimized version to finish them up. */ if (vectorizable_bytes < blocksize) { shuffle_generic_inline(bytesoftype, vectorizable_bytes, blocksize, _src, _dest); } } /* Unshuffle a block. This can never fail. */ void unshuffle_avx2(const size_t bytesoftype, const size_t blocksize, const uint8_t* const _src, uint8_t* const _dest) { const size_t vectorized_chunk_size = bytesoftype * sizeof(__m256i); /* If the block size is too small to be vectorized, use the generic implementation. */ if (blocksize < vectorized_chunk_size) { unshuffle_generic(bytesoftype, blocksize, _src, _dest); return; } /* If the blocksize is not a multiple of both the typesize and the vector size, round the blocksize down to the next value which is a multiple of both. The vectorized unshuffle can be used for that portion of the data, and the naive implementation can be used for the remaining portion. */ const size_t vectorizable_bytes = blocksize - (blocksize % vectorized_chunk_size); const size_t vectorizable_elements = vectorizable_bytes / bytesoftype; const size_t total_elements = blocksize / bytesoftype; /* Optimized unshuffle implementations */ switch (bytesoftype) { case 2: unshuffle2_avx2(_dest, _src, vectorizable_elements, total_elements); break; case 4: unshuffle4_avx2(_dest, _src, vectorizable_elements, total_elements); break; case 8: unshuffle8_avx2(_dest, _src, vectorizable_elements, total_elements); break; case 16: unshuffle16_avx2(_dest, _src, vectorizable_elements, total_elements); break; default: /* For types larger than 16 bytes, use the AVX2 tiled unshuffle. */ if (bytesoftype > sizeof(__m128i)) { unshuffle16_tiled_avx2(_dest, _src, vectorizable_elements, total_elements, bytesoftype); } else { /* Non-optimized unshuffle */ unshuffle_generic(bytesoftype, blocksize, _src, _dest); /* The non-optimized function covers the whole buffer, so we're done processing here. */ return; } } /* If the buffer had any bytes at the end which couldn't be handled by the vectorized implementations, use the non-optimized version to finish them up. */ if (vectorizable_bytes < blocksize) { unshuffle_generic_inline(bytesoftype, vectorizable_bytes, blocksize, _src, _dest); } } c-blosc-1.7.0/blosc/shuffle-avx2.h000066400000000000000000000017151254622165100166360ustar00rootroot00000000000000/********************************************************************* Blosc - Blocked Shuffling and Compression Library Author: Francesc Alted See LICENSES/BLOSC.txt for details about copyright and rights to use. **********************************************************************/ /* AVX2-accelerated shuffle/unshuffle routines. */ #ifndef SHUFFLE_AVX2_H #define SHUFFLE_AVX2_H #include "shuffle-common.h" #ifdef __cplusplus extern "C" { #endif /** AVX2-accelerated shuffle routine. */ BLOSC_NO_EXPORT void shuffle_avx2(const size_t bytesoftype, const size_t blocksize, const uint8_t* const _src, uint8_t* const _dest); /** AVX2-accelerated unshuffle routine. */ BLOSC_NO_EXPORT void unshuffle_avx2(const size_t bytesoftype, const size_t blocksize, const uint8_t* const _src, uint8_t* const _dest); #ifdef __cplusplus } #endif #endif /* SHUFFLE_AVX2_H */ c-blosc-1.7.0/blosc/shuffle-common.h000066400000000000000000000020271254622165100172430ustar00rootroot00000000000000/********************************************************************* Blosc - Blocked Shuffling and Compression Library Author: Francesc Alted See LICENSES/BLOSC.txt for details about copyright and rights to use. **********************************************************************/ #ifndef SHUFFLE_COMMON_H #define SHUFFLE_COMMON_H #include "blosc-export.h" /* Define the __SSE2__ symbol if compiling with Visual C++ and targeting the minimum architecture level supporting SSE2. Other compilers define this as expected and emit warnings when it is re-defined. */ #if !defined(__SSE2__) && defined(_MSC_VER) && \ (defined(_M_X64) || (defined(_M_IX86) && _M_IX86_FP >= 2)) #define __SSE2__ #endif /* Import standard integer type definitions */ #if defined(_WIN32) && !defined(__MINGW32__) #include #include "win32/stdint-windows.h" #else #include #include #include #include #endif /* _WIN32 */ #endif /* SHUFFLE_COMMON_H */ c-blosc-1.7.0/blosc/shuffle-generic.c000066400000000000000000000016231254622165100173630ustar00rootroot00000000000000/********************************************************************* Blosc - Blocked Shuffling and Compression Library Author: Francesc Alted See LICENSES/BLOSC.txt for details about copyright and rights to use. **********************************************************************/ #include "shuffle-generic.h" /* Shuffle a block. This can never fail. */ void shuffle_generic(const size_t bytesoftype, const size_t blocksize, const uint8_t* const _src, uint8_t* const _dest) { /* Non-optimized shuffle */ shuffle_generic_inline(bytesoftype, 0, blocksize, _src, _dest); } /* Unshuffle a block. This can never fail. */ void unshuffle_generic(const size_t bytesoftype, const size_t blocksize, const uint8_t* const _src, uint8_t* const _dest) { /* Non-optimized unshuffle */ unshuffle_generic_inline(bytesoftype, 0, blocksize, _src, _dest); } c-blosc-1.7.0/blosc/shuffle-generic.h000066400000000000000000000067431254622165100174000ustar00rootroot00000000000000/********************************************************************* Blosc - Blocked Shuffling and Compression Library Author: Francesc Alted See LICENSES/BLOSC.txt for details about copyright and rights to use. **********************************************************************/ /* Generic (non-hardware-accelerated) shuffle/unshuffle routines. These are used when hardware-accelerated functions aren't available for a particular platform; they are also used by the hardware- accelerated functions to handle any remaining elements in a block which isn't a multiple of the hardware's vector size. */ #ifndef SHUFFLE_GENERIC_H #define SHUFFLE_GENERIC_H #include "shuffle-common.h" #include #ifdef __cplusplus extern "C" { #endif /** Generic (non-hardware-accelerated) shuffle routine. This is the pure element-copying nested loop. It is used by the generic shuffle implementation and also by the vectorized shuffle implementations to process any remaining elements in a block which is not a multiple of (type_size * vector_size). */ static void shuffle_generic_inline(const size_t type_size, const size_t vectorizable_blocksize, const size_t blocksize, const uint8_t* const _src, uint8_t* const _dest) { size_t i, j; /* Calculate the number of elements in the block. */ const size_t neblock_quot = blocksize / type_size; const size_t neblock_rem = blocksize % type_size; const size_t vectorizable_elements = vectorizable_blocksize / type_size; /* Non-optimized shuffle */ for (j = 0; j < type_size; j++) { for (i = vectorizable_elements; i < (size_t)neblock_quot; i++) { _dest[j*neblock_quot+i] = _src[i*type_size+j]; } } /* Copy any leftover bytes in the block without shuffling them. */ memcpy(_dest + (blocksize - neblock_rem), _src + (blocksize - neblock_rem), neblock_rem); } /** Generic (non-hardware-accelerated) unshuffle routine. This is the pure element-copying nested loop. It is used by the generic unshuffle implementation and also by the vectorized unshuffle implementations to process any remaining elements in a block which is not a multiple of (type_size * vector_size). */ static void unshuffle_generic_inline(const size_t type_size, const size_t vectorizable_blocksize, const size_t blocksize, const uint8_t* const _src, uint8_t* const _dest) { size_t i, j; /* Calculate the number of elements in the block. */ const size_t neblock_quot = blocksize / type_size; const size_t neblock_rem = blocksize % type_size; const size_t vectorizable_elements = vectorizable_blocksize / type_size; /* Non-optimized unshuffle */ for (i = vectorizable_elements; i < (size_t)neblock_quot; i++) { for (j = 0; j < type_size; j++) { _dest[i*type_size+j] = _src[j*neblock_quot+i]; } } /* Copy any leftover bytes in the block without unshuffling them. */ memcpy(_dest + (blocksize - neblock_rem), _src + (blocksize - neblock_rem), neblock_rem); } /** Generic (non-hardware-accelerated) shuffle routine. */ BLOSC_NO_EXPORT void shuffle_generic(const size_t bytesoftype, const size_t blocksize, const uint8_t* const _src, uint8_t* const _dest); /** Generic (non-hardware-accelerated) unshuffle routine. */ BLOSC_NO_EXPORT void unshuffle_generic(const size_t bytesoftype, const size_t blocksize, const uint8_t* const _src, uint8_t* const _dest); #ifdef __cplusplus } #endif #endif /* SHUFFLE_GENERIC_H */ c-blosc-1.7.0/blosc/shuffle-sse2.c000066400000000000000000000614711254622165100166320ustar00rootroot00000000000000/********************************************************************* Blosc - Blocked Shuffling and Compression Library Author: Francesc Alted See LICENSES/BLOSC.txt for details about copyright and rights to use. **********************************************************************/ #include "shuffle-generic.h" #include "shuffle-sse2.h" /* Make sure SSE2 is available for the compilation target and compiler. */ #if !defined(__SSE2__) #error SSE2 is not supported by the target architecture/platform and/or this compiler. #endif #include /* The next is useful for debugging purposes */ #if 0 #include #include static void printxmm(__m128i xmm0) { uint8_t buf[16]; ((__m128i *)buf)[0] = xmm0; printf("%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x\n", buf[0], buf[1], buf[2], buf[3], buf[4], buf[5], buf[6], buf[7], buf[8], buf[9], buf[10], buf[11], buf[12], buf[13], buf[14], buf[15]); } #endif /* Routine optimized for shuffling a buffer for a type size of 2 bytes. */ static void shuffle2_sse2(uint8_t* const dest, const uint8_t* const src, const size_t vectorizable_elements, const size_t total_elements) { static const size_t bytesoftype = 2; size_t j; int k; uint8_t* dest_for_jth_element; __m128i xmm0[2], xmm1[2]; for (j = 0; j < vectorizable_elements; j += sizeof(__m128i)) { /* Fetch 16 elements (32 bytes) then transpose bytes, words and double words. */ for (k = 0; k < 2; k++) { xmm0[k] = _mm_loadu_si128((__m128i*)(src + (j * bytesoftype) + (k * sizeof(__m128i)))); xmm0[k] = _mm_shufflelo_epi16(xmm0[k], 0xd8); xmm0[k] = _mm_shufflehi_epi16(xmm0[k], 0xd8); xmm0[k] = _mm_shuffle_epi32(xmm0[k], 0xd8); xmm1[k] = _mm_shuffle_epi32(xmm0[k], 0x4e); xmm0[k] = _mm_unpacklo_epi8(xmm0[k], xmm1[k]); xmm0[k] = _mm_shuffle_epi32(xmm0[k], 0xd8); xmm1[k] = _mm_shuffle_epi32(xmm0[k], 0x4e); xmm0[k] = _mm_unpacklo_epi16(xmm0[k], xmm1[k]); xmm0[k] = _mm_shuffle_epi32(xmm0[k], 0xd8); } /* Transpose quad words */ for (k = 0; k < 1; k++) { xmm1[k*2] = _mm_unpacklo_epi64(xmm0[k], xmm0[k+1]); xmm1[k*2+1] = _mm_unpackhi_epi64(xmm0[k], xmm0[k+1]); } /* Store the result vectors */ dest_for_jth_element = dest + j; for (k = 0; k < 2; k++) { _mm_storeu_si128((__m128i*)(dest_for_jth_element + (k * total_elements)), xmm1[k]); } } } /* Routine optimized for shuffling a buffer for a type size of 4 bytes. */ static void shuffle4_sse2(uint8_t* const dest, const uint8_t* const src, const size_t vectorizable_elements, const size_t total_elements) { static const size_t bytesoftype = 4; size_t i; int j; uint8_t* dest_for_ith_element; __m128i xmm0[4], xmm1[4]; for (i = 0; i < vectorizable_elements; i += sizeof(__m128i)) { /* Fetch 16 elements (64 bytes) then transpose bytes and words. */ for (j = 0; j < 4; j++) { xmm0[j] = _mm_loadu_si128((__m128i*)(src + (i * bytesoftype) + (j * sizeof(__m128i)))); xmm1[j] = _mm_shuffle_epi32(xmm0[j], 0xd8); xmm0[j] = _mm_shuffle_epi32(xmm0[j], 0x8d); xmm0[j] = _mm_unpacklo_epi8(xmm1[j], xmm0[j]); xmm1[j] = _mm_shuffle_epi32(xmm0[j], 0x04e); xmm0[j] = _mm_unpacklo_epi16(xmm0[j], xmm1[j]); } /* Transpose double words */ for (j = 0; j < 2; j++) { xmm1[j*2] = _mm_unpacklo_epi32(xmm0[j*2], xmm0[j*2+1]); xmm1[j*2+1] = _mm_unpackhi_epi32(xmm0[j*2], xmm0[j*2+1]); } /* Transpose quad words */ for (j = 0; j < 2; j++) { xmm0[j*2] = _mm_unpacklo_epi64(xmm1[j], xmm1[j+2]); xmm0[j*2+1] = _mm_unpackhi_epi64(xmm1[j], xmm1[j+2]); } /* Store the result vectors */ dest_for_ith_element = dest + i; for (j = 0; j < 4; j++) { _mm_storeu_si128((__m128i*)(dest_for_ith_element + (j * total_elements)), xmm0[j]); } } } /* Routine optimized for shuffling a buffer for a type size of 8 bytes. */ static void shuffle8_sse2(uint8_t* const dest, const uint8_t* const src, const size_t vectorizable_elements, const size_t total_elements) { static const size_t bytesoftype = 8; size_t j; int k, l; uint8_t* dest_for_jth_element; __m128i xmm0[8], xmm1[8]; for (j = 0; j < vectorizable_elements; j += sizeof(__m128i)) { /* Fetch 16 elements (128 bytes) then transpose bytes. */ for (k = 0; k < 8; k++) { xmm0[k] = _mm_loadu_si128((__m128i*)(src + (j * bytesoftype) + (k * sizeof(__m128i)))); xmm1[k] = _mm_shuffle_epi32(xmm0[k], 0x4e); xmm1[k] = _mm_unpacklo_epi8(xmm0[k], xmm1[k]); } /* Transpose words */ for (k = 0, l = 0; k < 4; k++, l +=2) { xmm0[k*2] = _mm_unpacklo_epi16(xmm1[l], xmm1[l+1]); xmm0[k*2+1] = _mm_unpackhi_epi16(xmm1[l], xmm1[l+1]); } /* Transpose double words */ for (k = 0, l = 0; k < 4; k++, l++) { if (k == 2) l += 2; xmm1[k*2] = _mm_unpacklo_epi32(xmm0[l], xmm0[l+2]); xmm1[k*2+1] = _mm_unpackhi_epi32(xmm0[l], xmm0[l+2]); } /* Transpose quad words */ for (k = 0; k < 4; k++) { xmm0[k*2] = _mm_unpacklo_epi64(xmm1[k], xmm1[k+4]); xmm0[k*2+1] = _mm_unpackhi_epi64(xmm1[k], xmm1[k+4]); } /* Store the result vectors */ dest_for_jth_element = dest + j; for (k = 0; k < 8; k++) { _mm_storeu_si128((__m128i*)(dest_for_jth_element + (k * total_elements)), xmm0[k]); } } } /* Routine optimized for shuffling a buffer for a type size of 16 bytes. */ static void shuffle16_sse2(uint8_t* const dest, const uint8_t* const src, const size_t vectorizable_elements, const size_t total_elements) { static const size_t bytesoftype = 16; size_t j; int k, l; uint8_t* dest_for_jth_element; __m128i xmm0[16], xmm1[16]; for (j = 0; j < vectorizable_elements; j += sizeof(__m128i)) { /* Fetch 16 elements (256 bytes). */ for (k = 0; k < 16; k++) { xmm0[k] = _mm_loadu_si128((__m128i*)(src + (j * bytesoftype) + (k * sizeof(__m128i)))); } /* Transpose bytes */ for (k = 0, l = 0; k < 8; k++, l +=2) { xmm1[k*2] = _mm_unpacklo_epi8(xmm0[l], xmm0[l+1]); xmm1[k*2+1] = _mm_unpackhi_epi8(xmm0[l], xmm0[l+1]); } /* Transpose words */ for (k = 0, l = -2; k < 8; k++, l++) { if ((k%2) == 0) l += 2; xmm0[k*2] = _mm_unpacklo_epi16(xmm1[l], xmm1[l+2]); xmm0[k*2+1] = _mm_unpackhi_epi16(xmm1[l], xmm1[l+2]); } /* Transpose double words */ for (k = 0, l = -4; k < 8; k++, l++) { if ((k%4) == 0) l += 4; xmm1[k*2] = _mm_unpacklo_epi32(xmm0[l], xmm0[l+4]); xmm1[k*2+1] = _mm_unpackhi_epi32(xmm0[l], xmm0[l+4]); } /* Transpose quad words */ for (k = 0; k < 8; k++) { xmm0[k*2] = _mm_unpacklo_epi64(xmm1[k], xmm1[k+8]); xmm0[k*2+1] = _mm_unpackhi_epi64(xmm1[k], xmm1[k+8]); } /* Store the result vectors */ dest_for_jth_element = dest + j; for (k = 0; k < 16; k++) { _mm_storeu_si128((__m128i*)(dest_for_jth_element + (k * total_elements)), xmm0[k]); } } } /* Routine optimized for shuffling a buffer for a type size larger than 16 bytes. */ static void shuffle16_tiled_sse2(uint8_t* const dest, const uint8_t* const src, const size_t vectorizable_elements, const size_t total_elements, const size_t bytesoftype) { size_t j; const size_t vecs_per_el_rem = bytesoftype % sizeof(__m128i); int k, l; uint8_t* dest_for_jth_element; __m128i xmm0[16], xmm1[16]; for (j = 0; j < vectorizable_elements; j += sizeof(__m128i)) { /* Advance the offset into the type by the vector size (in bytes), unless this is the initial iteration and the type size is not a multiple of the vector size. In that case, only advance by the number of bytes necessary so that the number of remaining bytes in the type will be a multiple of the vector size. */ size_t offset_into_type; for (offset_into_type = 0; offset_into_type < bytesoftype; offset_into_type += (offset_into_type == 0 && vecs_per_el_rem > 0 ? vecs_per_el_rem : sizeof(__m128i))) { /* Fetch elements in groups of 256 bytes */ const uint8_t* const src_with_offset = src + offset_into_type; for (k = 0; k < 16; k++) { xmm0[k] = _mm_loadu_si128((__m128i*)(src_with_offset + (j + k) * bytesoftype)); } /* Transpose bytes */ for (k = 0, l = 0; k < 8; k++, l +=2) { xmm1[k*2] = _mm_unpacklo_epi8(xmm0[l], xmm0[l+1]); xmm1[k*2+1] = _mm_unpackhi_epi8(xmm0[l], xmm0[l+1]); } /* Transpose words */ for (k = 0, l = -2; k < 8; k++, l++) { if ((k%2) == 0) l += 2; xmm0[k*2] = _mm_unpacklo_epi16(xmm1[l], xmm1[l+2]); xmm0[k*2+1] = _mm_unpackhi_epi16(xmm1[l], xmm1[l+2]); } /* Transpose double words */ for (k = 0, l = -4; k < 8; k++, l++) { if ((k%4) == 0) l += 4; xmm1[k*2] = _mm_unpacklo_epi32(xmm0[l], xmm0[l+4]); xmm1[k*2+1] = _mm_unpackhi_epi32(xmm0[l], xmm0[l+4]); } /* Transpose quad words */ for (k = 0; k < 8; k++) { xmm0[k*2] = _mm_unpacklo_epi64(xmm1[k], xmm1[k+8]); xmm0[k*2+1] = _mm_unpackhi_epi64(xmm1[k], xmm1[k+8]); } /* Store the result vectors */ dest_for_jth_element = dest + j; for (k = 0; k < 16; k++) { _mm_storeu_si128((__m128i*)(dest_for_jth_element + (total_elements * (offset_into_type + k))), xmm0[k]); } } } } /* Routine optimized for unshuffling a buffer for a type size of 2 bytes. */ static void unshuffle2_sse2(uint8_t* const dest, const uint8_t* const src, const size_t vectorizable_elements, const size_t total_elements) { static const size_t bytesoftype = 2; size_t i; int j; __m128i xmm0[2], xmm1[2]; for (i = 0; i < vectorizable_elements; i += sizeof(__m128i)) { /* Load 16 elements (32 bytes) into 2 XMM registers. */ const uint8_t* const src_for_ith_element = src + i; for (j = 0; j < 2; j++) { xmm0[j] = _mm_loadu_si128((__m128i*)(src_for_ith_element + (j * total_elements))); } /* Shuffle bytes */ /* Compute the low 32 bytes */ xmm1[0] = _mm_unpacklo_epi8(xmm0[0], xmm0[1]); /* Compute the hi 32 bytes */ xmm1[1] = _mm_unpackhi_epi8(xmm0[0], xmm0[1]); /* Store the result vectors in proper order */ _mm_storeu_si128((__m128i*)(dest + (i * bytesoftype) + (0 * sizeof(__m128i))), xmm1[0]); _mm_storeu_si128((__m128i*)(dest + (i * bytesoftype) + (1 * sizeof(__m128i))), xmm1[1]); } } /* Routine optimized for unshuffling a buffer for a type size of 4 bytes. */ static void unshuffle4_sse2(uint8_t* const dest, const uint8_t* const src, const size_t vectorizable_elements, const size_t total_elements) { static const size_t bytesoftype = 4; size_t i; int j; __m128i xmm0[4], xmm1[4]; for (i = 0; i < vectorizable_elements; i += sizeof(__m128i)) { /* Load 16 elements (64 bytes) into 4 XMM registers. */ const uint8_t* const src_for_ith_element = src + i; for (j = 0; j < 4; j++) { xmm0[j] = _mm_loadu_si128((__m128i*)(src_for_ith_element + (j * total_elements))); } /* Shuffle bytes */ for (j = 0; j < 2; j++) { /* Compute the low 32 bytes */ xmm1[j] = _mm_unpacklo_epi8(xmm0[j*2], xmm0[j*2+1]); /* Compute the hi 32 bytes */ xmm1[2+j] = _mm_unpackhi_epi8(xmm0[j*2], xmm0[j*2+1]); } /* Shuffle 2-byte words */ for (j = 0; j < 2; j++) { /* Compute the low 32 bytes */ xmm0[j] = _mm_unpacklo_epi16(xmm1[j*2], xmm1[j*2+1]); /* Compute the hi 32 bytes */ xmm0[2+j] = _mm_unpackhi_epi16(xmm1[j*2], xmm1[j*2+1]); } /* Store the result vectors in proper order */ _mm_storeu_si128((__m128i*)(dest + (i * bytesoftype) + (0 * sizeof(__m128i))), xmm0[0]); _mm_storeu_si128((__m128i*)(dest + (i * bytesoftype) + (1 * sizeof(__m128i))), xmm0[2]); _mm_storeu_si128((__m128i*)(dest + (i * bytesoftype) + (2 * sizeof(__m128i))), xmm0[1]); _mm_storeu_si128((__m128i*)(dest + (i * bytesoftype) + (3 * sizeof(__m128i))), xmm0[3]); } } /* Routine optimized for unshuffling a buffer for a type size of 8 bytes. */ static void unshuffle8_sse2(uint8_t* const dest, const uint8_t* const src, const size_t vectorizable_elements, const size_t total_elements) { static const size_t bytesoftype = 8; size_t i; int j; __m128i xmm0[8], xmm1[8]; for (i = 0; i < vectorizable_elements; i += sizeof(__m128i)) { /* Load 16 elements (128 bytes) into 8 XMM registers. */ const uint8_t* const src_for_ith_element = src + i; for (j = 0; j < 8; j++) { xmm0[j] = _mm_loadu_si128((__m128i*)(src_for_ith_element + (j * total_elements))); } /* Shuffle bytes */ for (j = 0; j < 4; j++) { /* Compute the low 32 bytes */ xmm1[j] = _mm_unpacklo_epi8(xmm0[j*2], xmm0[j*2+1]); /* Compute the hi 32 bytes */ xmm1[4+j] = _mm_unpackhi_epi8(xmm0[j*2], xmm0[j*2+1]); } /* Shuffle 2-byte words */ for (j = 0; j < 4; j++) { /* Compute the low 32 bytes */ xmm0[j] = _mm_unpacklo_epi16(xmm1[j*2], xmm1[j*2+1]); /* Compute the hi 32 bytes */ xmm0[4+j] = _mm_unpackhi_epi16(xmm1[j*2], xmm1[j*2+1]); } /* Shuffle 4-byte dwords */ for (j = 0; j < 4; j++) { /* Compute the low 32 bytes */ xmm1[j] = _mm_unpacklo_epi32(xmm0[j*2], xmm0[j*2+1]); /* Compute the hi 32 bytes */ xmm1[4+j] = _mm_unpackhi_epi32(xmm0[j*2], xmm0[j*2+1]); } /* Store the result vectors in proper order */ _mm_storeu_si128((__m128i*)(dest + (i * bytesoftype) + (0 * sizeof(__m128i))), xmm1[0]); _mm_storeu_si128((__m128i*)(dest + (i * bytesoftype) + (1 * sizeof(__m128i))), xmm1[4]); _mm_storeu_si128((__m128i*)(dest + (i * bytesoftype) + (2 * sizeof(__m128i))), xmm1[2]); _mm_storeu_si128((__m128i*)(dest + (i * bytesoftype) + (3 * sizeof(__m128i))), xmm1[6]); _mm_storeu_si128((__m128i*)(dest + (i * bytesoftype) + (4 * sizeof(__m128i))), xmm1[1]); _mm_storeu_si128((__m128i*)(dest + (i * bytesoftype) + (5 * sizeof(__m128i))), xmm1[5]); _mm_storeu_si128((__m128i*)(dest + (i * bytesoftype) + (6 * sizeof(__m128i))), xmm1[3]); _mm_storeu_si128((__m128i*)(dest + (i * bytesoftype) + (7 * sizeof(__m128i))), xmm1[7]); } } /* Routine optimized for unshuffling a buffer for a type size of 16 bytes. */ static void unshuffle16_sse2(uint8_t* const dest, const uint8_t* const src, const size_t vectorizable_elements, const size_t total_elements) { static const size_t bytesoftype = 16; size_t i; int j; __m128i xmm1[16], xmm2[16]; for (i = 0; i < vectorizable_elements; i += sizeof(__m128i)) { /* Load 16 elements (256 bytes) into 16 XMM registers. */ const uint8_t* const src_for_ith_element = src + i; for (j = 0; j < 16; j++) { xmm1[j] = _mm_loadu_si128((__m128i*)(src_for_ith_element + (j * total_elements))); } /* Shuffle bytes */ for (j = 0; j < 8; j++) { /* Compute the low 32 bytes */ xmm2[j] = _mm_unpacklo_epi8(xmm1[j*2], xmm1[j*2+1]); /* Compute the hi 32 bytes */ xmm2[8+j] = _mm_unpackhi_epi8(xmm1[j*2], xmm1[j*2+1]); } /* Shuffle 2-byte words */ for (j = 0; j < 8; j++) { /* Compute the low 32 bytes */ xmm1[j] = _mm_unpacklo_epi16(xmm2[j*2], xmm2[j*2+1]); /* Compute the hi 32 bytes */ xmm1[8+j] = _mm_unpackhi_epi16(xmm2[j*2], xmm2[j*2+1]); } /* Shuffle 4-byte dwords */ for (j = 0; j < 8; j++) { /* Compute the low 32 bytes */ xmm2[j] = _mm_unpacklo_epi32(xmm1[j*2], xmm1[j*2+1]); /* Compute the hi 32 bytes */ xmm2[8+j] = _mm_unpackhi_epi32(xmm1[j*2], xmm1[j*2+1]); } /* Shuffle 8-byte qwords */ for (j = 0; j < 8; j++) { /* Compute the low 32 bytes */ xmm1[j] = _mm_unpacklo_epi64(xmm2[j*2], xmm2[j*2+1]); /* Compute the hi 32 bytes */ xmm1[8+j] = _mm_unpackhi_epi64(xmm2[j*2], xmm2[j*2+1]); } /* Store the result vectors in proper order */ _mm_storeu_si128((__m128i*)(dest + (i * bytesoftype) + (0 * sizeof(__m128i))), xmm1[0]); _mm_storeu_si128((__m128i*)(dest + (i * bytesoftype) + (1 * sizeof(__m128i))), xmm1[8]); _mm_storeu_si128((__m128i*)(dest + (i * bytesoftype) + (2 * sizeof(__m128i))), xmm1[4]); _mm_storeu_si128((__m128i*)(dest + (i * bytesoftype) + (3 * sizeof(__m128i))), xmm1[12]); _mm_storeu_si128((__m128i*)(dest + (i * bytesoftype) + (4 * sizeof(__m128i))), xmm1[2]); _mm_storeu_si128((__m128i*)(dest + (i * bytesoftype) + (5 * sizeof(__m128i))), xmm1[10]); _mm_storeu_si128((__m128i*)(dest + (i * bytesoftype) + (6 * sizeof(__m128i))), xmm1[6]); _mm_storeu_si128((__m128i*)(dest + (i * bytesoftype) + (7 * sizeof(__m128i))), xmm1[14]); _mm_storeu_si128((__m128i*)(dest + (i * bytesoftype) + (8 * sizeof(__m128i))), xmm1[1]); _mm_storeu_si128((__m128i*)(dest + (i * bytesoftype) + (9 * sizeof(__m128i))), xmm1[9]); _mm_storeu_si128((__m128i*)(dest + (i * bytesoftype) + (10 * sizeof(__m128i))), xmm1[5]); _mm_storeu_si128((__m128i*)(dest + (i * bytesoftype) + (11 * sizeof(__m128i))), xmm1[13]); _mm_storeu_si128((__m128i*)(dest + (i * bytesoftype) + (12 * sizeof(__m128i))), xmm1[3]); _mm_storeu_si128((__m128i*)(dest + (i * bytesoftype) + (13 * sizeof(__m128i))), xmm1[11]); _mm_storeu_si128((__m128i*)(dest + (i * bytesoftype) + (14 * sizeof(__m128i))), xmm1[7]); _mm_storeu_si128((__m128i*)(dest + (i * bytesoftype) + (15 * sizeof(__m128i))), xmm1[15]); } } /* Routine optimized for unshuffling a buffer for a type size larger than 16 bytes. */ static void unshuffle16_tiled_sse2(uint8_t* const dest, const uint8_t* const orig, const size_t vectorizable_elements, const size_t total_elements, const size_t bytesoftype) { size_t i; const size_t vecs_per_el_rem = bytesoftype % sizeof(__m128i); int j; uint8_t* dest_with_offset; __m128i xmm1[16], xmm2[16]; /* The unshuffle loops are inverted (compared to shuffle_tiled16_sse2) to optimize cache utilization. */ size_t offset_into_type; for (offset_into_type = 0; offset_into_type < bytesoftype; offset_into_type += (offset_into_type == 0 && vecs_per_el_rem > 0 ? vecs_per_el_rem : sizeof(__m128i))) { for (i = 0; i < vectorizable_elements; i += sizeof(__m128i)) { /* Load the first 128 bytes in 16 XMM registers */ const uint8_t* const src_for_ith_element = orig + i; for (j = 0; j < 16; j++) { xmm1[j] = _mm_loadu_si128((__m128i*)(src_for_ith_element + (total_elements * (offset_into_type + j)))); } /* Shuffle bytes */ for (j = 0; j < 8; j++) { /* Compute the low 32 bytes */ xmm2[j] = _mm_unpacklo_epi8(xmm1[j*2], xmm1[j*2+1]); /* Compute the hi 32 bytes */ xmm2[8+j] = _mm_unpackhi_epi8(xmm1[j*2], xmm1[j*2+1]); } /* Shuffle 2-byte words */ for (j = 0; j < 8; j++) { /* Compute the low 32 bytes */ xmm1[j] = _mm_unpacklo_epi16(xmm2[j*2], xmm2[j*2+1]); /* Compute the hi 32 bytes */ xmm1[8+j] = _mm_unpackhi_epi16(xmm2[j*2], xmm2[j*2+1]); } /* Shuffle 4-byte dwords */ for (j = 0; j < 8; j++) { /* Compute the low 32 bytes */ xmm2[j] = _mm_unpacklo_epi32(xmm1[j*2], xmm1[j*2+1]); /* Compute the hi 32 bytes */ xmm2[8+j] = _mm_unpackhi_epi32(xmm1[j*2], xmm1[j*2+1]); } /* Shuffle 8-byte qwords */ for (j = 0; j < 8; j++) { /* Compute the low 32 bytes */ xmm1[j] = _mm_unpacklo_epi64(xmm2[j*2], xmm2[j*2+1]); /* Compute the hi 32 bytes */ xmm1[8+j] = _mm_unpackhi_epi64(xmm2[j*2], xmm2[j*2+1]); } /* Store the result vectors in proper order */ dest_with_offset = dest + offset_into_type; _mm_storeu_si128((__m128i*)(dest_with_offset + (i + 0) * bytesoftype), xmm1[0]); _mm_storeu_si128((__m128i*)(dest_with_offset + (i + 1) * bytesoftype), xmm1[8]); _mm_storeu_si128((__m128i*)(dest_with_offset + (i + 2) * bytesoftype), xmm1[4]); _mm_storeu_si128((__m128i*)(dest_with_offset + (i + 3) * bytesoftype), xmm1[12]); _mm_storeu_si128((__m128i*)(dest_with_offset + (i + 4) * bytesoftype), xmm1[2]); _mm_storeu_si128((__m128i*)(dest_with_offset + (i + 5) * bytesoftype), xmm1[10]); _mm_storeu_si128((__m128i*)(dest_with_offset + (i + 6) * bytesoftype), xmm1[6]); _mm_storeu_si128((__m128i*)(dest_with_offset + (i + 7) * bytesoftype), xmm1[14]); _mm_storeu_si128((__m128i*)(dest_with_offset + (i + 8) * bytesoftype), xmm1[1]); _mm_storeu_si128((__m128i*)(dest_with_offset + (i + 9) * bytesoftype), xmm1[9]); _mm_storeu_si128((__m128i*)(dest_with_offset + (i + 10) * bytesoftype), xmm1[5]); _mm_storeu_si128((__m128i*)(dest_with_offset + (i + 11) * bytesoftype), xmm1[13]); _mm_storeu_si128((__m128i*)(dest_with_offset + (i + 12) * bytesoftype), xmm1[3]); _mm_storeu_si128((__m128i*)(dest_with_offset + (i + 13) * bytesoftype), xmm1[11]); _mm_storeu_si128((__m128i*)(dest_with_offset + (i + 14) * bytesoftype), xmm1[7]); _mm_storeu_si128((__m128i*)(dest_with_offset + (i + 15) * bytesoftype), xmm1[15]); } } } /* Shuffle a block. This can never fail. */ void shuffle_sse2(const size_t bytesoftype, const size_t blocksize, const uint8_t* const _src, uint8_t* const _dest) { const size_t vectorized_chunk_size = bytesoftype * sizeof(__m128i); /* If the blocksize is not a multiple of both the typesize and the vector size, round the blocksize down to the next value which is a multiple of both. The vectorized shuffle can be used for that portion of the data, and the naive implementation can be used for the remaining portion. */ const size_t vectorizable_bytes = blocksize - (blocksize % vectorized_chunk_size); const size_t vectorizable_elements = vectorizable_bytes / bytesoftype; const size_t total_elements = blocksize / bytesoftype; /* If the block size is too small to be vectorized, use the generic implementation. */ if (blocksize < vectorized_chunk_size) { shuffle_generic(bytesoftype, blocksize, _src, _dest); return; } /* Optimized shuffle implementations */ switch (bytesoftype) { case 2: shuffle2_sse2(_dest, _src, vectorizable_elements, total_elements); break; case 4: shuffle4_sse2(_dest, _src, vectorizable_elements, total_elements); break; case 8: shuffle8_sse2(_dest, _src, vectorizable_elements, total_elements); break; case 16: shuffle16_sse2(_dest, _src, vectorizable_elements, total_elements); break; default: if (bytesoftype > sizeof(__m128i)) { shuffle16_tiled_sse2(_dest, _src, vectorizable_elements, total_elements, bytesoftype); } else { /* Non-optimized shuffle */ shuffle_generic(bytesoftype, blocksize, _src, _dest); /* The non-optimized function covers the whole buffer, so we're done processing here. */ return; } } /* If the buffer had any bytes at the end which couldn't be handled by the vectorized implementations, use the non-optimized version to finish them up. */ if (vectorizable_bytes < blocksize) { shuffle_generic_inline(bytesoftype, vectorizable_bytes, blocksize, _src, _dest); } } /* Unshuffle a block. This can never fail. */ void unshuffle_sse2(const size_t bytesoftype, const size_t blocksize, const uint8_t* const _src, uint8_t* const _dest) { const size_t vectorized_chunk_size = bytesoftype * sizeof(__m128i); /* If the blocksize is not a multiple of both the typesize and the vector size, round the blocksize down to the next value which is a multiple of both. The vectorized unshuffle can be used for that portion of the data, and the naive implementation can be used for the remaining portion. */ const size_t vectorizable_bytes = blocksize - (blocksize % vectorized_chunk_size); const size_t vectorizable_elements = vectorizable_bytes / bytesoftype; const size_t total_elements = blocksize / bytesoftype; /* If the block size is too small to be vectorized, use the generic implementation. */ if (blocksize < vectorized_chunk_size) { unshuffle_generic(bytesoftype, blocksize, _src, _dest); return; } /* Optimized unshuffle implementations */ switch (bytesoftype) { case 2: unshuffle2_sse2(_dest, _src, vectorizable_elements, total_elements); break; case 4: unshuffle4_sse2(_dest, _src, vectorizable_elements, total_elements); break; case 8: unshuffle8_sse2(_dest, _src, vectorizable_elements, total_elements); break; case 16: unshuffle16_sse2(_dest, _src, vectorizable_elements, total_elements); break; default: if (bytesoftype > sizeof(__m128i)) { unshuffle16_tiled_sse2(_dest, _src, vectorizable_elements, total_elements, bytesoftype); } else { /* Non-optimized unshuffle */ unshuffle_generic(bytesoftype, blocksize, _src, _dest); /* The non-optimized function covers the whole buffer, so we're done processing here. */ return; } } /* If the buffer had any bytes at the end which couldn't be handled by the vectorized implementations, use the non-optimized version to finish them up. */ if (vectorizable_bytes < blocksize) { unshuffle_generic_inline(bytesoftype, vectorizable_bytes, blocksize, _src, _dest); } } c-blosc-1.7.0/blosc/shuffle-sse2.h000066400000000000000000000017201254622165100166260ustar00rootroot00000000000000/********************************************************************* Blosc - Blocked Shuffling and Compression Library Author: Francesc Alted See LICENSES/BLOSC.txt for details about copyright and rights to use. **********************************************************************/ /* SSE2-accelerated shuffle/unshuffle routines. */ #ifndef SHUFFLE_SSE2_H #define SHUFFLE_SSE2_H #include "shuffle-common.h" #ifdef __cplusplus extern "C" { #endif /** SSE2-accelerated shuffle routine. */ BLOSC_NO_EXPORT void shuffle_sse2(const size_t bytesoftype, const size_t blocksize, const uint8_t* const _src, uint8_t* const _dest); /** SSE2-accelerated unshuffle routine. */ BLOSC_NO_EXPORT void unshuffle_sse2(const size_t bytesoftype, const size_t blocksize, const uint8_t* const _src, uint8_t* const _dest); #ifdef __cplusplus } #endif #endif /* SHUFFLE_SSE2_H */ c-blosc-1.7.0/blosc/shuffle.c000066400000000000000000000347771254622165100157710ustar00rootroot00000000000000/********************************************************************* Blosc - Blocked Shuffling and Compression Library Author: Francesc Alted Creation date: 2009-05-20 See LICENSES/BLOSC.txt for details about copyright and rights to use. **********************************************************************/ #include "shuffle.h" #include "shuffle-common.h" #include "shuffle-generic.h" #include "bitshuffle-generic.h" #include #include /* Visual Studio < 2013 does not have stdbool.h so here it is a replacement: */ #if defined __STDC__ && defined __STDC_VERSION__ && __STDC_VERSION__ >= 199901L /* have a C99 compiler */ typedef _Bool bool; #else /* do not have a C99 compiler */ typedef unsigned char bool; #endif static const bool false = 0; static const bool true = 1; #if !defined(__clang__) && defined(__GNUC__) && defined(__GNUC_MINOR__) && \ __GNUC__ >= 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8) #define HAVE_CPU_FEAT_INTRIN #endif /* Include hardware-accelerated shuffle/unshuffle routines based on the target architecture. Note that a target architecture may support more than one type of acceleration!*/ #if defined(SHUFFLE_AVX2_ENABLED) #include "shuffle-avx2.h" #include "bitshuffle-avx2.h" #endif /* defined(SHUFFLE_AVX2_ENABLED) */ #if defined(SHUFFLE_SSE2_ENABLED) #include "shuffle-sse2.h" #include "bitshuffle-sse2.h" #endif /* defined(SHUFFLE_SSE2_ENABLED) */ /* Define function pointer types for shuffle/unshuffle routines. */ typedef void(*shuffle_func)(const size_t, const size_t, const uint8_t*, const uint8_t*); typedef void(*unshuffle_func)(const size_t, const size_t, const uint8_t*, const uint8_t*); typedef int64_t(*bitshuffle_func)(void*, void*, const size_t, const size_t, void*); typedef int64_t(*bitunshuffle_func)(void*, void*, const size_t, const size_t, void*); /* An implementation of shuffle/unshuffle routines. */ typedef struct shuffle_implementation { /* Name of this implementation. */ const char* name; /* Function pointer to the shuffle routine for this implementation. */ shuffle_func shuffle; /* Function pointer to the unshuffle routine for this implementation. */ unshuffle_func unshuffle; /* Function pointer to the bitshuffle routine for this implementation. */ bitshuffle_func bitshuffle; /* Function pointer to the bitunshuffle routine for this implementation. */ bitunshuffle_func bitunshuffle; } shuffle_implementation_t; typedef enum { BLOSC_HAVE_NOTHING = 0, BLOSC_HAVE_SSE2 = 1, BLOSC_HAVE_AVX2 = 2 } blosc_cpu_features; /* Detect hardware and set function pointers to the best shuffle/unshuffle implementations supported by the host processor. */ #if defined(SHUFFLE_AVX2_ENABLED) || defined(SHUFFLE_SSE2_ENABLED) /* Intel/i686 */ #ifdef HAVE_CPU_FEAT_INTRIN static blosc_cpu_features blosc_get_cpu_features(void) { blosc_cpu_features cpu_features = BLOSC_HAVE_NOTHING; if (__builtin_cpu_supports("sse2")) { cpu_features |= BLOSC_HAVE_SSE2; } if (__builtin_cpu_supports("avx2")) { cpu_features |= BLOSC_HAVE_AVX2; } return cpu_features; } #else #if defined(_MSC_VER) && !defined(__clang__) #include /* Needed for _xgetbv */ #include /* Needed for __cpuid */ #else /* Implement the __cpuid and __cpuidex intrinsics for GCC, Clang, and others using inline assembly. */ __attribute__((always_inline)) static inline void __cpuidex(int32_t cpuInfo[4], int32_t function_id, int32_t subfunction_id) { __asm__ __volatile__ ( # if defined(__i386__) && defined (__PIC__) /* Can't clobber ebx with PIC running under 32-bit, so it needs to be manually restored. https://software.intel.com/en-us/articles/how-to-detect-new-instruction-support-in-the-4th-generation-intel-core-processor-family */ "movl %%ebx, %%edi\n\t" "cpuid\n\t" "xchgl %%ebx, %%edi": "=D" (cpuInfo[1]), #else "cpuid": "=b" (cpuInfo[1]), #endif /* defined(__i386) && defined(__PIC__) */ "=a" (cpuInfo[0]), "=c" (cpuInfo[2]), "=d" (cpuInfo[3]) : "a" (function_id), "c" (subfunction_id) ); } #define __cpuid(cpuInfo, function_id) __cpuidex(cpuInfo, function_id, 0) #define _XCR_XFEATURE_ENABLED_MASK 0 /* Reads the content of an extended control register. https://software.intel.com/en-us/articles/how-to-detect-new-instruction-support-in-the-4th-generation-intel-core-processor-family */ static inline uint64_t _xgetbv(uint32_t xcr) { uint32_t eax, edx; __asm__ __volatile__ ( /* "xgetbv" This is specified as raw instruction bytes due to some older compilers having issues with the mnemonic form. */ ".byte 0x0f, 0x01, 0xd0": "=a" (eax), "=d" (edx) : "c" (xcr) ); return ((uint64_t)edx << 32) | eax; } #endif /* defined(_MSC_VER) */ #ifndef _XCR_XFEATURE_ENABLED_MASK #define _XCR_XFEATURE_ENABLED_MASK 0x0 #endif static blosc_cpu_features blosc_get_cpu_features(void) { blosc_cpu_features result = BLOSC_HAVE_NOTHING; /* Holds the values of eax, ebx, ecx, edx set by the `cpuid` instruction */ int32_t cpu_info[4]; /* Get the number of basic functions available. */ __cpuid(cpu_info, 0); int32_t max_basic_function_id = cpu_info[0]; /* Check for SSE-based features and required OS support */ __cpuid(cpu_info, 1); const bool sse2_available = (cpu_info[3] & (1 << 26)) != 0; const bool sse3_available = (cpu_info[2] & (1 << 0)) != 0; const bool ssse3_available = (cpu_info[2] & (1 << 9)) != 0; const bool sse41_available = (cpu_info[2] & (1 << 19)) != 0; const bool sse42_available = (cpu_info[2] & (1 << 20)) != 0; const bool xsave_available = (cpu_info[2] & (1 << 26)) != 0; const bool xsave_enabled_by_os = (cpu_info[2] & (1 << 27)) != 0; /* Check for AVX-based features, if the processor supports extended features. */ bool avx2_available = false; bool avx512bw_available = false; if (max_basic_function_id >= 7) { __cpuid(cpu_info, 7); avx2_available = (cpu_info[1] & (1 << 5)) != 0; avx512bw_available = (cpu_info[1] & (1 << 30)) != 0; } /* Even if certain features are supported by the CPU, they may not be supported by the OS (in which case using them would crash the process or system). If xsave is available and enabled by the OS, check the contents of the extended control register XCR0 to see if the CPU features are enabled. */ bool xmm_state_enabled = false; bool ymm_state_enabled = false; bool zmm_state_enabled = false; #if defined(_XCR_XFEATURE_ENABLED_MASK) if (xsave_available && xsave_enabled_by_os && ( sse2_available || sse3_available || ssse3_available || sse41_available || sse42_available || avx2_available || avx512bw_available)) { /* Determine which register states can be restored by the OS. */ uint64_t xcr0_contents = _xgetbv(_XCR_XFEATURE_ENABLED_MASK); xmm_state_enabled = (xcr0_contents & (1UL << 1)) != 0; ymm_state_enabled = (xcr0_contents & (1UL << 2)) != 0; /* Require support for both the upper 256-bits of zmm0-zmm15 to be restored as well as all of zmm16-zmm31 and the opmask registers. */ zmm_state_enabled = (xcr0_contents & 0x70) == 0x70; } #endif /* defined(_XCR_XFEATURE_ENABLED_MASK) */ #if defined(BLOSC_DUMP_CPU_INFO) printf("Shuffle CPU Information:\n"); printf("SSE2 available: %s\n", sse2_available ? "True" : "False"); printf("SSE3 available: %s\n", sse3_available ? "True" : "False"); printf("SSSE3 available: %s\n", ssse3_available ? "True" : "False"); printf("SSE4.1 available: %s\n", sse41_available ? "True" : "False"); printf("SSE4.2 available: %s\n", sse42_available ? "True" : "False"); printf("AVX2 available: %s\n", avx2_available ? "True" : "False"); printf("AVX512BW available: %s\n", avx512bw_available ? "True" : "False"); printf("XSAVE available: %s\n", xsave_available ? "True" : "False"); printf("XSAVE enabled: %s\n", xsave_enabled_by_os ? "True" : "False"); printf("XMM state enabled: %s\n", xmm_state_enabled ? "True" : "False"); printf("YMM state enabled: %s\n", ymm_state_enabled ? "True" : "False"); printf("ZMM state enabled: %s\n", zmm_state_enabled ? "True" : "False"); #endif /* defined(BLOSC_DUMP_CPU_INFO) */ /* Using the gathered CPU information, determine which implementation to use. */ /* technically could fail on sse2 cpu on os without xmm support, but that * shouldn't exist anymore */ if (sse2_available) { result |= BLOSC_HAVE_SSE2; } if (xmm_state_enabled && ymm_state_enabled && avx2_available) { result |= BLOSC_HAVE_AVX2; } return result; } #endif #else /* No hardware acceleration supported for the target architecture. */ #if defined(_MSC_VER) #pragma message("Hardware-acceleration detection not implemented for the target architecture. Only the generic shuffle/unshuffle routines will be available.") #else #warning Hardware-acceleration detection not implemented for the target architecture. Only the generic shuffle/unshuffle routines will be available. #endif static blosc_cpu_features blosc_get_cpu_features(void) { return BLOSC_HAVE_NOTHING; } #endif static shuffle_implementation_t get_shuffle_implementation() { blosc_cpu_features cpu_features = blosc_get_cpu_features(); #if defined(SHUFFLE_AVX2_ENABLED) if (cpu_features & BLOSC_HAVE_AVX2) { shuffle_implementation_t impl_avx2; impl_avx2.name = "avx2"; impl_avx2.shuffle = (shuffle_func)shuffle_avx2; impl_avx2.unshuffle = (unshuffle_func)unshuffle_avx2; impl_avx2.bitshuffle = (bitshuffle_func)bshuf_trans_bit_elem_avx2; impl_avx2.bitunshuffle = (bitunshuffle_func)bshuf_untrans_bit_elem_avx2; return impl_avx2; } #endif /* defined(SHUFFLE_AVX2_ENABLED) */ #if defined(SHUFFLE_SSE2_ENABLED) if (cpu_features & BLOSC_HAVE_SSE2) { shuffle_implementation_t impl_sse2; impl_sse2.name = "sse2"; impl_sse2.shuffle = (shuffle_func)shuffle_sse2; impl_sse2.unshuffle = (unshuffle_func)unshuffle_sse2; impl_sse2.bitshuffle = (bitshuffle_func)bshuf_trans_bit_elem_sse2; impl_sse2.bitunshuffle = (bitunshuffle_func)bshuf_untrans_bit_elem_sse2; return impl_sse2; } #endif /* defined(SHUFFLE_SSE2_ENABLED) */ /* Processor doesn't support any of the hardware-accelerated implementations, so use the generic implementation. */ shuffle_implementation_t impl_generic; impl_generic.name = "generic"; impl_generic.shuffle = (shuffle_func)shuffle_generic; impl_generic.unshuffle = (unshuffle_func)unshuffle_generic; impl_generic.bitshuffle = (bitshuffle_func)bshuf_trans_bit_elem_scal; impl_generic.bitunshuffle = (bitunshuffle_func)bshuf_untrans_bit_elem_scal; return impl_generic; } /* Flag indicating whether the implementation has been initialized. Zero means it hasn't been initialized, non-zero means it has. */ static int32_t implementation_initialized; /* The dynamically-chosen shuffle/unshuffle implementation. This is only safe to use once `implementation_initialized` is set. */ static shuffle_implementation_t host_implementation; /* Initialize the shuffle implementation, if necessary. */ #if defined(__GNUC__) || defined(__clang__) __attribute__((always_inline)) #endif static #if defined(_MSC_VER) __forceinline #else inline #endif void init_shuffle_implementation() { /* Initialization could (in rare cases) take place concurrently on multiple threads, but it shouldn't matter because the initialization should return the same result on each thread (so the implementation will be the same). Since that's the case we can avoid complicated synchronization here and get a small performance benefit because we don't need to perform a volatile load on the initialization variable each time this function is called. */ #if defined(__GNUC__) || defined(__clang__) if (__builtin_expect(!implementation_initialized, 0)) { #else if (!implementation_initialized) { #endif /* Initialize the implementation. */ host_implementation = get_shuffle_implementation(); /* Set the flag indicating the implementation has been initialized. */ implementation_initialized = 1; } } /* Shuffle a block by dynamically dispatching to the appropriate hardware-accelerated routine at run-time. */ void shuffle(const size_t bytesoftype, const size_t blocksize, const uint8_t* _src, const uint8_t* _dest) { /* Initialize the shuffle implementation if necessary. */ init_shuffle_implementation(); /* The implementation is initialized. Dispatch to it's shuffle routine. */ (host_implementation.shuffle)(bytesoftype, blocksize, _src, _dest); } /* Unshuffle a block by dynamically dispatching to the appropriate hardware-accelerated routine at run-time. */ void unshuffle(const size_t bytesoftype, const size_t blocksize, const uint8_t* _src, const uint8_t* _dest) { /* Initialize the shuffle implementation if necessary. */ init_shuffle_implementation(); /* The implementation is initialized. Dispatch to it's unshuffle routine. */ (host_implementation.unshuffle)(bytesoftype, blocksize, _src, _dest); } /* Bit-shuffle a block by dynamically dispatching to the appropriate hardware-accelerated routine at run-time. */ int bitshuffle(const size_t bytesoftype, const size_t blocksize, const uint8_t* const _src, const uint8_t* _dest, const uint8_t* _tmp) { int size = blocksize / bytesoftype; /* Initialize the shuffle implementation if necessary. */ init_shuffle_implementation(); if ((size % 8) == 0) /* The number of elems is a multiple of 8 which is supported by bitshuffle. */ return (int)(host_implementation.bitshuffle)((void*)_src, (void*)_dest, blocksize / bytesoftype, bytesoftype, (void*)_tmp); else memcpy((void*)_dest, (void*)_src, blocksize); return size; } /* Bit-unshuffle a block by dynamically dispatching to the appropriate hardware-accelerated routine at run-time. */ int bitunshuffle(const size_t bytesoftype, const size_t blocksize, const uint8_t* const _src, const uint8_t* _dest, const uint8_t* _tmp) { int size = blocksize / bytesoftype; /* Initialize the shuffle implementation if necessary. */ init_shuffle_implementation(); if ((size % 8) == 0) /* The number of elems is a multiple of 8 which is supported by bitshuffle. */ return (int)(host_implementation.bitunshuffle)((void*)_src, (void*)_dest, blocksize / bytesoftype, bytesoftype, (void*)_tmp); else memcpy((void*)_dest, (void*)_src, blocksize); return size; } c-blosc-1.7.0/blosc/shuffle.h000066400000000000000000000046551254622165100157660ustar00rootroot00000000000000/********************************************************************* Blosc - Blocked Shuffling and Compression Library Author: Francesc Alted See LICENSES/BLOSC.txt for details about copyright and rights to use. **********************************************************************/ /* Shuffle/unshuffle routines which dynamically dispatch to hardware- accelerated routines based on the processor's architecture. Consumers should almost always prefer to call these routines instead of directly calling one of the hardware-accelerated routines, since these are cross-platform and future-proof. */ #ifndef SHUFFLE_H #define SHUFFLE_H #include "shuffle-common.h" #ifdef __cplusplus extern "C" { #endif /** Primary shuffle and bitshuffle routines. This function dynamically dispatches to the appropriate hardware-accelerated routine based on the host processor's architecture. If the host processor is not supported by any of the hardware-accelerated routines, the generic (non-accelerated) implementation is used instead. Consumers should almost always prefer to call this routine instead of directly calling the hardware-accelerated routines because this method is both cross- platform and future-proof. */ BLOSC_NO_EXPORT void shuffle(const size_t bytesoftype, const size_t blocksize, const uint8_t* _src, const uint8_t* _dest); BLOSC_NO_EXPORT int bitshuffle(const size_t bytesoftype, const size_t blocksize, const uint8_t* const _src, const uint8_t* _dest, const uint8_t* _tmp); /** Primary unshuffle and bitunshuffle routine. This function dynamically dispatches to the appropriate hardware-accelerated routine based on the host processor's architecture. If the host processor is not supported by any of the hardware-accelerated routines, the generic (non-accelerated) implementation is used instead. Consumers should almost always prefer to call this routine instead of directly calling the hardware-accelerated routines because this method is both cross- platform and future-proof. */ BLOSC_NO_EXPORT void unshuffle(const size_t bytesoftype, const size_t blocksize, const uint8_t* _src, const uint8_t* _dest); BLOSC_NO_EXPORT int bitunshuffle(const size_t bytesoftype, const size_t blocksize, const uint8_t* const _src, const uint8_t* _dest, const uint8_t* _tmp); #ifdef __cplusplus } #endif #endif /* SHUFFLE_H */ c-blosc-1.7.0/blosc/win32/000077500000000000000000000000001254622165100151115ustar00rootroot00000000000000c-blosc-1.7.0/blosc/win32/pthread.c000066400000000000000000000145471254622165100167170ustar00rootroot00000000000000/* * Code for simulating pthreads API on Windows. This is Git-specific, * but it is enough for Numexpr needs too. * * Copyright (C) 2009 Andrzej K. Haczewski * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * DISCLAIMER: The implementation is Git-specific, it is subset of original * Pthreads API, without lots of other features that Git doesn't use. * Git also makes sure that the passed arguments are valid, so there's * no need for double-checking. */ #include "pthread.h" #include #include #include #include #include void die(const char *err, ...) { printf("%s", err); exit(-1); } static unsigned __stdcall win32_start_routine(void *arg) { pthread_t *thread = (pthread_t*)arg; thread->arg = thread->start_routine(thread->arg); return 0; } int pthread_create(pthread_t *thread, const void *unused, void *(*start_routine)(void*), void *arg) { thread->arg = arg; thread->start_routine = start_routine; thread->handle = (HANDLE) _beginthreadex(NULL, 0, win32_start_routine, thread, 0, NULL); if (!thread->handle) return errno; else return 0; } int win32_pthread_join(pthread_t *thread, void **value_ptr) { DWORD result = WaitForSingleObject(thread->handle, INFINITE); switch (result) { case WAIT_OBJECT_0: if (value_ptr) *value_ptr = thread->arg; return 0; case WAIT_ABANDONED: return EINVAL; default: return GetLastError(); } } int pthread_cond_init(pthread_cond_t *cond, const void *unused) { cond->waiters = 0; cond->was_broadcast = 0; InitializeCriticalSection(&cond->waiters_lock); cond->sema = CreateSemaphore(NULL, 0, LONG_MAX, NULL); if (!cond->sema) die("CreateSemaphore() failed"); cond->continue_broadcast = CreateEvent(NULL, /* security */ FALSE, /* auto-reset */ FALSE, /* not signaled */ NULL); /* name */ if (!cond->continue_broadcast) die("CreateEvent() failed"); return 0; } int pthread_cond_destroy(pthread_cond_t *cond) { CloseHandle(cond->sema); CloseHandle(cond->continue_broadcast); DeleteCriticalSection(&cond->waiters_lock); return 0; } int pthread_cond_wait(pthread_cond_t *cond, CRITICAL_SECTION *mutex) { int last_waiter; EnterCriticalSection(&cond->waiters_lock); cond->waiters++; LeaveCriticalSection(&cond->waiters_lock); /* * Unlock external mutex and wait for signal. * NOTE: we've held mutex locked long enough to increment * waiters count above, so there's no problem with * leaving mutex unlocked before we wait on semaphore. */ LeaveCriticalSection(mutex); /* let's wait - ignore return value */ WaitForSingleObject(cond->sema, INFINITE); /* * Decrease waiters count. If we are the last waiter, then we must * notify the broadcasting thread that it can continue. * But if we continued due to cond_signal, we do not have to do that * because the signaling thread knows that only one waiter continued. */ EnterCriticalSection(&cond->waiters_lock); cond->waiters--; last_waiter = cond->was_broadcast && cond->waiters == 0; LeaveCriticalSection(&cond->waiters_lock); if (last_waiter) { /* * cond_broadcast was issued while mutex was held. This means * that all other waiters have continued, but are contending * for the mutex at the end of this function because the * broadcasting thread did not leave cond_broadcast, yet. * (This is so that it can be sure that each waiter has * consumed exactly one slice of the semaphor.) * The last waiter must tell the broadcasting thread that it * can go on. */ SetEvent(cond->continue_broadcast); /* * Now we go on to contend with all other waiters for * the mutex. Auf in den Kampf! */ } /* lock external mutex again */ EnterCriticalSection(mutex); return 0; } /* * IMPORTANT: This implementation requires that pthread_cond_signal * is called while the mutex is held that is used in the corresponding * pthread_cond_wait calls! */ int pthread_cond_signal(pthread_cond_t *cond) { int have_waiters; EnterCriticalSection(&cond->waiters_lock); have_waiters = cond->waiters > 0; LeaveCriticalSection(&cond->waiters_lock); /* * Signal only when there are waiters */ if (have_waiters) return ReleaseSemaphore(cond->sema, 1, NULL) ? 0 : GetLastError(); else return 0; } /* * DOUBLY IMPORTANT: This implementation requires that pthread_cond_broadcast * is called while the mutex is held that is used in the corresponding * pthread_cond_wait calls! */ int pthread_cond_broadcast(pthread_cond_t *cond) { EnterCriticalSection(&cond->waiters_lock); if ((cond->was_broadcast = cond->waiters > 0)) { /* wake up all waiters */ ReleaseSemaphore(cond->sema, cond->waiters, NULL); LeaveCriticalSection(&cond->waiters_lock); /* * At this point all waiters continue. Each one takes its * slice of the semaphor. Now it's our turn to wait: Since * the external mutex is held, no thread can leave cond_wait, * yet. For this reason, we can be sure that no thread gets * a chance to eat *more* than one slice. OTOH, it means * that the last waiter must send us a wake-up. */ WaitForSingleObject(cond->continue_broadcast, INFINITE); /* * Since the external mutex is held, no thread can enter * cond_wait, and, hence, it is safe to reset this flag * without cond->waiters_lock held. */ cond->was_broadcast = 0; } else { LeaveCriticalSection(&cond->waiters_lock); } return 0; } c-blosc-1.7.0/blosc/win32/pthread.h000066400000000000000000000062701254622165100167160ustar00rootroot00000000000000/* * Code for simulating pthreads API on Windows. This is Git-specific, * but it is enough for Numexpr needs too. * * Copyright (C) 2009 Andrzej K. Haczewski * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * DISCLAIMER: The implementation is Git-specific, it is subset of original * Pthreads API, without lots of other features that Git doesn't use. * Git also makes sure that the passed arguments are valid, so there's * no need for double-checking. */ #ifndef PTHREAD_H #define PTHREAD_H #ifndef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN #endif #include /* * Defines that adapt Windows API threads to pthreads API */ #define pthread_mutex_t CRITICAL_SECTION #define pthread_mutex_init(a,b) InitializeCriticalSection((a)) #define pthread_mutex_destroy(a) DeleteCriticalSection((a)) #define pthread_mutex_lock EnterCriticalSection #define pthread_mutex_unlock LeaveCriticalSection /* * Implement simple condition variable for Windows threads, based on ACE * implementation. * * See original implementation: http://bit.ly/1vkDjo * ACE homepage: http://www.cse.wustl.edu/~schmidt/ACE.html * See also: http://www.cse.wustl.edu/~schmidt/win32-cv-1.html */ typedef struct { LONG waiters; int was_broadcast; CRITICAL_SECTION waiters_lock; HANDLE sema; HANDLE continue_broadcast; } pthread_cond_t; extern int pthread_cond_init(pthread_cond_t *cond, const void *unused); extern int pthread_cond_destroy(pthread_cond_t *cond); extern int pthread_cond_wait(pthread_cond_t *cond, CRITICAL_SECTION *mutex); extern int pthread_cond_signal(pthread_cond_t *cond); extern int pthread_cond_broadcast(pthread_cond_t *cond); /* * Simple thread creation implementation using pthread API */ typedef struct { HANDLE handle; void *(*start_routine)(void*); void *arg; } pthread_t; extern int pthread_create(pthread_t *thread, const void *unused, void *(*start_routine)(void*), void *arg); /* * To avoid the need of copying a struct, we use small macro wrapper to pass * pointer to win32_pthread_join instead. */ #define pthread_join(a, b) win32_pthread_join(&(a), (b)) extern int win32_pthread_join(pthread_t *thread, void **value_ptr); #endif /* PTHREAD_H */ c-blosc-1.7.0/blosc/win32/stdint-windows.h000066400000000000000000000176451254622165100202740ustar00rootroot00000000000000// ISO C9x compliant stdint.h for Microsoft Visual Studio // Based on ISO/IEC 9899:TC2 Committee draft (May 6, 2005) WG14/N1124 // // Copyright (c) 2006-2013 Alexander Chemeris // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are met: // // 1. Redistributions of source code must retain the above copyright notice, // this list of conditions and the following disclaimer. // // 2. Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // // 3. Neither the name of the product nor the names of its contributors may // be used to endorse or promote products derived from this software // without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED // WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF // MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO // EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; // OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, // WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR // OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF // ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // /////////////////////////////////////////////////////////////////////////////// #ifndef _MSC_VER // [ #error "Use this header only with Microsoft Visual C++ compilers!" #endif // _MSC_VER ] #ifndef _MSC_STDINT_H_ // [ #define _MSC_STDINT_H_ #if _MSC_VER > 1000 #pragma once #endif #if _MSC_VER >= 1600 // [ #include #else // ] _MSC_VER >= 1600 [ #include // For Visual Studio 6 in C++ mode and for many Visual Studio versions when // compiling for ARM we should wrap include with 'extern "C++" {}' // or compiler give many errors like this: // error C2733: second C linkage of overloaded function 'wmemchr' not allowed #ifdef __cplusplus extern "C" { #endif # include #ifdef __cplusplus } #endif // Define _W64 macros to mark types changing their size, like intptr_t. #ifndef _W64 # if !defined(__midl) && (defined(_X86_) || defined(_M_IX86)) && _MSC_VER >= 1300 # define _W64 __w64 # else # define _W64 # endif #endif // 7.18.1 Integer types // 7.18.1.1 Exact-width integer types // Visual Studio 6 and Embedded Visual C++ 4 doesn't // realize that, e.g. char has the same size as __int8 // so we give up on __intX for them. #if (_MSC_VER < 1300) typedef signed char int8_t; typedef signed short int16_t; typedef signed int int32_t; typedef unsigned char uint8_t; typedef unsigned short uint16_t; typedef unsigned int uint32_t; #else typedef signed __int8 int8_t; typedef signed __int16 int16_t; typedef signed __int32 int32_t; typedef unsigned __int8 uint8_t; typedef unsigned __int16 uint16_t; typedef unsigned __int32 uint32_t; #endif typedef signed __int64 int64_t; typedef unsigned __int64 uint64_t; // 7.18.1.2 Minimum-width integer types typedef int8_t int_least8_t; typedef int16_t int_least16_t; typedef int32_t int_least32_t; typedef int64_t int_least64_t; typedef uint8_t uint_least8_t; typedef uint16_t uint_least16_t; typedef uint32_t uint_least32_t; typedef uint64_t uint_least64_t; // 7.18.1.3 Fastest minimum-width integer types typedef int8_t int_fast8_t; typedef int16_t int_fast16_t; typedef int32_t int_fast32_t; typedef int64_t int_fast64_t; typedef uint8_t uint_fast8_t; typedef uint16_t uint_fast16_t; typedef uint32_t uint_fast32_t; typedef uint64_t uint_fast64_t; // 7.18.1.4 Integer types capable of holding object pointers #ifdef _WIN64 // [ typedef signed __int64 intptr_t; typedef unsigned __int64 uintptr_t; #else // _WIN64 ][ typedef _W64 signed int intptr_t; typedef _W64 unsigned int uintptr_t; #endif // _WIN64 ] // 7.18.1.5 Greatest-width integer types typedef int64_t intmax_t; typedef uint64_t uintmax_t; // 7.18.2 Limits of specified-width integer types #if !defined(__cplusplus) || defined(__STDC_LIMIT_MACROS) // [ See footnote 220 at page 257 and footnote 221 at page 259 // 7.18.2.1 Limits of exact-width integer types #define INT8_MIN ((int8_t)_I8_MIN) #define INT8_MAX _I8_MAX #define INT16_MIN ((int16_t)_I16_MIN) #define INT16_MAX _I16_MAX #define INT32_MIN ((int32_t)_I32_MIN) #define INT32_MAX _I32_MAX #define INT64_MIN ((int64_t)_I64_MIN) #define INT64_MAX _I64_MAX #define UINT8_MAX _UI8_MAX #define UINT16_MAX _UI16_MAX #define UINT32_MAX _UI32_MAX #define UINT64_MAX _UI64_MAX // 7.18.2.2 Limits of minimum-width integer types #define INT_LEAST8_MIN INT8_MIN #define INT_LEAST8_MAX INT8_MAX #define INT_LEAST16_MIN INT16_MIN #define INT_LEAST16_MAX INT16_MAX #define INT_LEAST32_MIN INT32_MIN #define INT_LEAST32_MAX INT32_MAX #define INT_LEAST64_MIN INT64_MIN #define INT_LEAST64_MAX INT64_MAX #define UINT_LEAST8_MAX UINT8_MAX #define UINT_LEAST16_MAX UINT16_MAX #define UINT_LEAST32_MAX UINT32_MAX #define UINT_LEAST64_MAX UINT64_MAX // 7.18.2.3 Limits of fastest minimum-width integer types #define INT_FAST8_MIN INT8_MIN #define INT_FAST8_MAX INT8_MAX #define INT_FAST16_MIN INT16_MIN #define INT_FAST16_MAX INT16_MAX #define INT_FAST32_MIN INT32_MIN #define INT_FAST32_MAX INT32_MAX #define INT_FAST64_MIN INT64_MIN #define INT_FAST64_MAX INT64_MAX #define UINT_FAST8_MAX UINT8_MAX #define UINT_FAST16_MAX UINT16_MAX #define UINT_FAST32_MAX UINT32_MAX #define UINT_FAST64_MAX UINT64_MAX // 7.18.2.4 Limits of integer types capable of holding object pointers #ifdef _WIN64 // [ # define INTPTR_MIN INT64_MIN # define INTPTR_MAX INT64_MAX # define UINTPTR_MAX UINT64_MAX #else // _WIN64 ][ # define INTPTR_MIN INT32_MIN # define INTPTR_MAX INT32_MAX # define UINTPTR_MAX UINT32_MAX #endif // _WIN64 ] // 7.18.2.5 Limits of greatest-width integer types #define INTMAX_MIN INT64_MIN #define INTMAX_MAX INT64_MAX #define UINTMAX_MAX UINT64_MAX // 7.18.3 Limits of other integer types #ifdef _WIN64 // [ # define PTRDIFF_MIN _I64_MIN # define PTRDIFF_MAX _I64_MAX #else // _WIN64 ][ # define PTRDIFF_MIN _I32_MIN # define PTRDIFF_MAX _I32_MAX #endif // _WIN64 ] #define SIG_ATOMIC_MIN INT_MIN #define SIG_ATOMIC_MAX INT_MAX #ifndef SIZE_MAX // [ # ifdef _WIN64 // [ # define SIZE_MAX _UI64_MAX # else // _WIN64 ][ # define SIZE_MAX _UI32_MAX # endif // _WIN64 ] #endif // SIZE_MAX ] // WCHAR_MIN and WCHAR_MAX are also defined in #ifndef WCHAR_MIN // [ # define WCHAR_MIN 0 #endif // WCHAR_MIN ] #ifndef WCHAR_MAX // [ # define WCHAR_MAX _UI16_MAX #endif // WCHAR_MAX ] #define WINT_MIN 0 #define WINT_MAX _UI16_MAX #endif // __STDC_LIMIT_MACROS ] // 7.18.4 Limits of other integer types #if !defined(__cplusplus) || defined(__STDC_CONSTANT_MACROS) // [ See footnote 224 at page 260 // 7.18.4.1 Macros for minimum-width integer constants #define INT8_C(val) val##i8 #define INT16_C(val) val##i16 #define INT32_C(val) val##i32 #define INT64_C(val) val##i64 #define UINT8_C(val) val##ui8 #define UINT16_C(val) val##ui16 #define UINT32_C(val) val##ui32 #define UINT64_C(val) val##ui64 // 7.18.4.2 Macros for greatest-width integer constants // These #ifndef's are needed to prevent collisions with . // Check out Issue 9 for the details. #ifndef INTMAX_C // [ # define INTMAX_C INT64_C #endif // INTMAX_C ] #ifndef UINTMAX_C // [ # define UINTMAX_C UINT64_C #endif // UINTMAX_C ] #endif // __STDC_CONSTANT_MACROS ] #endif // _MSC_VER >= 1600 ] #endif // _MSC_STDINT_H_ ] c-blosc-1.7.0/cmake/000077500000000000000000000000001254622165100141255ustar00rootroot00000000000000c-blosc-1.7.0/cmake/FindLZ4.cmake000066400000000000000000000004121254622165100163360ustar00rootroot00000000000000find_path(LZ4_INCLUDE_DIR lz4.h) find_library(LZ4_LIBRARY NAMES lz4) if (LZ4_INCLUDE_DIR AND LZ4_LIBRARY) set(LZ4_FOUND TRUE) message(STATUS "Found LZ4 library: ${LZ4_LIBRARY}") else () message(STATUS "No lz4 found. Using internal sources.") endif () c-blosc-1.7.0/cmake/FindSnappy.cmake000066400000000000000000000004521254622165100172030ustar00rootroot00000000000000find_path(SNAPPY_INCLUDE_DIR snappy-c.h) find_library(SNAPPY_LIBRARY NAMES snappy) if (SNAPPY_INCLUDE_DIR AND SNAPPY_LIBRARY) set(SNAPPY_FOUND TRUE) message(STATUS "Found SNAPPY library: ${SNAPPY_LIBRARY}") else () message(STATUS "No snappy found. Using internal sources.") endif () c-blosc-1.7.0/cmake_uninstall.cmake.in000066400000000000000000000017631254622165100176340ustar00rootroot00000000000000if (NOT EXISTS "@CMAKE_CURRENT_BINARY_DIR@/install_manifest.txt") message(FATAL_ERROR "Cannot find install manifest: \"@CMAKE_CURRENT_BINARY_DIR@/install_manifest.txt\"") endif(NOT EXISTS "@CMAKE_CURRENT_BINARY_DIR@/install_manifest.txt") file(READ "@CMAKE_CURRENT_BINARY_DIR@/install_manifest.txt" files) string(REGEX REPLACE "\n" ";" files "${files}") list(REVERSE files) foreach (file ${files}) message(STATUS "Uninstalling \"$ENV{DESTDIR}${file}\"") if (EXISTS "$ENV{DESTDIR}${file}") execute_process( COMMAND @CMAKE_COMMAND@ -E remove "$ENV{DESTDIR}${file}" OUTPUT_VARIABLE rm_out RESULT_VARIABLE rm_retval ) if(NOT ${rm_retval} EQUAL 0) message(FATAL_ERROR "Problem when removing \"$ENV{DESTDIR}${file}\"") endif (NOT ${rm_retval} EQUAL 0) else (EXISTS "$ENV{DESTDIR}${file}") message(STATUS "File \"$ENV{DESTDIR}${file}\" does not exist.") endif (EXISTS "$ENV{DESTDIR}${file}") endforeach(file) c-blosc-1.7.0/examples/000077500000000000000000000000001254622165100146635ustar00rootroot00000000000000c-blosc-1.7.0/examples/README.rst000066400000000000000000000010011254622165100163420ustar00rootroot00000000000000Examples on how to add Blosc support for your programs ====================================================== In this directory you can find a series of examples on how to link your apps with the Blosc library: * simple.c -- The simplest way to add Blosc to your app * multithread.c -- Add multithreading into the equation * many_compressors.c -- Use different compressors inside Blosc For more info, please visit the `official API documentation `_. c-blosc-1.7.0/examples/many_compressors.c000066400000000000000000000061471254622165100204420ustar00rootroot00000000000000/* Copyright (C) 2014 Francesc Alted http://blosc.org License: MIT (see LICENSE.txt) Example program demonstrating use of the Blosc filter from C code. To compile this program: gcc many_compressors.c -o many_compressors -lblosc -lpthread or, if you don't have the blosc library installed: gcc -O3 -msse2 many_compressors.c ../blosc/*.c -I../blosc \ -o many_compressors -lpthread \ -DHAVE_ZLIB -lz -DHAVE_LZ4 -llz4 -DHAVE_SNAPPY -lsnappy Using MSVC on Windows: cl /Ox /Femany_compressors.exe /Iblosc many_compressors.c blosc\*.c To run: ./many_compressors Blosc version info: 1.4.2.dev ($Date:: 2014-07-08 #$) Using 4 threads (previously using 1) Using blosclz compressor Compression: 4000000 -> 158494 (25.2x) Succesful roundtrip! Using lz4 compressor Compression: 4000000 -> 234238 (17.1x) Succesful roundtrip! Using lz4hc compressor Compression: 4000000 -> 38314 (104.4x) Succesful roundtrip! Using snappy compressor Compression: 4000000 -> 311617 (12.8x) Succesful roundtrip! Using zlib compressor Compression: 4000000 -> 22103 (181.0x) Succesful roundtrip! */ #include #include #define SIZE 100*100*100 #define SHAPE {100,100,100} #define CHUNKSHAPE {1,100,100} int main(){ static float data[SIZE]; static float data_out[SIZE]; static float data_dest[SIZE]; int isize = SIZE*sizeof(float), osize = SIZE*sizeof(float); int dsize = SIZE*sizeof(float), csize; int nthreads, pnthreads, i; char* compressors[] = {"blosclz", "lz4", "lz4hc", "snappy", "zlib"}; int ccode, rcode; for(i=0; i %d (%.1fx)\n", isize, csize, (1.*isize) / csize); /* Decompress */ dsize = blosc_decompress(data_out, data_dest, dsize); if (dsize < 0) { printf("Decompression error. Error code: %d\n", dsize); return dsize; } /* After using it, destroy the Blosc environment */ blosc_destroy(); for(i=0;i 158494 (25.2x) Succesful roundtrip! Using 2 threads (previously using 1) Compression: 4000000 -> 158494 (25.2x) Succesful roundtrip! Using 3 threads (previously using 2) Compression: 4000000 -> 158494 (25.2x) Succesful roundtrip! Using 4 threads (previously using 3) Compression: 4000000 -> 158494 (25.2x) Succesful roundtrip! */ #include #include #define SIZE 1000*1000 int main(){ static float data[SIZE]; static float data_out[SIZE]; static float data_dest[SIZE]; int isize = SIZE*sizeof(float), osize = SIZE*sizeof(float); int dsize = SIZE*sizeof(float), csize; int nthreads, pnthreads, i; for(i=0; i %d (%.1fx)\n", isize, csize, (1.*isize) / csize); /* Decompress */ dsize = blosc_decompress(data_out, data_dest, dsize); if (dsize < 0) { printf("Decompression error. Error code: %d\n", dsize); return dsize; } /* After using it, destroy the Blosc environment */ blosc_destroy(); for(i=0;i 158494 (25.2x) Decompression succesful! Succesful roundtrip! */ #include #include #define SIZE 100*100*100 #define SHAPE {100,100,100} #define CHUNKSHAPE {1,100,100} int main(){ static float data[SIZE]; static float data_out[SIZE]; static float data_dest[SIZE]; int isize = SIZE*sizeof(float), osize = SIZE*sizeof(float); int dsize = SIZE*sizeof(float), csize; int i; for(i=0; i %d (%.1fx)\n", isize, csize, (1.*isize) / csize); /* Decompress */ dsize = blosc_decompress(data_out, data_dest, dsize); if (dsize < 0) { printf("Decompression error. Error code: %d\n", dsize); return dsize; } printf("Decompression succesful!\n"); /* After using it, destroy the Blosc environment */ blosc_destroy(); for(i=0;i 19928862 (20.1x) Decompression succesful! Succesful roundtrip! */ #include #include #include #define SIZE 100*1000*1000 #define SHAPE {100,1000,1000} #define CHUNKSHAPE {1,1000,1000} /* Definition for the compression and decompression blosc routines */ typedef int (__cdecl *COMPRESS_CTX)(int clevel, int doshuffle, size_t typesize, size_t nbytes, const void* src, void* dest, size_t destsize, const char* compressor, size_t blocksize, int numinternalthreads); typedef int (__cdecl *DECOMPRESS_CTX)(const void *src, void *dest, size_t destsize, int numinternalthreads); typedef char* (__cdecl *GET_VERSION_STRING)(void); int main(){ HINSTANCE BDLL; /* Handle to DLL */ COMPRESS_CTX blosc_compress_ctx; /* Function pointer for compression */ DECOMPRESS_CTX blosc_decompress_ctx; /* Function pointer for decompression */ GET_VERSION_STRING blosc_get_version_string; static float data[SIZE]; static float data_out[SIZE]; static float data_dest[SIZE]; int isize = SIZE*sizeof(float), osize = SIZE*sizeof(float); int dsize = SIZE*sizeof(float), csize; int i; BDLL = LoadLibrary(TEXT("myblosc.dll")); if (BDLL == NULL) { printf("Cannot find myblosc.dll library!\n"); goto out; } blosc_compress_ctx = (COMPRESS_CTX)GetProcAddress(BDLL, "blosc_compress_ctx"); if (!blosc_compress_ctx) { // handle the error printf("Cannot find blosc_compress_ctx() function!\n"); goto out; } blosc_decompress_ctx = (DECOMPRESS_CTX)GetProcAddress(BDLL, "blosc_decompress_ctx"); if (!blosc_decompress_ctx) { // handle the error printf("Cannot find blosc_decompress_ctx() function!\n"); goto out; } blosc_get_version_string = (GET_VERSION_STRING)GetProcAddress(BDLL, "blosc_get_version_string"); if (!blosc_get_version_string) { // handle the error printf("Cannot find blosc_get_version_string() function!\n"); goto out; } for(i=0; i %d (%.1fx)\n", isize, csize, (1.*isize) / csize); /* Decompress */ dsize = blosc_decompress_ctx(data_out, data_dest, dsize, 1); if (dsize < 0) { printf("Decompression error. Error code: %d\n", dsize); return dsize; } printf("Decompression succesful!\n"); for(i=0;i # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ # pragma warning(disable : 4293) /* disable: C4293: too large shift (32-bits) */ #else # if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ # if defined(__GNUC__) || defined(__clang__) # define FORCE_INLINE static inline __attribute__((always_inline)) # else # define FORCE_INLINE static inline # endif # else # define FORCE_INLINE static # endif /* __STDC_VERSION__ */ #endif /* _MSC_VER */ /* LZ4_GCC_VERSION is defined into lz4.h */ #if (LZ4_GCC_VERSION >= 302) || (__INTEL_COMPILER >= 800) || defined(__clang__) # define expect(expr,value) (__builtin_expect ((expr),(value)) ) #else # define expect(expr,value) (expr) #endif #define likely(expr) expect((expr) != 0, 1) #define unlikely(expr) expect((expr) != 0, 0) /************************************** * Memory routines **************************************/ #include /* malloc, calloc, free */ #define ALLOCATOR(n,s) calloc(n,s) #define FREEMEM free #include /* memset, memcpy */ #define MEM_INIT memset /************************************** * Basic Types **************************************/ #if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ # include typedef uint8_t BYTE; typedef uint16_t U16; typedef uint32_t U32; typedef int32_t S32; typedef uint64_t U64; #else typedef unsigned char BYTE; typedef unsigned short U16; typedef unsigned int U32; typedef signed int S32; typedef unsigned long long U64; #endif /************************************** * Reading and writing into memory **************************************/ #define STEPSIZE sizeof(size_t) static unsigned LZ4_64bits(void) { return sizeof(void*)==8; } static unsigned LZ4_isLittleEndian(void) { const union { U32 i; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ return one.c[0]; } static U16 LZ4_read16(const void* memPtr) { U16 val16; memcpy(&val16, memPtr, 2); return val16; } static U16 LZ4_readLE16(const void* memPtr) { if (LZ4_isLittleEndian()) { return LZ4_read16(memPtr); } else { const BYTE* p = (const BYTE*)memPtr; return (U16)((U16)p[0] + (p[1]<<8)); } } static void LZ4_writeLE16(void* memPtr, U16 value) { if (LZ4_isLittleEndian()) { memcpy(memPtr, &value, 2); } else { BYTE* p = (BYTE*)memPtr; p[0] = (BYTE) value; p[1] = (BYTE)(value>>8); } } static U32 LZ4_read32(const void* memPtr) { U32 val32; memcpy(&val32, memPtr, 4); return val32; } static U64 LZ4_read64(const void* memPtr) { U64 val64; memcpy(&val64, memPtr, 8); return val64; } static size_t LZ4_read_ARCH(const void* p) { if (LZ4_64bits()) return (size_t)LZ4_read64(p); else return (size_t)LZ4_read32(p); } static void LZ4_copy4(void* dstPtr, const void* srcPtr) { memcpy(dstPtr, srcPtr, 4); } static void LZ4_copy8(void* dstPtr, const void* srcPtr) { memcpy(dstPtr, srcPtr, 8); } /* customized version of memcpy, which may overwrite up to 7 bytes beyond dstEnd */ static void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd) { BYTE* d = (BYTE*)dstPtr; const BYTE* s = (const BYTE*)srcPtr; BYTE* e = (BYTE*)dstEnd; do { LZ4_copy8(d,s); d+=8; s+=8; } while (d>3); # elif (defined(__clang__) || (LZ4_GCC_VERSION >= 304)) && !defined(LZ4_FORCE_SW_BITCOUNT) return (__builtin_ctzll((U64)val) >> 3); # else static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; # endif } else /* 32 bits */ { # if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) unsigned long r; _BitScanForward( &r, (U32)val ); return (int)(r>>3); # elif (defined(__clang__) || (LZ4_GCC_VERSION >= 304)) && !defined(LZ4_FORCE_SW_BITCOUNT) return (__builtin_ctz((U32)val) >> 3); # else static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; # endif } } else /* Big Endian CPU */ { if (LZ4_64bits()) { # if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT) unsigned long r = 0; _BitScanReverse64( &r, val ); return (unsigned)(r>>3); # elif (defined(__clang__) || (LZ4_GCC_VERSION >= 304)) && !defined(LZ4_FORCE_SW_BITCOUNT) return (__builtin_clzll((U64)val) >> 3); # else unsigned r; if (!(val>>32)) { r=4; } else { r=0; val>>=32; } if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } r += (!val); return r; # endif } else /* 32 bits */ { # if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) unsigned long r = 0; _BitScanReverse( &r, (unsigned long)val ); return (unsigned)(r>>3); # elif (defined(__clang__) || (LZ4_GCC_VERSION >= 304)) && !defined(LZ4_FORCE_SW_BITCOUNT) return (__builtin_clz((U32)val) >> 3); # else unsigned r; if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } r += (!val); return r; # endif } } } static unsigned LZ4_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit) { const BYTE* const pStart = pIn; while (likely(pIn compression run slower on incompressible data */ /************************************** * Local Structures and types **************************************/ typedef struct { U32 hashTable[HASH_SIZE_U32]; U32 currentOffset; U32 initCheck; const BYTE* dictionary; BYTE* bufferStart; /* obsolete, used for slideInputBuffer */ U32 dictSize; } LZ4_stream_t_internal; typedef enum { notLimited = 0, limitedOutput = 1 } limitedOutput_directive; typedef enum { byPtr, byU32, byU16 } tableType_t; typedef enum { noDict = 0, withPrefix64k, usingExtDict } dict_directive; typedef enum { noDictIssue = 0, dictSmall } dictIssue_directive; typedef enum { endOnOutputSize = 0, endOnInputSize = 1 } endCondition_directive; typedef enum { full = 0, partial = 1 } earlyEnd_directive; /************************************** * Local Utils **************************************/ int LZ4_versionNumber (void) { return LZ4_VERSION_NUMBER; } int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); } int LZ4_sizeofState() { return LZ4_STREAMSIZE; } /******************************** * Compression functions ********************************/ static U32 LZ4_hashSequence(U32 sequence, tableType_t const tableType) { if (tableType == byU16) return (((sequence) * 2654435761U) >> ((MINMATCH*8)-(LZ4_HASHLOG+1))); else return (((sequence) * 2654435761U) >> ((MINMATCH*8)-LZ4_HASHLOG)); } static const U64 prime5bytes = 889523592379ULL; static U32 LZ4_hashSequence64(size_t sequence, tableType_t const tableType) { const U32 hashLog = (tableType == byU16) ? LZ4_HASHLOG+1 : LZ4_HASHLOG; const U32 hashMask = (1<> (40 - hashLog)) & hashMask; } static U32 LZ4_hashSequenceT(size_t sequence, tableType_t const tableType) { if (LZ4_64bits()) return LZ4_hashSequence64(sequence, tableType); return LZ4_hashSequence((U32)sequence, tableType); } static U32 LZ4_hashPosition(const void* p, tableType_t tableType) { return LZ4_hashSequenceT(LZ4_read_ARCH(p), tableType); } static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableType_t const tableType, const BYTE* srcBase) { switch (tableType) { case byPtr: { const BYTE** hashTable = (const BYTE**)tableBase; hashTable[h] = p; return; } case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = (U32)(p-srcBase); return; } case byU16: { U16* hashTable = (U16*) tableBase; hashTable[h] = (U16)(p-srcBase); return; } } } static void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) { U32 h = LZ4_hashPosition(p, tableType); LZ4_putPositionOnHash(p, h, tableBase, tableType, srcBase); } static const BYTE* LZ4_getPositionOnHash(U32 h, void* tableBase, tableType_t tableType, const BYTE* srcBase) { if (tableType == byPtr) { const BYTE** hashTable = (const BYTE**) tableBase; return hashTable[h]; } if (tableType == byU32) { U32* hashTable = (U32*) tableBase; return hashTable[h] + srcBase; } { U16* hashTable = (U16*) tableBase; return hashTable[h] + srcBase; } /* default, to ensure a return */ } static const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) { U32 h = LZ4_hashPosition(p, tableType); return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase); } FORCE_INLINE int LZ4_compress_generic( void* const ctx, const char* const source, char* const dest, const int inputSize, const int maxOutputSize, const limitedOutput_directive outputLimited, const tableType_t tableType, const dict_directive dict, const dictIssue_directive dictIssue, const U32 acceleration) { LZ4_stream_t_internal* const dictPtr = (LZ4_stream_t_internal*)ctx; const BYTE* ip = (const BYTE*) source; const BYTE* base; const BYTE* lowLimit; const BYTE* const lowRefLimit = ip - dictPtr->dictSize; const BYTE* const dictionary = dictPtr->dictionary; const BYTE* const dictEnd = dictionary + dictPtr->dictSize; const size_t dictDelta = dictEnd - (const BYTE*)source; const BYTE* anchor = (const BYTE*) source; const BYTE* const iend = ip + inputSize; const BYTE* const mflimit = iend - MFLIMIT; const BYTE* const matchlimit = iend - LASTLITERALS; BYTE* op = (BYTE*) dest; BYTE* const olimit = op + maxOutputSize; U32 forwardH; size_t refDelta=0; /* Init conditions */ if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size, too large (or negative) */ switch(dict) { case noDict: default: base = (const BYTE*)source; lowLimit = (const BYTE*)source; break; case withPrefix64k: base = (const BYTE*)source - dictPtr->currentOffset; lowLimit = (const BYTE*)source - dictPtr->dictSize; break; case usingExtDict: base = (const BYTE*)source - dictPtr->currentOffset; lowLimit = (const BYTE*)source; break; } if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0; /* Size too large (not within 64K limit) */ if (inputSize> LZ4_skipTrigger); if (unlikely(forwardIp > mflimit)) goto _last_literals; match = LZ4_getPositionOnHash(h, ctx, tableType, base); if (dict==usingExtDict) { if (match<(const BYTE*)source) { refDelta = dictDelta; lowLimit = dictionary; } else { refDelta = 0; lowLimit = (const BYTE*)source; } } forwardH = LZ4_hashPosition(forwardIp, tableType); LZ4_putPositionOnHash(ip, h, ctx, tableType, base); } while ( ((dictIssue==dictSmall) ? (match < lowRefLimit) : 0) || ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip)) || (LZ4_read32(match+refDelta) != LZ4_read32(ip)) ); } /* Catch up */ while ((ip>anchor) && (match+refDelta > lowLimit) && (unlikely(ip[-1]==match[refDelta-1]))) { ip--; match--; } { /* Encode Literal length */ unsigned litLength = (unsigned)(ip - anchor); token = op++; if ((outputLimited) && (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit))) return 0; /* Check output limit */ if (litLength>=RUN_MASK) { int len = (int)litLength-RUN_MASK; *token=(RUN_MASK<= 255 ; len-=255) *op++ = 255; *op++ = (BYTE)len; } else *token = (BYTE)(litLength< matchlimit) limit = matchlimit; matchLength = LZ4_count(ip+MINMATCH, match+MINMATCH, limit); ip += MINMATCH + matchLength; if (ip==limit) { unsigned more = LZ4_count(ip, (const BYTE*)source, matchlimit); matchLength += more; ip += more; } } else { matchLength = LZ4_count(ip+MINMATCH, match+MINMATCH, matchlimit); ip += MINMATCH + matchLength; } if ((outputLimited) && (unlikely(op + (1 + LASTLITERALS) + (matchLength>>8) > olimit))) return 0; /* Check output limit */ if (matchLength>=ML_MASK) { *token += ML_MASK; matchLength -= ML_MASK; for (; matchLength >= 510 ; matchLength-=510) { *op++ = 255; *op++ = 255; } if (matchLength >= 255) { matchLength-=255; *op++ = 255; } *op++ = (BYTE)matchLength; } else *token += (BYTE)(matchLength); } anchor = ip; /* Test end of chunk */ if (ip > mflimit) break; /* Fill table */ LZ4_putPosition(ip-2, ctx, tableType, base); /* Test next position */ match = LZ4_getPosition(ip, ctx, tableType, base); if (dict==usingExtDict) { if (match<(const BYTE*)source) { refDelta = dictDelta; lowLimit = dictionary; } else { refDelta = 0; lowLimit = (const BYTE*)source; } } LZ4_putPosition(ip, ctx, tableType, base); if ( ((dictIssue==dictSmall) ? (match>=lowRefLimit) : 1) && (match+MAX_DISTANCE>=ip) && (LZ4_read32(match+refDelta)==LZ4_read32(ip)) ) { token=op++; *token=0; goto _next_match; } /* Prepare next loop */ forwardH = LZ4_hashPosition(++ip, tableType); } _last_literals: /* Encode Last Literals */ { const size_t lastRun = (size_t)(iend - anchor); if ((outputLimited) && ((op - (BYTE*)dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0; /* Check output limit */ if (lastRun >= RUN_MASK) { size_t accumulator = lastRun - RUN_MASK; *op++ = RUN_MASK << ML_BITS; for(; accumulator >= 255 ; accumulator-=255) *op++ = 255; *op++ = (BYTE) accumulator; } else { *op++ = (BYTE)(lastRun<= LZ4_compressBound(inputSize)) { if (inputSize < LZ4_64Klimit) return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue, acceleration); else return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, acceleration); } else { if (inputSize < LZ4_64Klimit) return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); else return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, acceleration); } } int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) { #if (HEAPMODE) void* ctxPtr = ALLOCATOR(1, sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ #else LZ4_stream_t ctx; void* ctxPtr = &ctx; #endif int result = LZ4_compress_fast_extState(ctxPtr, source, dest, inputSize, maxOutputSize, acceleration); #if (HEAPMODE) FREEMEM(ctxPtr); #endif return result; } int LZ4_compress_default(const char* source, char* dest, int inputSize, int maxOutputSize) { return LZ4_compress_fast(source, dest, inputSize, maxOutputSize, 1); } /* hidden debug function */ /* strangely enough, gcc generates faster code when this function is uncommented, even if unused */ int LZ4_compress_fast_force(const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) { LZ4_stream_t ctx; LZ4_resetStream(&ctx); if (inputSize < LZ4_64Klimit) return LZ4_compress_generic(&ctx, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); else return LZ4_compress_generic(&ctx, source, dest, inputSize, maxOutputSize, limitedOutput, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, acceleration); } /******************************** * destSize variant ********************************/ static int LZ4_compress_destSize_generic( void* const ctx, const char* const src, char* const dst, int* const srcSizePtr, const int targetDstSize, const tableType_t tableType) { const BYTE* ip = (const BYTE*) src; const BYTE* base = (const BYTE*) src; const BYTE* lowLimit = (const BYTE*) src; const BYTE* anchor = ip; const BYTE* const iend = ip + *srcSizePtr; const BYTE* const mflimit = iend - MFLIMIT; const BYTE* const matchlimit = iend - LASTLITERALS; BYTE* op = (BYTE*) dst; BYTE* const oend = op + targetDstSize; BYTE* const oMaxLit = op + targetDstSize - 2 /* offset */ - 8 /* because 8+MINMATCH==MFLIMIT */ - 1 /* token */; BYTE* const oMaxMatch = op + targetDstSize - (LASTLITERALS + 1 /* token */); BYTE* const oMaxSeq = oMaxLit - 1 /* token */; U32 forwardH; /* Init conditions */ if (targetDstSize < 1) return 0; /* Impossible to store anything */ if ((U32)*srcSizePtr > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size, too large (or negative) */ if ((tableType == byU16) && (*srcSizePtr>=LZ4_64Klimit)) return 0; /* Size too large (not within 64K limit) */ if (*srcSizePtr> LZ4_skipTrigger); if (unlikely(forwardIp > mflimit)) goto _last_literals; match = LZ4_getPositionOnHash(h, ctx, tableType, base); forwardH = LZ4_hashPosition(forwardIp, tableType); LZ4_putPositionOnHash(ip, h, ctx, tableType, base); } while ( ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip)) || (LZ4_read32(match) != LZ4_read32(ip)) ); } /* Catch up */ while ((ip>anchor) && (match > lowLimit) && (unlikely(ip[-1]==match[-1]))) { ip--; match--; } { /* Encode Literal length */ unsigned litLength = (unsigned)(ip - anchor); token = op++; if (op + ((litLength+240)/255) + litLength > oMaxLit) { /* Not enough space for a last match */ op--; goto _last_literals; } if (litLength>=RUN_MASK) { unsigned len = litLength - RUN_MASK; *token=(RUN_MASK<= 255 ; len-=255) *op++ = 255; *op++ = (BYTE)len; } else *token = (BYTE)(litLength< oMaxMatch) { /* Match description too long : reduce it */ matchLength = (15-1) + (oMaxMatch-op) * 255; } //printf("offset %5i, matchLength%5i \n", (int)(ip-match), matchLength + MINMATCH); ip += MINMATCH + matchLength; if (matchLength>=ML_MASK) { *token += ML_MASK; matchLength -= ML_MASK; while (matchLength >= 255) { matchLength-=255; *op++ = 255; } *op++ = (BYTE)matchLength; } else *token += (BYTE)(matchLength); } anchor = ip; /* Test end of block */ if (ip > mflimit) break; if (op > oMaxSeq) break; /* Fill table */ LZ4_putPosition(ip-2, ctx, tableType, base); /* Test next position */ match = LZ4_getPosition(ip, ctx, tableType, base); LZ4_putPosition(ip, ctx, tableType, base); if ( (match+MAX_DISTANCE>=ip) && (LZ4_read32(match)==LZ4_read32(ip)) ) { token=op++; *token=0; goto _next_match; } /* Prepare next loop */ forwardH = LZ4_hashPosition(++ip, tableType); } _last_literals: /* Encode Last Literals */ { size_t lastRunSize = (size_t)(iend - anchor); if (op + 1 /* token */ + ((lastRunSize+240)/255) /* litLength */ + lastRunSize /* literals */ > oend) { /* adapt lastRunSize to fill 'dst' */ lastRunSize = (oend-op) - 1; lastRunSize -= (lastRunSize+240)/255; } ip = anchor + lastRunSize; if (lastRunSize >= RUN_MASK) { size_t accumulator = lastRunSize - RUN_MASK; *op++ = RUN_MASK << ML_BITS; for(; accumulator >= 255 ; accumulator-=255) *op++ = 255; *op++ = (BYTE) accumulator; } else { *op++ = (BYTE)(lastRunSize<= LZ4_compressBound(*srcSizePtr)) /* compression success is guaranteed */ { return LZ4_compress_fast_extState(state, src, dst, *srcSizePtr, targetDstSize, 1); } else { if (*srcSizePtr < LZ4_64Klimit) return LZ4_compress_destSize_generic(state, src, dst, srcSizePtr, targetDstSize, byU16); else return LZ4_compress_destSize_generic(state, src, dst, srcSizePtr, targetDstSize, LZ4_64bits() ? byU32 : byPtr); } } int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targetDstSize) { #if (HEAPMODE) void* ctx = ALLOCATOR(1, sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ #else LZ4_stream_t ctxBody; void* ctx = &ctxBody; #endif int result = LZ4_compress_destSize_extState(ctx, src, dst, srcSizePtr, targetDstSize); #if (HEAPMODE) FREEMEM(ctx); #endif return result; } /******************************** * Streaming functions ********************************/ LZ4_stream_t* LZ4_createStream(void) { LZ4_stream_t* lz4s = (LZ4_stream_t*)ALLOCATOR(8, LZ4_STREAMSIZE_U64); LZ4_STATIC_ASSERT(LZ4_STREAMSIZE >= sizeof(LZ4_stream_t_internal)); /* A compilation error here means LZ4_STREAMSIZE is not large enough */ LZ4_resetStream(lz4s); return lz4s; } void LZ4_resetStream (LZ4_stream_t* LZ4_stream) { MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t)); } int LZ4_freeStream (LZ4_stream_t* LZ4_stream) { FREEMEM(LZ4_stream); return (0); } #define HASH_UNIT sizeof(size_t) int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize) { LZ4_stream_t_internal* dict = (LZ4_stream_t_internal*) LZ4_dict; const BYTE* p = (const BYTE*)dictionary; const BYTE* const dictEnd = p + dictSize; const BYTE* base; if ((dict->initCheck) || (dict->currentOffset > 1 GB)) /* Uninitialized structure, or reuse overflow */ LZ4_resetStream(LZ4_dict); if (dictSize < (int)HASH_UNIT) { dict->dictionary = NULL; dict->dictSize = 0; return 0; } if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB; dict->currentOffset += 64 KB; base = p - dict->currentOffset; dict->dictionary = p; dict->dictSize = (U32)(dictEnd - p); dict->currentOffset += dict->dictSize; while (p <= dictEnd-HASH_UNIT) { LZ4_putPosition(p, dict->hashTable, byU32, base); p+=3; } return dict->dictSize; } static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, const BYTE* src) { if ((LZ4_dict->currentOffset > 0x80000000) || ((size_t)LZ4_dict->currentOffset > (size_t)src)) /* address space overflow */ { /* rescale hash table */ U32 delta = LZ4_dict->currentOffset - 64 KB; const BYTE* dictEnd = LZ4_dict->dictionary + LZ4_dict->dictSize; int i; for (i=0; ihashTable[i] < delta) LZ4_dict->hashTable[i]=0; else LZ4_dict->hashTable[i] -= delta; } LZ4_dict->currentOffset = 64 KB; if (LZ4_dict->dictSize > 64 KB) LZ4_dict->dictSize = 64 KB; LZ4_dict->dictionary = dictEnd - LZ4_dict->dictSize; } } int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) { LZ4_stream_t_internal* streamPtr = (LZ4_stream_t_internal*)LZ4_stream; const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; const BYTE* smallest = (const BYTE*) source; if (streamPtr->initCheck) return 0; /* Uninitialized structure detected */ if ((streamPtr->dictSize>0) && (smallest>dictEnd)) smallest = dictEnd; LZ4_renormDictT(streamPtr, smallest); if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; /* Check overlapping input/dictionary space */ { const BYTE* sourceEnd = (const BYTE*) source + inputSize; if ((sourceEnd > streamPtr->dictionary) && (sourceEnd < dictEnd)) { streamPtr->dictSize = (U32)(dictEnd - sourceEnd); if (streamPtr->dictSize > 64 KB) streamPtr->dictSize = 64 KB; if (streamPtr->dictSize < 4) streamPtr->dictSize = 0; streamPtr->dictionary = dictEnd - streamPtr->dictSize; } } /* prefix mode : source data follows dictionary */ if (dictEnd == (const BYTE*)source) { int result; if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, dictSmall, acceleration); else result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, noDictIssue, acceleration); streamPtr->dictSize += (U32)inputSize; streamPtr->currentOffset += (U32)inputSize; return result; } /* external dictionary mode */ { int result; if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, dictSmall, acceleration); else result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, noDictIssue, acceleration); streamPtr->dictionary = (const BYTE*)source; streamPtr->dictSize = (U32)inputSize; streamPtr->currentOffset += (U32)inputSize; return result; } } /* Hidden debug function, to force external dictionary mode */ int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int inputSize) { LZ4_stream_t_internal* streamPtr = (LZ4_stream_t_internal*)LZ4_dict; int result; const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; const BYTE* smallest = dictEnd; if (smallest > (const BYTE*) source) smallest = (const BYTE*) source; LZ4_renormDictT((LZ4_stream_t_internal*)LZ4_dict, smallest); result = LZ4_compress_generic(LZ4_dict, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, noDictIssue, 1); streamPtr->dictionary = (const BYTE*)source; streamPtr->dictSize = (U32)inputSize; streamPtr->currentOffset += (U32)inputSize; return result; } int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize) { LZ4_stream_t_internal* dict = (LZ4_stream_t_internal*) LZ4_dict; const BYTE* previousDictEnd = dict->dictionary + dict->dictSize; if ((U32)dictSize > 64 KB) dictSize = 64 KB; /* useless to define a dictionary > 64 KB */ if ((U32)dictSize > dict->dictSize) dictSize = dict->dictSize; memmove(safeBuffer, previousDictEnd - dictSize, dictSize); dict->dictionary = (const BYTE*)safeBuffer; dict->dictSize = (U32)dictSize; return dictSize; } /******************************* * Decompression functions *******************************/ /* * This generic decompression function cover all use cases. * It shall be instantiated several times, using different sets of directives * Note that it is essential this generic function is really inlined, * in order to remove useless branches during compilation optimization. */ FORCE_INLINE int LZ4_decompress_generic( const char* const source, char* const dest, int inputSize, int outputSize, /* If endOnInput==endOnInputSize, this value is the max size of Output Buffer. */ int endOnInput, /* endOnOutputSize, endOnInputSize */ int partialDecoding, /* full, partial */ int targetOutputSize, /* only used if partialDecoding==partial */ int dict, /* noDict, withPrefix64k, usingExtDict */ const BYTE* const lowPrefix, /* == dest if dict == noDict */ const BYTE* const dictStart, /* only if dict==usingExtDict */ const size_t dictSize /* note : = 0 if noDict */ ) { /* Local Variables */ const BYTE* ip = (const BYTE*) source; const BYTE* const iend = ip + inputSize; BYTE* op = (BYTE*) dest; BYTE* const oend = op + outputSize; BYTE* cpy; BYTE* oexit = op + targetOutputSize; const BYTE* const lowLimit = lowPrefix - dictSize; const BYTE* const dictEnd = (const BYTE*)dictStart + dictSize; const size_t dec32table[] = {4, 1, 2, 1, 4, 4, 4, 4}; const size_t dec64table[] = {0, 0, 0, (size_t)-1, 0, 1, 2, 3}; const int safeDecode = (endOnInput==endOnInputSize); const int checkOffset = ((safeDecode) && (dictSize < (int)(64 KB))); /* Special cases */ if ((partialDecoding) && (oexit> oend-MFLIMIT)) oexit = oend-MFLIMIT; /* targetOutputSize too high => decode everything */ if ((endOnInput) && (unlikely(outputSize==0))) return ((inputSize==1) && (*ip==0)) ? 0 : -1; /* Empty output buffer */ if ((!endOnInput) && (unlikely(outputSize==0))) return (*ip==0?1:-1); /* Main Loop */ while (1) { unsigned token; size_t length; const BYTE* match; /* get literal length */ token = *ip++; if ((length=(token>>ML_BITS)) == RUN_MASK) { unsigned s; do { s = *ip++; length += s; } while (likely((endOnInput)?ip(partialDecoding?oexit:oend-MFLIMIT)) || (ip+length>iend-(2+1+LASTLITERALS))) ) || ((!endOnInput) && (cpy>oend-COPYLENGTH))) { if (partialDecoding) { if (cpy > oend) goto _output_error; /* Error : write attempt beyond end of output buffer */ if ((endOnInput) && (ip+length > iend)) goto _output_error; /* Error : read attempt beyond end of input buffer */ } else { if ((!endOnInput) && (cpy != oend)) goto _output_error; /* Error : block decoding must stop exactly there */ if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) goto _output_error; /* Error : input must be consumed */ } memcpy(op, ip, length); ip += length; op += length; break; /* Necessarily EOF, due to parsing restrictions */ } LZ4_wildCopy(op, ip, cpy); ip += length; op = cpy; /* get offset */ match = cpy - LZ4_readLE16(ip); ip+=2; if ((checkOffset) && (unlikely(match < lowLimit))) goto _output_error; /* Error : offset outside destination buffer */ /* get matchlength */ length = token & ML_MASK; if (length == ML_MASK) { unsigned s; do { if ((endOnInput) && (ip > iend-LASTLITERALS)) goto _output_error; s = *ip++; length += s; } while (s==255); if ((safeDecode) && unlikely((size_t)(op+length)<(size_t)op)) goto _output_error; /* overflow detection */ } length += MINMATCH; /* check external dictionary */ if ((dict==usingExtDict) && (match < lowPrefix)) { if (unlikely(op+length > oend-LASTLITERALS)) goto _output_error; /* doesn't respect parsing restriction */ if (length <= (size_t)(lowPrefix-match)) { /* match can be copied as a single segment from external dictionary */ match = dictEnd - (lowPrefix-match); memmove(op, match, length); op += length; } else { /* match encompass external dictionary and current segment */ size_t copySize = (size_t)(lowPrefix-match); memcpy(op, dictEnd - copySize, copySize); op += copySize; copySize = length - copySize; if (copySize > (size_t)(op-lowPrefix)) /* overlap within current segment */ { BYTE* const endOfMatch = op + copySize; const BYTE* copyFrom = lowPrefix; while (op < endOfMatch) *op++ = *copyFrom++; } else { memcpy(op, lowPrefix, copySize); op += copySize; } } continue; } /* copy repeated sequence */ cpy = op + length; if (unlikely((op-match)<8)) { const size_t dec64 = dec64table[op-match]; op[0] = match[0]; op[1] = match[1]; op[2] = match[2]; op[3] = match[3]; match += dec32table[op-match]; LZ4_copy4(op+4, match); op += 8; match -= dec64; } else { LZ4_copy8(op, match); op+=8; match+=8; } if (unlikely(cpy>oend-12)) { if (cpy > oend-LASTLITERALS) goto _output_error; /* Error : last LASTLITERALS bytes must be literals */ if (op < oend-8) { LZ4_wildCopy(op, match, oend-8); match += (oend-8) - op; op = oend-8; } while (opprefixSize = (size_t) dictSize; lz4sd->prefixEnd = (const BYTE*) dictionary + dictSize; lz4sd->externalDict = NULL; lz4sd->extDictSize = 0; return 1; } /* *_continue() : These decoding functions allow decompression of multiple blocks in "streaming" mode. Previously decoded blocks must still be available at the memory position where they were decoded. If it's not possible, save the relevant part of decoded data into a safe buffer, and indicate where it stands using LZ4_setStreamDecode() */ int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxOutputSize) { LZ4_streamDecode_t_internal* lz4sd = (LZ4_streamDecode_t_internal*) LZ4_streamDecode; int result; if (lz4sd->prefixEnd == (BYTE*)dest) { result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, usingExtDict, lz4sd->prefixEnd - lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); if (result <= 0) return result; lz4sd->prefixSize += result; lz4sd->prefixEnd += result; } else { lz4sd->extDictSize = lz4sd->prefixSize; lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, usingExtDict, (BYTE*)dest, lz4sd->externalDict, lz4sd->extDictSize); if (result <= 0) return result; lz4sd->prefixSize = result; lz4sd->prefixEnd = (BYTE*)dest + result; } return result; } int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize) { LZ4_streamDecode_t_internal* lz4sd = (LZ4_streamDecode_t_internal*) LZ4_streamDecode; int result; if (lz4sd->prefixEnd == (BYTE*)dest) { result = LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, usingExtDict, lz4sd->prefixEnd - lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); if (result <= 0) return result; lz4sd->prefixSize += originalSize; lz4sd->prefixEnd += originalSize; } else { lz4sd->extDictSize = lz4sd->prefixSize; lz4sd->externalDict = (BYTE*)dest - lz4sd->extDictSize; result = LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, usingExtDict, (BYTE*)dest, lz4sd->externalDict, lz4sd->extDictSize); if (result <= 0) return result; lz4sd->prefixSize = originalSize; lz4sd->prefixEnd = (BYTE*)dest + originalSize; } return result; } /* Advanced decoding functions : *_usingDict() : These decoding functions work the same as "_continue" ones, the dictionary must be explicitly provided within parameters */ FORCE_INLINE int LZ4_decompress_usingDict_generic(const char* source, char* dest, int compressedSize, int maxOutputSize, int safe, const char* dictStart, int dictSize) { if (dictSize==0) return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest, NULL, 0); if (dictStart+dictSize == dest) { if (dictSize >= (int)(64 KB - 1)) return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, withPrefix64k, (BYTE*)dest-64 KB, NULL, 0); return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest-dictSize, NULL, 0); } return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize); } int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) { return LZ4_decompress_usingDict_generic(source, dest, compressedSize, maxOutputSize, 1, dictStart, dictSize); } int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSize, const char* dictStart, int dictSize) { return LZ4_decompress_usingDict_generic(source, dest, 0, originalSize, 0, dictStart, dictSize); } /* debug function */ int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) { return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize); } /*************************************************** * Obsolete Functions ***************************************************/ /* obsolete compression functions */ int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) { return LZ4_compress_default(source, dest, inputSize, maxOutputSize); } int LZ4_compress(const char* source, char* dest, int inputSize) { return LZ4_compress_default(source, dest, inputSize, LZ4_compressBound(inputSize)); } int LZ4_compress_limitedOutput_withState (void* state, const char* src, char* dst, int srcSize, int dstSize) { return LZ4_compress_fast_extState(state, src, dst, srcSize, dstSize, 1); } int LZ4_compress_withState (void* state, const char* src, char* dst, int srcSize) { return LZ4_compress_fast_extState(state, src, dst, srcSize, LZ4_compressBound(srcSize), 1); } int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_fast_continue(LZ4_stream, src, dst, srcSize, maxDstSize, 1); } int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize) { return LZ4_compress_fast_continue(LZ4_stream, source, dest, inputSize, LZ4_compressBound(inputSize), 1); } /* These function names are deprecated and should no longer be used. They are only provided here for compatibility with older user programs. - LZ4_uncompress is totally equivalent to LZ4_decompress_fast - LZ4_uncompress_unknownOutputSize is totally equivalent to LZ4_decompress_safe */ int LZ4_uncompress (const char* source, char* dest, int outputSize) { return LZ4_decompress_fast(source, dest, outputSize); } int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) { return LZ4_decompress_safe(source, dest, isize, maxOutputSize); } /* Obsolete Streaming functions */ int LZ4_sizeofStreamState() { return LZ4_STREAMSIZE; } static void LZ4_init(LZ4_stream_t_internal* lz4ds, BYTE* base) { MEM_INIT(lz4ds, 0, LZ4_STREAMSIZE); lz4ds->bufferStart = base; } int LZ4_resetStreamState(void* state, char* inputBuffer) { if ((((size_t)state) & 3) != 0) return 1; /* Error : pointer is not aligned on 4-bytes boundary */ LZ4_init((LZ4_stream_t_internal*)state, (BYTE*)inputBuffer); return 0; } void* LZ4_create (char* inputBuffer) { void* lz4ds = ALLOCATOR(8, LZ4_STREAMSIZE_U64); LZ4_init ((LZ4_stream_t_internal*)lz4ds, (BYTE*)inputBuffer); return lz4ds; } char* LZ4_slideInputBuffer (void* LZ4_Data) { LZ4_stream_t_internal* ctx = (LZ4_stream_t_internal*)LZ4_Data; int dictSize = LZ4_saveDict((LZ4_stream_t*)LZ4_Data, (char*)ctx->bufferStart, 64 KB); return (char*)(ctx->bufferStart + dictSize); } /* Obsolete streaming decompression functions */ int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize) { return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 64 KB); } int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int originalSize) { return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 64 KB); } #endif /* LZ4_COMMONDEFS_ONLY */ c-blosc-1.7.0/internal-complibs/lz4-1.7.0/lz4.h000066400000000000000000000446161254622165100205360ustar00rootroot00000000000000/* LZ4 - Fast LZ compression algorithm Header File Copyright (C) 2011-2015, Yann Collet. BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : - LZ4 source repository : https://github.com/Cyan4973/lz4 - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ #pragma once #if defined (__cplusplus) extern "C" { #endif /* * lz4.h provides block compression functions, and gives full buffer control to programmer. * If you need to generate inter-operable compressed data (respecting LZ4 frame specification), * and can let the library handle its own memory, please use lz4frame.h instead. */ /************************************** * Version **************************************/ #define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */ #define LZ4_VERSION_MINOR 7 /* for new (non-breaking) interface capabilities */ #define LZ4_VERSION_RELEASE 0 /* for tweaks, bug-fixes, or development */ #define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE) int LZ4_versionNumber (void); /************************************** * Tuning parameter **************************************/ /* * LZ4_MEMORY_USAGE : * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) * Increasing memory usage improves compression ratio * Reduced memory usage can improve speed, due to cache effect * Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */ #define LZ4_MEMORY_USAGE 14 /************************************** * Simple Functions **************************************/ int LZ4_compress_default(const char* source, char* dest, int sourceSize, int maxDestSize); int LZ4_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize); /* LZ4_compress_default() : Compresses 'sourceSize' bytes from buffer 'source' into already allocated 'dest' buffer of size 'maxDestSize'. Compression is guaranteed to succeed if 'maxDestSize' >= LZ4_compressBound(sourceSize). It also runs faster, so it's a recommended setting. If the function cannot compress 'source' into a more limited 'dest' budget, compression stops *immediately*, and the function result is zero. As a consequence, 'dest' content is not valid. This function never writes outside 'dest' buffer, nor read outside 'source' buffer. sourceSize : Max supported value is LZ4_MAX_INPUT_VALUE maxDestSize : full or partial size of buffer 'dest' (which must be already allocated) return : the number of bytes written into buffer 'dest' (necessarily <= maxOutputSize) or 0 if compression fails LZ4_decompress_safe() : compressedSize : is the precise full size of the compressed block. maxDecompressedSize : is the size of destination buffer, which must be already allocated. return : the number of bytes decompressed into destination buffer (necessarily <= maxDecompressedSize) If destination buffer is not large enough, decoding will stop and output an error code (<0). If the source stream is detected malformed, the function will stop decoding and return a negative result. This function is protected against buffer overflow exploits, including malicious data packets. It never writes outside output buffer, nor reads outside input buffer. */ /************************************** * Advanced Functions **************************************/ #define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */ #define LZ4_COMPRESSBOUND(isize) ((unsigned)(isize) > (unsigned)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16) /* LZ4_compressBound() : Provides the maximum size that LZ4 compression may output in a "worst case" scenario (input data not compressible) This function is primarily useful for memory allocation purposes (destination buffer size). Macro LZ4_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example). Note that LZ4_compress_default() compress faster when dest buffer size is >= LZ4_compressBound(srcSize) inputSize : max supported value is LZ4_MAX_INPUT_SIZE return : maximum output size in a "worst case" scenario or 0, if input size is too large ( > LZ4_MAX_INPUT_SIZE) */ int LZ4_compressBound(int inputSize); /* LZ4_compress_fast() : Same as LZ4_compress_default(), but allows to select an "acceleration" factor. The larger the acceleration value, the faster the algorithm, but also the lesser the compression. It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed. An acceleration value of "1" is the same as regular LZ4_compress_default() Values <= 0 will be replaced by ACCELERATION_DEFAULT (see lz4.c), which is 1. */ int LZ4_compress_fast (const char* source, char* dest, int sourceSize, int maxDestSize, int acceleration); /* LZ4_compress_fast_extState() : Same compression function, just using an externally allocated memory space to store compression state. Use LZ4_sizeofState() to know how much memory must be allocated, and allocate it on 8-bytes boundaries (using malloc() typically). Then, provide it as 'void* state' to compression function. */ int LZ4_sizeofState(void); int LZ4_compress_fast_extState (void* state, const char* source, char* dest, int inputSize, int maxDestSize, int acceleration); /* LZ4_compress_destSize() : Reverse the logic, by compressing as much data as possible from 'source' buffer into already allocated buffer 'dest' of size 'targetDestSize'. This function either compresses the entire 'source' content into 'dest' if it's large enough, or fill 'dest' buffer completely with as much data as possible from 'source'. *sourceSizePtr : will be modified to indicate how many bytes where read from 'source' to fill 'dest'. New value is necessarily <= old value. return : Nb bytes written into 'dest' (necessarily <= targetDestSize) or 0 if compression fails */ int LZ4_compress_destSize (const char* source, char* dest, int* sourceSizePtr, int targetDestSize); /* LZ4_decompress_fast() : originalSize : is the original and therefore uncompressed size return : the number of bytes read from the source buffer (in other words, the compressed size) If the source stream is detected malformed, the function will stop decoding and return a negative result. Destination buffer must be already allocated. Its size must be a minimum of 'originalSize' bytes. note : This function fully respect memory boundaries for properly formed compressed data. It is a bit faster than LZ4_decompress_safe(). However, it does not provide any protection against intentionally modified data stream (malicious input). Use this function in trusted environment only (data to decode comes from a trusted source). */ int LZ4_decompress_fast (const char* source, char* dest, int originalSize); /* LZ4_decompress_safe_partial() : This function decompress a compressed block of size 'compressedSize' at position 'source' into destination buffer 'dest' of size 'maxDecompressedSize'. The function tries to stop decompressing operation as soon as 'targetOutputSize' has been reached, reducing decompression time. return : the number of bytes decoded in the destination buffer (necessarily <= maxDecompressedSize) Note : this number can be < 'targetOutputSize' should the compressed block to decode be smaller. Always control how many bytes were decoded. If the source stream is detected malformed, the function will stop decoding and return a negative result. This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets */ int LZ4_decompress_safe_partial (const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize); /*********************************************** * Streaming Compression Functions ***********************************************/ #define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4) #define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(long long)) /* * LZ4_stream_t * information structure to track an LZ4 stream. * important : init this structure content before first use ! * note : only allocated directly the structure if you are statically linking LZ4 * If you are using liblz4 as a DLL, please use below construction methods instead. */ typedef struct { long long table[LZ4_STREAMSIZE_U64]; } LZ4_stream_t; /* * LZ4_resetStream * Use this function to init an allocated LZ4_stream_t structure */ void LZ4_resetStream (LZ4_stream_t* streamPtr); /* * LZ4_createStream will allocate and initialize an LZ4_stream_t structure * LZ4_freeStream releases its memory. * In the context of a DLL (liblz4), please use these methods rather than the static struct. * They are more future proof, in case of a change of LZ4_stream_t size. */ LZ4_stream_t* LZ4_createStream(void); int LZ4_freeStream (LZ4_stream_t* streamPtr); /* * LZ4_loadDict * Use this function to load a static dictionary into LZ4_stream. * Any previous data will be forgotten, only 'dictionary' will remain in memory. * Loading a size of 0 is allowed. * Return : dictionary size, in bytes (necessarily <= 64 KB) */ int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, int dictSize); /* * LZ4_compress_fast_continue * Compress buffer content 'src', using data from previously compressed blocks as dictionary to improve compression ratio. * Important : Previous data blocks are assumed to still be present and unmodified ! * 'dst' buffer must be already allocated. * If maxDstSize >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster. * If not, and if compressed data cannot fit into 'dst' buffer size, compression stops, and function returns a zero. */ int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int maxDstSize, int acceleration); /* * LZ4_saveDict * If previously compressed data block is not guaranteed to remain available at its memory location * save it into a safer place (char* safeBuffer) * Note : you don't need to call LZ4_loadDict() afterwards, * dictionary is immediately usable, you can therefore call LZ4_compress_fast_continue() * Return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error */ int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int dictSize); /************************************************ * Streaming Decompression Functions ************************************************/ #define LZ4_STREAMDECODESIZE_U64 4 #define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U64 * sizeof(unsigned long long)) typedef struct { unsigned long long table[LZ4_STREAMDECODESIZE_U64]; } LZ4_streamDecode_t; /* * LZ4_streamDecode_t * information structure to track an LZ4 stream. * init this structure content using LZ4_setStreamDecode or memset() before first use ! * * In the context of a DLL (liblz4) please prefer usage of construction methods below. * They are more future proof, in case of a change of LZ4_streamDecode_t size in the future. * LZ4_createStreamDecode will allocate and initialize an LZ4_streamDecode_t structure * LZ4_freeStreamDecode releases its memory. */ LZ4_streamDecode_t* LZ4_createStreamDecode(void); int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream); /* * LZ4_setStreamDecode * Use this function to instruct where to find the dictionary. * Setting a size of 0 is allowed (same effect as reset). * Return : 1 if OK, 0 if error */ int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize); /* *_continue() : These decoding functions allow decompression of multiple blocks in "streaming" mode. Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB) In the case of a ring buffers, decoding buffer must be either : - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions) In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB). - Larger than encoding buffer, by a minimum of maxBlockSize more bytes. maxBlockSize is implementation dependent. It's the maximum size you intend to compress into a single block. In which case, encoding and decoding buffers do not need to be synchronized, and encoding ring buffer can have any size, including small ones ( < 64 KB). - _At least_ 64 KB + 8 bytes + maxBlockSize. In which case, encoding and decoding buffers do not need to be synchronized, and encoding ring buffer can have any size, including larger than decoding buffer. Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer, and indicate where it is saved using LZ4_setStreamDecode() */ int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize); int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize); /* Advanced decoding functions : *_usingDict() : These decoding functions work the same as a combination of LZ4_setStreamDecode() followed by LZ4_decompress_x_continue() They are stand-alone. They don't need nor update an LZ4_streamDecode_t structure. */ int LZ4_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize); int LZ4_decompress_fast_usingDict (const char* source, char* dest, int originalSize, const char* dictStart, int dictSize); /************************************** * Obsolete Functions **************************************/ /* Deprecate Warnings */ /* Should these warnings messages be a problem, it is generally possible to disable them, with -Wno-deprecated-declarations for gcc or _CRT_SECURE_NO_WARNINGS in Visual for example. You can also define LZ4_DEPRECATE_WARNING_DEFBLOCK. */ #ifndef LZ4_DEPRECATE_WARNING_DEFBLOCK # define LZ4_DEPRECATE_WARNING_DEFBLOCK # define LZ4_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) # if (LZ4_GCC_VERSION >= 405) || defined(__clang__) # define LZ4_DEPRECATED(message) __attribute__((deprecated(message))) # elif (LZ4_GCC_VERSION >= 301) # define LZ4_DEPRECATED(message) __attribute__((deprecated)) # elif defined(_MSC_VER) # define LZ4_DEPRECATED(message) __declspec(deprecated(message)) # else # pragma message("WARNING: You need to implement LZ4_DEPRECATED for this compiler") # define LZ4_DEPRECATED(message) # endif #endif /* LZ4_DEPRECATE_WARNING_DEFBLOCK */ /* Obsolete compression functions */ /* These functions are planned to start generate warnings by r131 approximately */ int LZ4_compress (const char* source, char* dest, int sourceSize); int LZ4_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize); int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize); int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize); int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize); /* Obsolete decompression functions */ /* These function names are completely deprecated and must no longer be used. They are only provided here for compatibility with older programs. - LZ4_uncompress is the same as LZ4_decompress_fast - LZ4_uncompress_unknownOutputSize is the same as LZ4_decompress_safe These function prototypes are now disabled; uncomment them only if you really need them. It is highly recommended to stop using these prototypes and migrate to maintained ones */ /* int LZ4_uncompress (const char* source, char* dest, int outputSize); */ /* int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize); */ /* Obsolete streaming functions; use new streaming interface whenever possible */ LZ4_DEPRECATED("use LZ4_createStream() instead") void* LZ4_create (char* inputBuffer); LZ4_DEPRECATED("use LZ4_createStream() instead") int LZ4_sizeofStreamState(void); LZ4_DEPRECATED("use LZ4_resetStream() instead") int LZ4_resetStreamState(void* state, char* inputBuffer); LZ4_DEPRECATED("use LZ4_saveDict() instead") char* LZ4_slideInputBuffer (void* state); /* Obsolete streaming decoding functions */ LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize); LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize); #if defined (__cplusplus) } #endif c-blosc-1.7.0/internal-complibs/lz4-1.7.0/lz4hc.c000066400000000000000000000645261254622165100210460ustar00rootroot00000000000000/* LZ4 HC - High Compression Mode of LZ4 Copyright (C) 2011-2015, Yann Collet. BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : - LZ4 source repository : https://github.com/Cyan4973/lz4 - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ /************************************** * Tuning Parameter **************************************/ static const int LZ4HC_compressionLevel_default = 9; /************************************** * Includes **************************************/ #include "lz4hc.h" /************************************** * Local Compiler Options **************************************/ #if defined(__GNUC__) # pragma GCC diagnostic ignored "-Wunused-function" #endif #if defined (__clang__) # pragma clang diagnostic ignored "-Wunused-function" #endif /************************************** * Common LZ4 definition **************************************/ #define LZ4_COMMONDEFS_ONLY #include "lz4.c" /************************************** * Local Constants **************************************/ #define DICTIONARY_LOGSIZE 16 #define MAXD (1<> ((MINMATCH*8)-HASH_LOG)) //#define DELTANEXTU16(p) chainTable[(p) & MAXD_MASK] /* flexible, MAXD dependent */ #define DELTANEXTU16(p) chainTable[(U16)(p)] /* faster */ static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)); } /************************************** * HC Compression **************************************/ static void LZ4HC_init (LZ4HC_Data_Structure* hc4, const BYTE* start) { MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable)); MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable)); hc4->nextToUpdate = 64 KB; hc4->base = start - 64 KB; hc4->end = start; hc4->dictBase = start - 64 KB; hc4->dictLimit = 64 KB; hc4->lowLimit = 64 KB; } /* Update chains up to ip (excluded) */ FORCE_INLINE void LZ4HC_Insert (LZ4HC_Data_Structure* hc4, const BYTE* ip) { U16* chainTable = hc4->chainTable; U32* HashTable = hc4->hashTable; const BYTE* const base = hc4->base; const U32 target = (U32)(ip - base); U32 idx = hc4->nextToUpdate; while(idx < target) { U32 h = LZ4HC_hashPtr(base+idx); size_t delta = idx - HashTable[h]; if (delta>MAX_DISTANCE) delta = MAX_DISTANCE; DELTANEXTU16(idx) = (U16)delta; HashTable[h] = idx; idx++; } hc4->nextToUpdate = target; } FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, /* Index table will be updated */ const BYTE* ip, const BYTE* const iLimit, const BYTE** matchpos, const int maxNbAttempts) { U16* const chainTable = hc4->chainTable; U32* const HashTable = hc4->hashTable; const BYTE* const base = hc4->base; const BYTE* const dictBase = hc4->dictBase; const U32 dictLimit = hc4->dictLimit; const U32 lowLimit = (hc4->lowLimit + 64 KB > (U32)(ip-base)) ? hc4->lowLimit : (U32)(ip - base) - (64 KB - 1); U32 matchIndex; const BYTE* match; int nbAttempts=maxNbAttempts; size_t ml=0; /* HC4 match finder */ LZ4HC_Insert(hc4, ip); matchIndex = HashTable[LZ4HC_hashPtr(ip)]; while ((matchIndex>=lowLimit) && (nbAttempts)) { nbAttempts--; if (matchIndex >= dictLimit) { match = base + matchIndex; if (*(match+ml) == *(ip+ml) && (LZ4_read32(match) == LZ4_read32(ip))) { size_t mlt = LZ4_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH; if (mlt > ml) { ml = mlt; *matchpos = match; } } } else { match = dictBase + matchIndex; if (LZ4_read32(match) == LZ4_read32(ip)) { size_t mlt; const BYTE* vLimit = ip + (dictLimit - matchIndex); if (vLimit > iLimit) vLimit = iLimit; mlt = LZ4_count(ip+MINMATCH, match+MINMATCH, vLimit) + MINMATCH; if ((ip+mlt == vLimit) && (vLimit < iLimit)) mlt += LZ4_count(ip+mlt, base+dictLimit, iLimit); if (mlt > ml) { ml = mlt; *matchpos = base + matchIndex; } /* virtual matchpos */ } } matchIndex -= DELTANEXTU16(matchIndex); } return (int)ml; } FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch ( LZ4HC_Data_Structure* hc4, const BYTE* const ip, const BYTE* const iLowLimit, const BYTE* const iHighLimit, int longest, const BYTE** matchpos, const BYTE** startpos, const int maxNbAttempts) { U16* const chainTable = hc4->chainTable; U32* const HashTable = hc4->hashTable; const BYTE* const base = hc4->base; const U32 dictLimit = hc4->dictLimit; const BYTE* const lowPrefixPtr = base + dictLimit; const U32 lowLimit = (hc4->lowLimit + 64 KB > (U32)(ip-base)) ? hc4->lowLimit : (U32)(ip - base) - (64 KB - 1); const BYTE* const dictBase = hc4->dictBase; U32 matchIndex; int nbAttempts = maxNbAttempts; int delta = (int)(ip-iLowLimit); /* First Match */ LZ4HC_Insert(hc4, ip); matchIndex = HashTable[LZ4HC_hashPtr(ip)]; while ((matchIndex>=lowLimit) && (nbAttempts)) { nbAttempts--; if (matchIndex >= dictLimit) { const BYTE* matchPtr = base + matchIndex; if (*(iLowLimit + longest) == *(matchPtr - delta + longest)) if (LZ4_read32(matchPtr) == LZ4_read32(ip)) { int mlt = MINMATCH + LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit); int back = 0; while ((ip+back>iLowLimit) && (matchPtr+back > lowPrefixPtr) && (ip[back-1] == matchPtr[back-1])) back--; mlt -= back; if (mlt > longest) { longest = (int)mlt; *matchpos = matchPtr+back; *startpos = ip+back; } } } else { const BYTE* matchPtr = dictBase + matchIndex; if (LZ4_read32(matchPtr) == LZ4_read32(ip)) { size_t mlt; int back=0; const BYTE* vLimit = ip + (dictLimit - matchIndex); if (vLimit > iHighLimit) vLimit = iHighLimit; mlt = LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH; if ((ip+mlt == vLimit) && (vLimit < iHighLimit)) mlt += LZ4_count(ip+mlt, base+dictLimit, iHighLimit); while ((ip+back > iLowLimit) && (matchIndex+back > lowLimit) && (ip[back-1] == matchPtr[back-1])) back--; mlt -= back; if ((int)mlt > longest) { longest = (int)mlt; *matchpos = base + matchIndex + back; *startpos = ip+back; } } } matchIndex -= DELTANEXTU16(matchIndex); } return longest; } typedef enum { noLimit = 0, limitedOutput = 1 } limitedOutput_directive; #define LZ4HC_DEBUG 0 #if LZ4HC_DEBUG static unsigned debug = 0; #endif FORCE_INLINE int LZ4HC_encodeSequence ( const BYTE** ip, BYTE** op, const BYTE** anchor, int matchLength, const BYTE* const match, limitedOutput_directive limitedOutputBuffer, BYTE* oend) { int length; BYTE* token; #if LZ4HC_DEBUG if (debug) printf("literal : %u -- match : %u -- offset : %u\n", (U32)(*ip - *anchor), (U32)matchLength, (U32)(*ip-match)); #endif /* Encode Literal length */ length = (int)(*ip - *anchor); token = (*op)++; if ((limitedOutputBuffer) && ((*op + (length>>8) + length + (2 + 1 + LASTLITERALS)) > oend)) return 1; /* Check output limit */ if (length>=(int)RUN_MASK) { int len; *token=(RUN_MASK< 254 ; len-=255) *(*op)++ = 255; *(*op)++ = (BYTE)len; } else *token = (BYTE)(length<>8) + (1 + LASTLITERALS) > oend)) return 1; /* Check output limit */ if (length>=(int)ML_MASK) { *token+=ML_MASK; length-=ML_MASK; for(; length > 509 ; length-=510) { *(*op)++ = 255; *(*op)++ = 255; } if (length > 254) { length-=255; *(*op)++ = 255; } *(*op)++ = (BYTE)length; } else *token += (BYTE)(length); /* Prepare next loop */ *ip += matchLength; *anchor = *ip; return 0; } static int LZ4HC_compress_generic ( void* ctxvoid, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel, limitedOutput_directive limit ) { LZ4HC_Data_Structure* ctx = (LZ4HC_Data_Structure*) ctxvoid; const BYTE* ip = (const BYTE*) source; const BYTE* anchor = ip; const BYTE* const iend = ip + inputSize; const BYTE* const mflimit = iend - MFLIMIT; const BYTE* const matchlimit = (iend - LASTLITERALS); BYTE* op = (BYTE*) dest; BYTE* const oend = op + maxOutputSize; unsigned maxNbAttempts; int ml, ml2, ml3, ml0; const BYTE* ref=NULL; const BYTE* start2=NULL; const BYTE* ref2=NULL; const BYTE* start3=NULL; const BYTE* ref3=NULL; const BYTE* start0; const BYTE* ref0; /* init */ if (compressionLevel > g_maxCompressionLevel) compressionLevel = g_maxCompressionLevel; if (compressionLevel < 1) compressionLevel = LZ4HC_compressionLevel_default; maxNbAttempts = 1 << (compressionLevel-1); ctx->end += inputSize; ip++; /* Main Loop */ while (ip < mflimit) { ml = LZ4HC_InsertAndFindBestMatch (ctx, ip, matchlimit, (&ref), maxNbAttempts); if (!ml) { ip++; continue; } /* saved, in case we would skip too much */ start0 = ip; ref0 = ref; ml0 = ml; _Search2: if (ip+ml < mflimit) ml2 = LZ4HC_InsertAndGetWiderMatch(ctx, ip + ml - 2, ip + 1, matchlimit, ml, &ref2, &start2, maxNbAttempts); else ml2 = ml; if (ml2 == ml) /* No better match */ { if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0; continue; } if (start0 < ip) { if (start2 < ip + ml0) /* empirical */ { ip = start0; ref = ref0; ml = ml0; } } /* Here, start0==ip */ if ((start2 - ip) < 3) /* First Match too small : removed */ { ml = ml2; ip = start2; ref =ref2; goto _Search2; } _Search3: /* * Currently we have : * ml2 > ml1, and * ip1+3 <= ip2 (usually < ip1+ml1) */ if ((start2 - ip) < OPTIMAL_ML) { int correction; int new_ml = ml; if (new_ml > OPTIMAL_ML) new_ml = OPTIMAL_ML; if (ip+new_ml > start2 + ml2 - MINMATCH) new_ml = (int)(start2 - ip) + ml2 - MINMATCH; correction = new_ml - (int)(start2 - ip); if (correction > 0) { start2 += correction; ref2 += correction; ml2 -= correction; } } /* Now, we have start2 = ip+new_ml, with new_ml = min(ml, OPTIMAL_ML=18) */ if (start2 + ml2 < mflimit) ml3 = LZ4HC_InsertAndGetWiderMatch(ctx, start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3, maxNbAttempts); else ml3 = ml2; if (ml3 == ml2) /* No better match : 2 sequences to encode */ { /* ip & ref are known; Now for ml */ if (start2 < ip+ml) ml = (int)(start2 - ip); /* Now, encode 2 sequences */ if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0; ip = start2; if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml2, ref2, limit, oend)) return 0; continue; } if (start3 < ip+ml+3) /* Not enough space for match 2 : remove it */ { if (start3 >= (ip+ml)) /* can write Seq1 immediately ==> Seq2 is removed, so Seq3 becomes Seq1 */ { if (start2 < ip+ml) { int correction = (int)(ip+ml - start2); start2 += correction; ref2 += correction; ml2 -= correction; if (ml2 < MINMATCH) { start2 = start3; ref2 = ref3; ml2 = ml3; } } if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0; ip = start3; ref = ref3; ml = ml3; start0 = start2; ref0 = ref2; ml0 = ml2; goto _Search2; } start2 = start3; ref2 = ref3; ml2 = ml3; goto _Search3; } /* * OK, now we have 3 ascending matches; let's write at least the first one * ip & ref are known; Now for ml */ if (start2 < ip+ml) { if ((start2 - ip) < (int)ML_MASK) { int correction; if (ml > OPTIMAL_ML) ml = OPTIMAL_ML; if (ip + ml > start2 + ml2 - MINMATCH) ml = (int)(start2 - ip) + ml2 - MINMATCH; correction = ml - (int)(start2 - ip); if (correction > 0) { start2 += correction; ref2 += correction; ml2 -= correction; } } else { ml = (int)(start2 - ip); } } if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0; ip = start2; ref = ref2; ml = ml2; start2 = start3; ref2 = ref3; ml2 = ml3; goto _Search3; } /* Encode Last Literals */ { int lastRun = (int)(iend - anchor); if ((limit) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0; /* Check output limit */ if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK< 254 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; } else *op++ = (BYTE)(lastRun<base = NULL; ((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->compressionLevel = (unsigned)compressionLevel; } int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize) { LZ4HC_Data_Structure* ctxPtr = (LZ4HC_Data_Structure*) LZ4_streamHCPtr; if (dictSize > 64 KB) { dictionary += dictSize - 64 KB; dictSize = 64 KB; } LZ4HC_init (ctxPtr, (const BYTE*)dictionary); if (dictSize >= 4) LZ4HC_Insert (ctxPtr, (const BYTE*)dictionary +(dictSize-3)); ctxPtr->end = (const BYTE*)dictionary + dictSize; return dictSize; } /* compression */ static void LZ4HC_setExternalDict(LZ4HC_Data_Structure* ctxPtr, const BYTE* newBlock) { if (ctxPtr->end >= ctxPtr->base + 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3); /* Referencing remaining dictionary content */ /* Only one memory segment for extDict, so any previous extDict is lost at this stage */ ctxPtr->lowLimit = ctxPtr->dictLimit; ctxPtr->dictLimit = (U32)(ctxPtr->end - ctxPtr->base); ctxPtr->dictBase = ctxPtr->base; ctxPtr->base = newBlock - ctxPtr->dictLimit; ctxPtr->end = newBlock; ctxPtr->nextToUpdate = ctxPtr->dictLimit; /* match referencing will resume from there */ } static int LZ4_compressHC_continue_generic (LZ4HC_Data_Structure* ctxPtr, const char* source, char* dest, int inputSize, int maxOutputSize, limitedOutput_directive limit) { /* auto-init if forgotten */ if (ctxPtr->base == NULL) LZ4HC_init (ctxPtr, (const BYTE*) source); /* Check overflow */ if ((size_t)(ctxPtr->end - ctxPtr->base) > 2 GB) { size_t dictSize = (size_t)(ctxPtr->end - ctxPtr->base) - ctxPtr->dictLimit; if (dictSize > 64 KB) dictSize = 64 KB; LZ4_loadDictHC((LZ4_streamHC_t*)ctxPtr, (const char*)(ctxPtr->end) - dictSize, (int)dictSize); } /* Check if blocks follow each other */ if ((const BYTE*)source != ctxPtr->end) LZ4HC_setExternalDict(ctxPtr, (const BYTE*)source); /* Check overlapping input/dictionary space */ { const BYTE* sourceEnd = (const BYTE*) source + inputSize; const BYTE* dictBegin = ctxPtr->dictBase + ctxPtr->lowLimit; const BYTE* dictEnd = ctxPtr->dictBase + ctxPtr->dictLimit; if ((sourceEnd > dictBegin) && ((const BYTE*)source < dictEnd)) { if (sourceEnd > dictEnd) sourceEnd = dictEnd; ctxPtr->lowLimit = (U32)(sourceEnd - ctxPtr->dictBase); if (ctxPtr->dictLimit - ctxPtr->lowLimit < 4) ctxPtr->lowLimit = ctxPtr->dictLimit; } } return LZ4HC_compress_generic (ctxPtr, source, dest, inputSize, maxOutputSize, ctxPtr->compressionLevel, limit); } int LZ4_compress_HC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize) { if (maxOutputSize < LZ4_compressBound(inputSize)) return LZ4_compressHC_continue_generic ((LZ4HC_Data_Structure*)LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, limitedOutput); else return LZ4_compressHC_continue_generic ((LZ4HC_Data_Structure*)LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, noLimit); } /* dictionary saving */ int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictSize) { LZ4HC_Data_Structure* streamPtr = (LZ4HC_Data_Structure*)LZ4_streamHCPtr; int prefixSize = (int)(streamPtr->end - (streamPtr->base + streamPtr->dictLimit)); if (dictSize > 64 KB) dictSize = 64 KB; if (dictSize < 4) dictSize = 0; if (dictSize > prefixSize) dictSize = prefixSize; memmove(safeBuffer, streamPtr->end - dictSize, dictSize); { U32 endIndex = (U32)(streamPtr->end - streamPtr->base); streamPtr->end = (const BYTE*)safeBuffer + dictSize; streamPtr->base = streamPtr->end - endIndex; streamPtr->dictLimit = endIndex - dictSize; streamPtr->lowLimit = endIndex - dictSize; if (streamPtr->nextToUpdate < streamPtr->dictLimit) streamPtr->nextToUpdate = streamPtr->dictLimit; } return dictSize; } /*********************************** * Deprecated Functions ***********************************/ /* Deprecated compression functions */ /* These functions are planned to start generate warnings by r131 approximately */ int LZ4_compressHC(const char* src, char* dst, int srcSize) { return LZ4_compress_HC (src, dst, srcSize, LZ4_compressBound(srcSize), 0); } int LZ4_compressHC_limitedOutput(const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_HC(src, dst, srcSize, maxDstSize, 0); } int LZ4_compressHC2(const char* src, char* dst, int srcSize, int cLevel) { return LZ4_compress_HC (src, dst, srcSize, LZ4_compressBound(srcSize), cLevel); } int LZ4_compressHC2_limitedOutput(const char* src, char* dst, int srcSize, int maxDstSize, int cLevel) { return LZ4_compress_HC(src, dst, srcSize, maxDstSize, cLevel); } int LZ4_compressHC_withStateHC (void* state, const char* src, char* dst, int srcSize) { return LZ4_compress_HC_extStateHC (state, src, dst, srcSize, LZ4_compressBound(srcSize), 0); } int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_HC_extStateHC (state, src, dst, srcSize, maxDstSize, 0); } int LZ4_compressHC2_withStateHC (void* state, const char* src, char* dst, int srcSize, int cLevel) { return LZ4_compress_HC_extStateHC(state, src, dst, srcSize, LZ4_compressBound(srcSize), cLevel); } int LZ4_compressHC2_limitedOutput_withStateHC (void* state, const char* src, char* dst, int srcSize, int maxDstSize, int cLevel) { return LZ4_compress_HC_extStateHC(state, src, dst, srcSize, maxDstSize, cLevel); } int LZ4_compressHC_continue (LZ4_streamHC_t* ctx, const char* src, char* dst, int srcSize) { return LZ4_compress_HC_continue (ctx, src, dst, srcSize, LZ4_compressBound(srcSize)); } int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* ctx, const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_HC_continue (ctx, src, dst, srcSize, maxDstSize); } /* Deprecated streaming functions */ /* These functions currently generate deprecation warnings */ int LZ4_sizeofStreamStateHC(void) { return LZ4_STREAMHCSIZE; } int LZ4_resetStreamStateHC(void* state, char* inputBuffer) { if ((((size_t)state) & (sizeof(void*)-1)) != 0) return 1; /* Error : pointer is not aligned for pointer (32 or 64 bits) */ LZ4HC_init((LZ4HC_Data_Structure*)state, (const BYTE*)inputBuffer); ((LZ4HC_Data_Structure*)state)->inputBuffer = (BYTE*)inputBuffer; return 0; } void* LZ4_createHC (char* inputBuffer) { void* hc4 = ALLOCATOR(1, sizeof(LZ4HC_Data_Structure)); LZ4HC_init ((LZ4HC_Data_Structure*)hc4, (const BYTE*)inputBuffer); ((LZ4HC_Data_Structure*)hc4)->inputBuffer = (BYTE*)inputBuffer; return hc4; } int LZ4_freeHC (void* LZ4HC_Data) { FREEMEM(LZ4HC_Data); return (0); } int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel) { return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, 0, compressionLevel, noLimit); } int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel) { return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, maxOutputSize, compressionLevel, limitedOutput); } char* LZ4_slideInputBufferHC(void* LZ4HC_Data) { LZ4HC_Data_Structure* hc4 = (LZ4HC_Data_Structure*)LZ4HC_Data; int dictSize = LZ4_saveDictHC((LZ4_streamHC_t*)LZ4HC_Data, (char*)(hc4->inputBuffer), 64 KB); return (char*)(hc4->inputBuffer + dictSize); } c-blosc-1.7.0/internal-complibs/lz4-1.7.0/lz4hc.h000066400000000000000000000226221254622165100210420ustar00rootroot00000000000000/* LZ4 HC - High Compression Mode of LZ4 Header File Copyright (C) 2011-2015, Yann Collet. BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : - LZ4 source repository : https://github.com/Cyan4973/lz4 - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c */ #pragma once #if defined (__cplusplus) extern "C" { #endif /***************************** * Includes *****************************/ #include /* size_t */ /************************************** * Block Compression **************************************/ int LZ4_compress_HC (const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel); /* LZ4_compress_HC : Destination buffer 'dst' must be already allocated. Compression completion is guaranteed if 'dst' buffer is sized to handle worst circumstances (data not compressible) Worst size evaluation is provided by function LZ4_compressBound() (see "lz4.h") srcSize : Max supported value is LZ4_MAX_INPUT_SIZE (see "lz4.h") compressionLevel : Recommended values are between 4 and 9, although any value between 0 and 16 will work. 0 means "use default value" (see lz4hc.c). Values >16 behave the same as 16. return : the number of bytes written into buffer 'dst' or 0 if compression fails. */ /* Note : Decompression functions are provided within LZ4 source code (see "lz4.h") (BSD license) */ int LZ4_sizeofStateHC(void); int LZ4_compress_HC_extStateHC(void* state, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel); /* LZ4_compress_HC_extStateHC() : Use this function if you prefer to manually allocate memory for compression tables. To know how much memory must be allocated for the compression tables, use : int LZ4_sizeofStateHC(); Allocated memory must be aligned on 8-bytes boundaries (which a normal malloc() will do properly). The allocated memory can then be provided to the compression functions using 'void* state' parameter. LZ4_compress_HC_extStateHC() is equivalent to previously described function. It just uses externally allocated memory for stateHC. */ /************************************** * Streaming Compression **************************************/ #define LZ4_STREAMHCSIZE 262192 #define LZ4_STREAMHCSIZE_SIZET (LZ4_STREAMHCSIZE / sizeof(size_t)) typedef struct { size_t table[LZ4_STREAMHCSIZE_SIZET]; } LZ4_streamHC_t; /* LZ4_streamHC_t This structure allows static allocation of LZ4 HC streaming state. State must then be initialized using LZ4_resetStreamHC() before first use. Static allocation should only be used in combination with static linking. If you want to use LZ4 as a DLL, please use construction functions below, which are future-proof. */ LZ4_streamHC_t* LZ4_createStreamHC(void); int LZ4_freeStreamHC (LZ4_streamHC_t* streamHCPtr); /* These functions create and release memory for LZ4 HC streaming state. Newly created states are already initialized. Existing state space can be re-used anytime using LZ4_resetStreamHC(). If you use LZ4 as a DLL, use these functions instead of static structure allocation, to avoid size mismatch between different versions. */ void LZ4_resetStreamHC (LZ4_streamHC_t* streamHCPtr, int compressionLevel); int LZ4_loadDictHC (LZ4_streamHC_t* streamHCPtr, const char* dictionary, int dictSize); int LZ4_compress_HC_continue (LZ4_streamHC_t* streamHCPtr, const char* src, char* dst, int srcSize, int maxDstSize); int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, int maxDictSize); /* These functions compress data in successive blocks of any size, using previous blocks as dictionary. One key assumption is that previous blocks (up to 64 KB) remain read-accessible while compressing next blocks. There is an exception for ring buffers, which can be smaller 64 KB. Such case is automatically detected and correctly handled by LZ4_compress_HC_continue(). Before starting compression, state must be properly initialized, using LZ4_resetStreamHC(). A first "fictional block" can then be designated as initial dictionary, using LZ4_loadDictHC() (Optional). Then, use LZ4_compress_HC_continue() to compress each successive block. It works like LZ4_compress_HC(), but use previous memory blocks as dictionary to improve compression. Previous memory blocks (including initial dictionary when present) must remain accessible and unmodified during compression. As a reminder, size 'dst' buffer to handle worst cases, using LZ4_compressBound(), to ensure success of compression operation. If, for any reason, previous data blocks can't be preserved unmodified in memory during next compression block, you must save it to a safer memory space, using LZ4_saveDictHC(). Return value of LZ4_saveDictHC() is the size of dictionary effectively saved into 'safeBuffer'. */ /************************************** * Deprecated Functions **************************************/ /* Deprecate Warnings */ /* Should these warnings messages be a problem, it is generally possible to disable them, with -Wno-deprecated-declarations for gcc or _CRT_SECURE_NO_WARNINGS in Visual for example. You can also define LZ4_DEPRECATE_WARNING_DEFBLOCK. */ #ifndef LZ4_DEPRECATE_WARNING_DEFBLOCK # define LZ4_DEPRECATE_WARNING_DEFBLOCK # define LZ4_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) # if (LZ4_GCC_VERSION >= 405) || defined(__clang__) # define LZ4_DEPRECATED(message) __attribute__((deprecated(message))) # elif (LZ4_GCC_VERSION >= 301) # define LZ4_DEPRECATED(message) __attribute__((deprecated)) # elif defined(_MSC_VER) # define LZ4_DEPRECATED(message) __declspec(deprecated(message)) # else # pragma message("WARNING: You need to implement LZ4_DEPRECATED for this compiler") # define LZ4_DEPRECATED(message) # endif #endif // LZ4_DEPRECATE_WARNING_DEFBLOCK /* compression functions */ /* these functions are planned to trigger warning messages by r131 approximately */ int LZ4_compressHC (const char* source, char* dest, int inputSize); int LZ4_compressHC_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize); int LZ4_compressHC2 (const char* source, char* dest, int inputSize, int compressionLevel); int LZ4_compressHC2_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); int LZ4_compressHC_withStateHC (void* state, const char* source, char* dest, int inputSize); int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); int LZ4_compressHC2_withStateHC (void* state, const char* source, char* dest, int inputSize, int compressionLevel); int LZ4_compressHC2_limitedOutput_withStateHC(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); int LZ4_compressHC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize); int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize); /* Streaming functions following the older model; should no longer be used */ LZ4_DEPRECATED("use LZ4_createStreamHC() instead") void* LZ4_createHC (char* inputBuffer); LZ4_DEPRECATED("use LZ4_saveDictHC() instead") char* LZ4_slideInputBufferHC (void* LZ4HC_Data); LZ4_DEPRECATED("use LZ4_freeStreamHC() instead") int LZ4_freeHC (void* LZ4HC_Data); LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel); LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); LZ4_DEPRECATED("use LZ4_createStreamHC() instead") int LZ4_sizeofStreamStateHC(void); LZ4_DEPRECATED("use LZ4_resetStreamHC() instead") int LZ4_resetStreamStateHC(void* state, char* inputBuffer); #if defined (__cplusplus) } #endif c-blosc-1.7.0/internal-complibs/snappy-1.1.1/000077500000000000000000000000001254622165100204355ustar00rootroot00000000000000c-blosc-1.7.0/internal-complibs/snappy-1.1.1/add-version.patch000066400000000000000000000011461254622165100236730ustar00rootroot00000000000000diff --git a/internal-complibs/snappy-1.1.1/snappy-c.h b/internal-complibs/snappy-1.1.1/snappy-c.h index c6c2a86..eabe3ae 100644 --- a/internal-complibs/snappy-1.1.1/snappy-c.h +++ b/internal-complibs/snappy-1.1.1/snappy-c.h @@ -37,6 +37,14 @@ extern "C" { #endif +// The next is for getting the Snappy version even if used the C API +// Please note that this is only defined in the Blosc sources of Snappy. +#define SNAPPY_MAJOR 1 +#define SNAPPY_MINOR 1 +#define SNAPPY_PATCHLEVEL 1 +#define SNAPPY_VERSION \ + ((SNAPPY_MAJOR << 16) | (SNAPPY_MINOR << 8) | SNAPPY_PATCHLEVEL) + #include /* c-blosc-1.7.0/internal-complibs/snappy-1.1.1/msvc1.patch000066400000000000000000000007021254622165100225060ustar00rootroot00000000000000--- a/internal-complibs/snappy-1.1.1/snappy.h +++ b/internal-complibs/snappy-1.1.1/snappy.h @@ -44,6 +44,14 @@ #include "snappy-stubs-public.h" +// Windows does not define ssize_t by default. This is a workaround. +// Please note that this is only defined in the Blosc sources of Snappy. +#if defined(_WIN32) && !defined(__MINGW32__) +#include +typedef SSIZE_T ssize_t; +#endif + + namespace snappy { class Source; class Sink; c-blosc-1.7.0/internal-complibs/snappy-1.1.1/msvc2.patch000066400000000000000000000016471254622165100225200ustar00rootroot00000000000000diff --git a/internal-complibs/snappy-1.1.1/snappy-stubs-public.h b/internal-complibs/snappy-1.1.1/snappy-stubs-public.h index ecda439..4cc8965 100644 --- a/internal-complibs/snappy-1.1.1/snappy-stubs-public.h +++ b/internal-complibs/snappy-1.1.1/snappy-stubs-public.h @@ -36,8 +36,21 @@ #ifndef UTIL_SNAPPY_OPENSOURCE_SNAPPY_STUBS_PUBLIC_H_ #define UTIL_SNAPPY_OPENSOURCE_SNAPPY_STUBS_PUBLIC_H_ -#if 1 +// MSVC 2008 does not include stdint.h. This is a workaround by Mark W. +// Please note that this is only defined in the Blosc sources of Snappy. +#if !defined(_MSC_VER) || _MSC_VER >= 1600 #include +#else +typedef signed char int8_t; +typedef short int16_t; +typedef int int32_t; +typedef __int64 int64_t; +typedef ptrdiff_t intptr_t; +typedef unsigned char uint8_t; +typedef unsigned short uint16_t; +typedef unsigned int uint32_t; +typedef unsigned __int64 uint64_t; +typedef size_t uintptr_t; #endif #if 1 c-blosc-1.7.0/internal-complibs/snappy-1.1.1/snappy-c.cc000066400000000000000000000070651254622165100225060ustar00rootroot00000000000000// Copyright 2011 Martin Gieseking . // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "snappy.h" #include "snappy-c.h" extern "C" { snappy_status snappy_compress(const char* input, size_t input_length, char* compressed, size_t *compressed_length) { if (*compressed_length < snappy_max_compressed_length(input_length)) { return SNAPPY_BUFFER_TOO_SMALL; } snappy::RawCompress(input, input_length, compressed, compressed_length); return SNAPPY_OK; } snappy_status snappy_uncompress(const char* compressed, size_t compressed_length, char* uncompressed, size_t* uncompressed_length) { size_t real_uncompressed_length; if (!snappy::GetUncompressedLength(compressed, compressed_length, &real_uncompressed_length)) { return SNAPPY_INVALID_INPUT; } if (*uncompressed_length < real_uncompressed_length) { return SNAPPY_BUFFER_TOO_SMALL; } if (!snappy::RawUncompress(compressed, compressed_length, uncompressed)) { return SNAPPY_INVALID_INPUT; } *uncompressed_length = real_uncompressed_length; return SNAPPY_OK; } size_t snappy_max_compressed_length(size_t source_length) { return snappy::MaxCompressedLength(source_length); } snappy_status snappy_uncompressed_length(const char *compressed, size_t compressed_length, size_t *result) { if (snappy::GetUncompressedLength(compressed, compressed_length, result)) { return SNAPPY_OK; } else { return SNAPPY_INVALID_INPUT; } } snappy_status snappy_validate_compressed_buffer(const char *compressed, size_t compressed_length) { if (snappy::IsValidCompressedBuffer(compressed, compressed_length)) { return SNAPPY_OK; } else { return SNAPPY_INVALID_INPUT; } } } // extern "C" c-blosc-1.7.0/internal-complibs/snappy-1.1.1/snappy-c.h000066400000000000000000000131111254622165100223350ustar00rootroot00000000000000/* * Copyright 2011 Martin Gieseking . * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are * met: * * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following disclaimer * in the documentation and/or other materials provided with the * distribution. * * Neither the name of Google Inc. nor the names of its * contributors may be used to endorse or promote products derived from * this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * Plain C interface (a wrapper around the C++ implementation). */ #ifndef UTIL_SNAPPY_OPENSOURCE_SNAPPY_C_H_ #define UTIL_SNAPPY_OPENSOURCE_SNAPPY_C_H_ #ifdef __cplusplus extern "C" { #endif // The next is for getting the Snappy version even if used the C API. // Please note that this is only defined in the Blosc sources of Snappy. #define SNAPPY_MAJOR 1 #define SNAPPY_MINOR 1 #define SNAPPY_PATCHLEVEL 1 #define SNAPPY_VERSION \ ((SNAPPY_MAJOR << 16) | (SNAPPY_MINOR << 8) | SNAPPY_PATCHLEVEL) #include /* * Return values; see the documentation for each function to know * what each can return. */ typedef enum { SNAPPY_OK = 0, SNAPPY_INVALID_INPUT = 1, SNAPPY_BUFFER_TOO_SMALL = 2 } snappy_status; /* * Takes the data stored in "input[0..input_length-1]" and stores * it in the array pointed to by "compressed". * * signals the space available in "compressed". * If it is not at least equal to "snappy_max_compressed_length(input_length)", * SNAPPY_BUFFER_TOO_SMALL is returned. After successful compression, * contains the true length of the compressed output, * and SNAPPY_OK is returned. * * Example: * size_t output_length = snappy_max_compressed_length(input_length); * char* output = (char*)malloc(output_length); * if (snappy_compress(input, input_length, output, &output_length) * == SNAPPY_OK) { * ... Process(output, output_length) ... * } * free(output); */ snappy_status snappy_compress(const char* input, size_t input_length, char* compressed, size_t* compressed_length); /* * Given data in "compressed[0..compressed_length-1]" generated by * calling the snappy_compress routine, this routine stores * the uncompressed data to * uncompressed[0..uncompressed_length-1]. * Returns failure (a value not equal to SNAPPY_OK) if the message * is corrupted and could not be decrypted. * * signals the space available in "uncompressed". * If it is not at least equal to the value returned by * snappy_uncompressed_length for this stream, SNAPPY_BUFFER_TOO_SMALL * is returned. After successful decompression, * contains the true length of the decompressed output. * * Example: * size_t output_length; * if (snappy_uncompressed_length(input, input_length, &output_length) * != SNAPPY_OK) { * ... fail ... * } * char* output = (char*)malloc(output_length); * if (snappy_uncompress(input, input_length, output, &output_length) * == SNAPPY_OK) { * ... Process(output, output_length) ... * } * free(output); */ snappy_status snappy_uncompress(const char* compressed, size_t compressed_length, char* uncompressed, size_t* uncompressed_length); /* * Returns the maximal size of the compressed representation of * input data that is "source_length" bytes in length. */ size_t snappy_max_compressed_length(size_t source_length); /* * REQUIRES: "compressed[]" was produced by snappy_compress() * Returns SNAPPY_OK and stores the length of the uncompressed data in * *result normally. Returns SNAPPY_INVALID_INPUT on parsing error. * This operation takes O(1) time. */ snappy_status snappy_uncompressed_length(const char* compressed, size_t compressed_length, size_t* result); /* * Check if the contents of "compressed[]" can be uncompressed successfully. * Does not return the uncompressed data; if so, returns SNAPPY_OK, * or if not, returns SNAPPY_INVALID_INPUT. * Takes time proportional to compressed_length, but is usually at least a * factor of four faster than actual decompression. */ snappy_status snappy_validate_compressed_buffer(const char* compressed, size_t compressed_length); #ifdef __cplusplus } // extern "C" #endif #endif /* UTIL_SNAPPY_OPENSOURCE_SNAPPY_C_H_ */ c-blosc-1.7.0/internal-complibs/snappy-1.1.1/snappy-internal.h000066400000000000000000000125241254622165100237360ustar00rootroot00000000000000// Copyright 2008 Google Inc. All Rights Reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // // Internals shared between the Snappy implementation and its unittest. #ifndef UTIL_SNAPPY_SNAPPY_INTERNAL_H_ #define UTIL_SNAPPY_SNAPPY_INTERNAL_H_ #include "snappy-stubs-internal.h" namespace snappy { namespace internal { class WorkingMemory { public: WorkingMemory() : large_table_(NULL) { } ~WorkingMemory() { delete[] large_table_; } // Allocates and clears a hash table using memory in "*this", // stores the number of buckets in "*table_size" and returns a pointer to // the base of the hash table. uint16* GetHashTable(size_t input_size, int* table_size); private: uint16 small_table_[1<<10]; // 2KB uint16* large_table_; // Allocated only when needed DISALLOW_COPY_AND_ASSIGN(WorkingMemory); }; // Flat array compression that does not emit the "uncompressed length" // prefix. Compresses "input" string to the "*op" buffer. // // REQUIRES: "input_length <= kBlockSize" // REQUIRES: "op" points to an array of memory that is at least // "MaxCompressedLength(input_length)" in size. // REQUIRES: All elements in "table[0..table_size-1]" are initialized to zero. // REQUIRES: "table_size" is a power of two // // Returns an "end" pointer into "op" buffer. // "end - op" is the compressed size of "input". char* CompressFragment(const char* input, size_t input_length, char* op, uint16* table, const int table_size); // Return the largest n such that // // s1[0,n-1] == s2[0,n-1] // and n <= (s2_limit - s2). // // Does not read *s2_limit or beyond. // Does not read *(s1 + (s2_limit - s2)) or beyond. // Requires that s2_limit >= s2. // // Separate implementation for x86_64, for speed. Uses the fact that // x86_64 is little endian. #if defined(ARCH_K8) static inline int FindMatchLength(const char* s1, const char* s2, const char* s2_limit) { assert(s2_limit >= s2); int matched = 0; // Find out how long the match is. We loop over the data 64 bits at a // time until we find a 64-bit block that doesn't match; then we find // the first non-matching bit and use that to calculate the total // length of the match. while (PREDICT_TRUE(s2 <= s2_limit - 8)) { if (PREDICT_FALSE(UNALIGNED_LOAD64(s2) == UNALIGNED_LOAD64(s1 + matched))) { s2 += 8; matched += 8; } else { // On current (mid-2008) Opteron models there is a 3% more // efficient code sequence to find the first non-matching byte. // However, what follows is ~10% better on Intel Core 2 and newer, // and we expect AMD's bsf instruction to improve. uint64 x = UNALIGNED_LOAD64(s2) ^ UNALIGNED_LOAD64(s1 + matched); int matching_bits = Bits::FindLSBSetNonZero64(x); matched += matching_bits >> 3; return matched; } } while (PREDICT_TRUE(s2 < s2_limit)) { if (PREDICT_TRUE(s1[matched] == *s2)) { ++s2; ++matched; } else { return matched; } } return matched; } #else static inline int FindMatchLength(const char* s1, const char* s2, const char* s2_limit) { // Implementation based on the x86-64 version, above. assert(s2_limit >= s2); int matched = 0; while (s2 <= s2_limit - 4 && UNALIGNED_LOAD32(s2) == UNALIGNED_LOAD32(s1 + matched)) { s2 += 4; matched += 4; } if (LittleEndian::IsLittleEndian() && s2 <= s2_limit - 4) { uint32 x = UNALIGNED_LOAD32(s2) ^ UNALIGNED_LOAD32(s1 + matched); int matching_bits = Bits::FindLSBSetNonZero(x); matched += matching_bits >> 3; } else { while ((s2 < s2_limit) && (s1[matched] == *s2)) { ++s2; ++matched; } } return matched; } #endif } // end namespace internal } // end namespace snappy #endif // UTIL_SNAPPY_SNAPPY_INTERNAL_H_ c-blosc-1.7.0/internal-complibs/snappy-1.1.1/snappy-sinksource.cc000066400000000000000000000044771254622165100244550ustar00rootroot00000000000000// Copyright 2011 Google Inc. All Rights Reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include "snappy-sinksource.h" namespace snappy { Source::~Source() { } Sink::~Sink() { } char* Sink::GetAppendBuffer(size_t length, char* scratch) { return scratch; } ByteArraySource::~ByteArraySource() { } size_t ByteArraySource::Available() const { return left_; } const char* ByteArraySource::Peek(size_t* len) { *len = left_; return ptr_; } void ByteArraySource::Skip(size_t n) { left_ -= n; ptr_ += n; } UncheckedByteArraySink::~UncheckedByteArraySink() { } void UncheckedByteArraySink::Append(const char* data, size_t n) { // Do no copying if the caller filled in the result of GetAppendBuffer() if (data != dest_) { memcpy(dest_, data, n); } dest_ += n; } char* UncheckedByteArraySink::GetAppendBuffer(size_t len, char* scratch) { return dest_; } } c-blosc-1.7.0/internal-complibs/snappy-1.1.1/snappy-sinksource.h000066400000000000000000000115031254622165100243030ustar00rootroot00000000000000// Copyright 2011 Google Inc. All Rights Reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #ifndef UTIL_SNAPPY_SNAPPY_SINKSOURCE_H_ #define UTIL_SNAPPY_SNAPPY_SINKSOURCE_H_ #include namespace snappy { // A Sink is an interface that consumes a sequence of bytes. class Sink { public: Sink() { } virtual ~Sink(); // Append "bytes[0,n-1]" to this. virtual void Append(const char* bytes, size_t n) = 0; // Returns a writable buffer of the specified length for appending. // May return a pointer to the caller-owned scratch buffer which // must have at least the indicated length. The returned buffer is // only valid until the next operation on this Sink. // // After writing at most "length" bytes, call Append() with the // pointer returned from this function and the number of bytes // written. Many Append() implementations will avoid copying // bytes if this function returned an internal buffer. // // If a non-scratch buffer is returned, the caller may only pass a // prefix of it to Append(). That is, it is not correct to pass an // interior pointer of the returned array to Append(). // // The default implementation always returns the scratch buffer. virtual char* GetAppendBuffer(size_t length, char* scratch); private: // No copying Sink(const Sink&); void operator=(const Sink&); }; // A Source is an interface that yields a sequence of bytes class Source { public: Source() { } virtual ~Source(); // Return the number of bytes left to read from the source virtual size_t Available() const = 0; // Peek at the next flat region of the source. Does not reposition // the source. The returned region is empty iff Available()==0. // // Returns a pointer to the beginning of the region and store its // length in *len. // // The returned region is valid until the next call to Skip() or // until this object is destroyed, whichever occurs first. // // The returned region may be larger than Available() (for example // if this ByteSource is a view on a substring of a larger source). // The caller is responsible for ensuring that it only reads the // Available() bytes. virtual const char* Peek(size_t* len) = 0; // Skip the next n bytes. Invalidates any buffer returned by // a previous call to Peek(). // REQUIRES: Available() >= n virtual void Skip(size_t n) = 0; private: // No copying Source(const Source&); void operator=(const Source&); }; // A Source implementation that yields the contents of a flat array class ByteArraySource : public Source { public: ByteArraySource(const char* p, size_t n) : ptr_(p), left_(n) { } virtual ~ByteArraySource(); virtual size_t Available() const; virtual const char* Peek(size_t* len); virtual void Skip(size_t n); private: const char* ptr_; size_t left_; }; // A Sink implementation that writes to a flat array without any bound checks. class UncheckedByteArraySink : public Sink { public: explicit UncheckedByteArraySink(char* dest) : dest_(dest) { } virtual ~UncheckedByteArraySink(); virtual void Append(const char* data, size_t n); virtual char* GetAppendBuffer(size_t len, char* scratch); // Return the current output pointer so that a caller can see how // many bytes were produced. // Note: this is not a Sink method. char* CurrentDestination() const { return dest_; } private: char* dest_; }; } #endif // UTIL_SNAPPY_SNAPPY_SINKSOURCE_H_ c-blosc-1.7.0/internal-complibs/snappy-1.1.1/snappy-stubs-internal.cc000066400000000000000000000034431254622165100252320ustar00rootroot00000000000000// Copyright 2011 Google Inc. All Rights Reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include #include #include "snappy-stubs-internal.h" namespace snappy { void Varint::Append32(string* s, uint32 value) { char buf[Varint::kMax32]; const char* p = Varint::Encode32(buf, value); s->append(buf, p - buf); } } // namespace snappy c-blosc-1.7.0/internal-complibs/snappy-1.1.1/snappy-stubs-internal.h000066400000000000000000000357231254622165100251020ustar00rootroot00000000000000// Copyright 2011 Google Inc. All Rights Reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // // Various stubs for the open-source version of Snappy. #ifndef UTIL_SNAPPY_OPENSOURCE_SNAPPY_STUBS_INTERNAL_H_ #define UTIL_SNAPPY_OPENSOURCE_SNAPPY_STUBS_INTERNAL_H_ #ifdef HAVE_CONFIG_H #include "config.h" #endif #include #include #include #include #ifdef HAVE_SYS_MMAN_H #include #endif #include "snappy-stubs-public.h" #if defined(__x86_64__) // Enable 64-bit optimized versions of some routines. #define ARCH_K8 1 #endif // Needed by OS X, among others. #ifndef MAP_ANONYMOUS #define MAP_ANONYMOUS MAP_ANON #endif // Pull in std::min, std::ostream, and the likes. This is safe because this // header file is never used from any public header files. using namespace std; // The size of an array, if known at compile-time. // Will give unexpected results if used on a pointer. // We undefine it first, since some compilers already have a definition. #ifdef ARRAYSIZE #undef ARRAYSIZE #endif #define ARRAYSIZE(a) (sizeof(a) / sizeof(*(a))) // Static prediction hints. #ifdef HAVE_BUILTIN_EXPECT #define PREDICT_FALSE(x) (__builtin_expect(x, 0)) #define PREDICT_TRUE(x) (__builtin_expect(!!(x), 1)) #else #define PREDICT_FALSE(x) x #define PREDICT_TRUE(x) x #endif // This is only used for recomputing the tag byte table used during // decompression; for simplicity we just remove it from the open-source // version (anyone who wants to regenerate it can just do the call // themselves within main()). #define DEFINE_bool(flag_name, default_value, description) \ bool FLAGS_ ## flag_name = default_value #define DECLARE_bool(flag_name) \ extern bool FLAGS_ ## flag_name namespace snappy { static const uint32 kuint32max = static_cast(0xFFFFFFFF); static const int64 kint64max = static_cast(0x7FFFFFFFFFFFFFFFLL); // Potentially unaligned loads and stores. // x86 and PowerPC can simply do these loads and stores native. #if defined(__i386__) || defined(__x86_64__) || defined(__powerpc__) #define UNALIGNED_LOAD16(_p) (*reinterpret_cast(_p)) #define UNALIGNED_LOAD32(_p) (*reinterpret_cast(_p)) #define UNALIGNED_LOAD64(_p) (*reinterpret_cast(_p)) #define UNALIGNED_STORE16(_p, _val) (*reinterpret_cast(_p) = (_val)) #define UNALIGNED_STORE32(_p, _val) (*reinterpret_cast(_p) = (_val)) #define UNALIGNED_STORE64(_p, _val) (*reinterpret_cast(_p) = (_val)) // ARMv7 and newer support native unaligned accesses, but only of 16-bit // and 32-bit values (not 64-bit); older versions either raise a fatal signal, // do an unaligned read and rotate the words around a bit, or do the reads very // slowly (trip through kernel mode). There's no simple #define that says just // “ARMv7 or higher”, so we have to filter away all ARMv5 and ARMv6 // sub-architectures. // // This is a mess, but there's not much we can do about it. #elif defined(__arm__) && \ !defined(__ARM_ARCH_4__) && \ !defined(__ARM_ARCH_4T__) && \ !defined(__ARM_ARCH_5__) && \ !defined(__ARM_ARCH_5T__) && \ !defined(__ARM_ARCH_5TE__) && \ !defined(__ARM_ARCH_5TEJ__) && \ !defined(__ARM_ARCH_6__) && \ !defined(__ARM_ARCH_6J__) && \ !defined(__ARM_ARCH_6K__) && \ !defined(__ARM_ARCH_6Z__) && \ !defined(__ARM_ARCH_6ZK__) && \ !defined(__ARM_ARCH_6T2__) #define UNALIGNED_LOAD16(_p) (*reinterpret_cast(_p)) #define UNALIGNED_LOAD32(_p) (*reinterpret_cast(_p)) #define UNALIGNED_STORE16(_p, _val) (*reinterpret_cast(_p) = (_val)) #define UNALIGNED_STORE32(_p, _val) (*reinterpret_cast(_p) = (_val)) // TODO(user): NEON supports unaligned 64-bit loads and stores. // See if that would be more efficient on platforms supporting it, // at least for copies. inline uint64 UNALIGNED_LOAD64(const void *p) { uint64 t; memcpy(&t, p, sizeof t); return t; } inline void UNALIGNED_STORE64(void *p, uint64 v) { memcpy(p, &v, sizeof v); } #else // These functions are provided for architectures that don't support // unaligned loads and stores. inline uint16 UNALIGNED_LOAD16(const void *p) { uint16 t; memcpy(&t, p, sizeof t); return t; } inline uint32 UNALIGNED_LOAD32(const void *p) { uint32 t; memcpy(&t, p, sizeof t); return t; } inline uint64 UNALIGNED_LOAD64(const void *p) { uint64 t; memcpy(&t, p, sizeof t); return t; } inline void UNALIGNED_STORE16(void *p, uint16 v) { memcpy(p, &v, sizeof v); } inline void UNALIGNED_STORE32(void *p, uint32 v) { memcpy(p, &v, sizeof v); } inline void UNALIGNED_STORE64(void *p, uint64 v) { memcpy(p, &v, sizeof v); } #endif // This can be more efficient than UNALIGNED_LOAD64 + UNALIGNED_STORE64 // on some platforms, in particular ARM. inline void UnalignedCopy64(const void *src, void *dst) { if (sizeof(void *) == 8) { UNALIGNED_STORE64(dst, UNALIGNED_LOAD64(src)); } else { const char *src_char = reinterpret_cast(src); char *dst_char = reinterpret_cast(dst); UNALIGNED_STORE32(dst_char, UNALIGNED_LOAD32(src_char)); UNALIGNED_STORE32(dst_char + 4, UNALIGNED_LOAD32(src_char + 4)); } } // The following guarantees declaration of the byte swap functions. #ifdef WORDS_BIGENDIAN #ifdef HAVE_SYS_BYTEORDER_H #include #endif #ifdef HAVE_SYS_ENDIAN_H #include #endif #ifdef _MSC_VER #include #define bswap_16(x) _byteswap_ushort(x) #define bswap_32(x) _byteswap_ulong(x) #define bswap_64(x) _byteswap_uint64(x) #elif defined(__APPLE__) // Mac OS X / Darwin features #include #define bswap_16(x) OSSwapInt16(x) #define bswap_32(x) OSSwapInt32(x) #define bswap_64(x) OSSwapInt64(x) #elif defined(HAVE_BYTESWAP_H) #include #elif defined(bswap32) // FreeBSD defines bswap{16,32,64} in (already #included). #define bswap_16(x) bswap16(x) #define bswap_32(x) bswap32(x) #define bswap_64(x) bswap64(x) #elif defined(BSWAP_64) // Solaris 10 defines BSWAP_{16,32,64} in (already #included). #define bswap_16(x) BSWAP_16(x) #define bswap_32(x) BSWAP_32(x) #define bswap_64(x) BSWAP_64(x) #else inline uint16 bswap_16(uint16 x) { return (x << 8) | (x >> 8); } inline uint32 bswap_32(uint32 x) { x = ((x & 0xff00ff00UL) >> 8) | ((x & 0x00ff00ffUL) << 8); return (x >> 16) | (x << 16); } inline uint64 bswap_64(uint64 x) { x = ((x & 0xff00ff00ff00ff00ULL) >> 8) | ((x & 0x00ff00ff00ff00ffULL) << 8); x = ((x & 0xffff0000ffff0000ULL) >> 16) | ((x & 0x0000ffff0000ffffULL) << 16); return (x >> 32) | (x << 32); } #endif #endif // WORDS_BIGENDIAN // Convert to little-endian storage, opposite of network format. // Convert x from host to little endian: x = LittleEndian.FromHost(x); // convert x from little endian to host: x = LittleEndian.ToHost(x); // // Store values into unaligned memory converting to little endian order: // LittleEndian.Store16(p, x); // // Load unaligned values stored in little endian converting to host order: // x = LittleEndian.Load16(p); class LittleEndian { public: // Conversion functions. #ifdef WORDS_BIGENDIAN static uint16 FromHost16(uint16 x) { return bswap_16(x); } static uint16 ToHost16(uint16 x) { return bswap_16(x); } static uint32 FromHost32(uint32 x) { return bswap_32(x); } static uint32 ToHost32(uint32 x) { return bswap_32(x); } static bool IsLittleEndian() { return false; } #else // !defined(WORDS_BIGENDIAN) static uint16 FromHost16(uint16 x) { return x; } static uint16 ToHost16(uint16 x) { return x; } static uint32 FromHost32(uint32 x) { return x; } static uint32 ToHost32(uint32 x) { return x; } static bool IsLittleEndian() { return true; } #endif // !defined(WORDS_BIGENDIAN) // Functions to do unaligned loads and stores in little-endian order. static uint16 Load16(const void *p) { return ToHost16(UNALIGNED_LOAD16(p)); } static void Store16(void *p, uint16 v) { UNALIGNED_STORE16(p, FromHost16(v)); } static uint32 Load32(const void *p) { return ToHost32(UNALIGNED_LOAD32(p)); } static void Store32(void *p, uint32 v) { UNALIGNED_STORE32(p, FromHost32(v)); } }; // Some bit-manipulation functions. class Bits { public: // Return floor(log2(n)) for positive integer n. Returns -1 iff n == 0. static int Log2Floor(uint32 n); // Return the first set least / most significant bit, 0-indexed. Returns an // undefined value if n == 0. FindLSBSetNonZero() is similar to ffs() except // that it's 0-indexed. static int FindLSBSetNonZero(uint32 n); static int FindLSBSetNonZero64(uint64 n); private: DISALLOW_COPY_AND_ASSIGN(Bits); }; #ifdef HAVE_BUILTIN_CTZ inline int Bits::Log2Floor(uint32 n) { return n == 0 ? -1 : 31 ^ __builtin_clz(n); } inline int Bits::FindLSBSetNonZero(uint32 n) { return __builtin_ctz(n); } inline int Bits::FindLSBSetNonZero64(uint64 n) { return __builtin_ctzll(n); } #else // Portable versions. inline int Bits::Log2Floor(uint32 n) { if (n == 0) return -1; int log = 0; uint32 value = n; for (int i = 4; i >= 0; --i) { int shift = (1 << i); uint32 x = value >> shift; if (x != 0) { value = x; log += shift; } } assert(value == 1); return log; } inline int Bits::FindLSBSetNonZero(uint32 n) { int rc = 31; for (int i = 4, shift = 1 << 4; i >= 0; --i) { const uint32 x = n << shift; if (x != 0) { n = x; rc -= shift; } shift >>= 1; } return rc; } // FindLSBSetNonZero64() is defined in terms of FindLSBSetNonZero(). inline int Bits::FindLSBSetNonZero64(uint64 n) { const uint32 bottombits = static_cast(n); if (bottombits == 0) { // Bottom bits are zero, so scan in top bits return 32 + FindLSBSetNonZero(static_cast(n >> 32)); } else { return FindLSBSetNonZero(bottombits); } } #endif // End portable versions. // Variable-length integer encoding. class Varint { public: // Maximum lengths of varint encoding of uint32. static const int kMax32 = 5; // Attempts to parse a varint32 from a prefix of the bytes in [ptr,limit-1]. // Never reads a character at or beyond limit. If a valid/terminated varint32 // was found in the range, stores it in *OUTPUT and returns a pointer just // past the last byte of the varint32. Else returns NULL. On success, // "result <= limit". static const char* Parse32WithLimit(const char* ptr, const char* limit, uint32* OUTPUT); // REQUIRES "ptr" points to a buffer of length sufficient to hold "v". // EFFECTS Encodes "v" into "ptr" and returns a pointer to the // byte just past the last encoded byte. static char* Encode32(char* ptr, uint32 v); // EFFECTS Appends the varint representation of "value" to "*s". static void Append32(string* s, uint32 value); }; inline const char* Varint::Parse32WithLimit(const char* p, const char* l, uint32* OUTPUT) { const unsigned char* ptr = reinterpret_cast(p); const unsigned char* limit = reinterpret_cast(l); uint32 b, result; if (ptr >= limit) return NULL; b = *(ptr++); result = b & 127; if (b < 128) goto done; if (ptr >= limit) return NULL; b = *(ptr++); result |= (b & 127) << 7; if (b < 128) goto done; if (ptr >= limit) return NULL; b = *(ptr++); result |= (b & 127) << 14; if (b < 128) goto done; if (ptr >= limit) return NULL; b = *(ptr++); result |= (b & 127) << 21; if (b < 128) goto done; if (ptr >= limit) return NULL; b = *(ptr++); result |= (b & 127) << 28; if (b < 16) goto done; return NULL; // Value is too long to be a varint32 done: *OUTPUT = result; return reinterpret_cast(ptr); } inline char* Varint::Encode32(char* sptr, uint32 v) { // Operate on characters as unsigneds unsigned char* ptr = reinterpret_cast(sptr); static const int B = 128; if (v < (1<<7)) { *(ptr++) = v; } else if (v < (1<<14)) { *(ptr++) = v | B; *(ptr++) = v>>7; } else if (v < (1<<21)) { *(ptr++) = v | B; *(ptr++) = (v>>7) | B; *(ptr++) = v>>14; } else if (v < (1<<28)) { *(ptr++) = v | B; *(ptr++) = (v>>7) | B; *(ptr++) = (v>>14) | B; *(ptr++) = v>>21; } else { *(ptr++) = v | B; *(ptr++) = (v>>7) | B; *(ptr++) = (v>>14) | B; *(ptr++) = (v>>21) | B; *(ptr++) = v>>28; } return reinterpret_cast(ptr); } // If you know the internal layout of the std::string in use, you can // replace this function with one that resizes the string without // filling the new space with zeros (if applicable) -- // it will be non-portable but faster. inline void STLStringResizeUninitialized(string* s, size_t new_size) { s->resize(new_size); } // Return a mutable char* pointing to a string's internal buffer, // which may not be null-terminated. Writing through this pointer will // modify the string. // // string_as_array(&str)[i] is valid for 0 <= i < str.size() until the // next call to a string method that invalidates iterators. // // As of 2006-04, there is no standard-blessed way of getting a // mutable reference to a string's internal buffer. However, issue 530 // (http://www.open-std.org/JTC1/SC22/WG21/docs/lwg-defects.html#530) // proposes this as the method. It will officially be part of the standard // for C++0x. This should already work on all current implementations. inline char* string_as_array(string* str) { return str->empty() ? NULL : &*str->begin(); } } // namespace snappy #endif // UTIL_SNAPPY_OPENSOURCE_SNAPPY_STUBS_INTERNAL_H_ c-blosc-1.7.0/internal-complibs/snappy-1.1.1/snappy-stubs-public.h000066400000000000000000000070731254622165100245410ustar00rootroot00000000000000// Copyright 2011 Google Inc. All Rights Reserved. // Author: sesse@google.com (Steinar H. Gunderson) // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // // Various type stubs for the open-source version of Snappy. // // This file cannot include config.h, as it is included from snappy.h, // which is a public header. Instead, snappy-stubs-public.h is generated by // from snappy-stubs-public.h.in at configure time. #ifndef UTIL_SNAPPY_OPENSOURCE_SNAPPY_STUBS_PUBLIC_H_ #define UTIL_SNAPPY_OPENSOURCE_SNAPPY_STUBS_PUBLIC_H_ // MSVC 2008 does not include stdint.h. This is a workaround by Mark W. // Please note that this is only defined in the Blosc sources of Snappy. #if !defined(_MSC_VER) || _MSC_VER >= 1600 #include #else typedef signed char int8_t; typedef short int16_t; typedef int int32_t; typedef __int64 int64_t; typedef ptrdiff_t intptr_t; typedef unsigned char uint8_t; typedef unsigned short uint16_t; typedef unsigned int uint32_t; typedef unsigned __int64 uint64_t; typedef size_t uintptr_t; #endif #if 1 #include #endif #if 0 #include #endif #define SNAPPY_MAJOR 1 #define SNAPPY_MINOR 1 #define SNAPPY_PATCHLEVEL 1 #define SNAPPY_VERSION \ ((SNAPPY_MAJOR << 16) | (SNAPPY_MINOR << 8) | SNAPPY_PATCHLEVEL) #include namespace snappy { #if 1 typedef int8_t int8; typedef uint8_t uint8; typedef int16_t int16; typedef uint16_t uint16; typedef int32_t int32; typedef uint32_t uint32; typedef int64_t int64; typedef uint64_t uint64; #else typedef signed char int8; typedef unsigned char uint8; typedef short int16; typedef unsigned short uint16; typedef int int32; typedef unsigned int uint32; typedef long long int64; typedef unsigned long long uint64; #endif typedef std::string string; #define DISALLOW_COPY_AND_ASSIGN(TypeName) \ TypeName(const TypeName&); \ void operator=(const TypeName&) #if !0 // Windows does not have an iovec type, yet the concept is universally useful. // It is simple to define it ourselves, so we put it inside our own namespace. struct iovec { void* iov_base; size_t iov_len; }; #endif } // namespace snappy #endif // UTIL_SNAPPY_OPENSOURCE_SNAPPY_STUBS_PUBLIC_H_ c-blosc-1.7.0/internal-complibs/snappy-1.1.1/snappy.cc000066400000000000000000001321131254622165100222570ustar00rootroot00000000000000// Copyright 2005 Google Inc. All Rights Reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "snappy.h" #include "snappy-internal.h" #include "snappy-sinksource.h" #include #include #include #include namespace snappy { // Any hash function will produce a valid compressed bitstream, but a good // hash function reduces the number of collisions and thus yields better // compression for compressible input, and more speed for incompressible // input. Of course, it doesn't hurt if the hash function is reasonably fast // either, as it gets called a lot. static inline uint32 HashBytes(uint32 bytes, int shift) { uint32 kMul = 0x1e35a7bd; return (bytes * kMul) >> shift; } static inline uint32 Hash(const char* p, int shift) { return HashBytes(UNALIGNED_LOAD32(p), shift); } size_t MaxCompressedLength(size_t source_len) { // Compressed data can be defined as: // compressed := item* literal* // item := literal* copy // // The trailing literal sequence has a space blowup of at most 62/60 // since a literal of length 60 needs one tag byte + one extra byte // for length information. // // Item blowup is trickier to measure. Suppose the "copy" op copies // 4 bytes of data. Because of a special check in the encoding code, // we produce a 4-byte copy only if the offset is < 65536. Therefore // the copy op takes 3 bytes to encode, and this type of item leads // to at most the 62/60 blowup for representing literals. // // Suppose the "copy" op copies 5 bytes of data. If the offset is big // enough, it will take 5 bytes to encode the copy op. Therefore the // worst case here is a one-byte literal followed by a five-byte copy. // I.e., 6 bytes of input turn into 7 bytes of "compressed" data. // // This last factor dominates the blowup, so the final estimate is: return 32 + source_len + source_len/6; } enum { LITERAL = 0, COPY_1_BYTE_OFFSET = 1, // 3 bit length + 3 bits of offset in opcode COPY_2_BYTE_OFFSET = 2, COPY_4_BYTE_OFFSET = 3 }; static const int kMaximumTagLength = 5; // COPY_4_BYTE_OFFSET plus the actual offset. // Copy "len" bytes from "src" to "op", one byte at a time. Used for // handling COPY operations where the input and output regions may // overlap. For example, suppose: // src == "ab" // op == src + 2 // len == 20 // After IncrementalCopy(src, op, len), the result will have // eleven copies of "ab" // ababababababababababab // Note that this does not match the semantics of either memcpy() // or memmove(). static inline void IncrementalCopy(const char* src, char* op, ssize_t len) { assert(len > 0); do { *op++ = *src++; } while (--len > 0); } // Equivalent to IncrementalCopy except that it can write up to ten extra // bytes after the end of the copy, and that it is faster. // // The main part of this loop is a simple copy of eight bytes at a time until // we've copied (at least) the requested amount of bytes. However, if op and // src are less than eight bytes apart (indicating a repeating pattern of // length < 8), we first need to expand the pattern in order to get the correct // results. For instance, if the buffer looks like this, with the eight-byte // and patterns marked as intervals: // // abxxxxxxxxxxxx // [------] src // [------] op // // a single eight-byte copy from to will repeat the pattern once, // after which we can move two bytes without moving : // // ababxxxxxxxxxx // [------] src // [------] op // // and repeat the exercise until the two no longer overlap. // // This allows us to do very well in the special case of one single byte // repeated many times, without taking a big hit for more general cases. // // The worst case of extra writing past the end of the match occurs when // op - src == 1 and len == 1; the last copy will read from byte positions // [0..7] and write to [4..11], whereas it was only supposed to write to // position 1. Thus, ten excess bytes. namespace { const int kMaxIncrementCopyOverflow = 10; inline void IncrementalCopyFastPath(const char* src, char* op, ssize_t len) { while (op - src < 8) { UnalignedCopy64(src, op); len -= op - src; op += op - src; } while (len > 0) { UnalignedCopy64(src, op); src += 8; op += 8; len -= 8; } } } // namespace static inline char* EmitLiteral(char* op, const char* literal, int len, bool allow_fast_path) { int n = len - 1; // Zero-length literals are disallowed if (n < 60) { // Fits in tag byte *op++ = LITERAL | (n << 2); // The vast majority of copies are below 16 bytes, for which a // call to memcpy is overkill. This fast path can sometimes // copy up to 15 bytes too much, but that is okay in the // main loop, since we have a bit to go on for both sides: // // - The input will always have kInputMarginBytes = 15 extra // available bytes, as long as we're in the main loop, and // if not, allow_fast_path = false. // - The output will always have 32 spare bytes (see // MaxCompressedLength). if (allow_fast_path && len <= 16) { UnalignedCopy64(literal, op); UnalignedCopy64(literal + 8, op + 8); return op + len; } } else { // Encode in upcoming bytes char* base = op; int count = 0; op++; while (n > 0) { *op++ = n & 0xff; n >>= 8; count++; } assert(count >= 1); assert(count <= 4); *base = LITERAL | ((59+count) << 2); } memcpy(op, literal, len); return op + len; } static inline char* EmitCopyLessThan64(char* op, size_t offset, int len) { assert(len <= 64); assert(len >= 4); assert(offset < 65536); if ((len < 12) && (offset < 2048)) { size_t len_minus_4 = len - 4; assert(len_minus_4 < 8); // Must fit in 3 bits *op++ = COPY_1_BYTE_OFFSET + ((len_minus_4) << 2) + ((offset >> 8) << 5); *op++ = offset & 0xff; } else { *op++ = COPY_2_BYTE_OFFSET + ((len-1) << 2); LittleEndian::Store16(op, offset); op += 2; } return op; } static inline char* EmitCopy(char* op, size_t offset, int len) { // Emit 64 byte copies but make sure to keep at least four bytes reserved while (len >= 68) { op = EmitCopyLessThan64(op, offset, 64); len -= 64; } // Emit an extra 60 byte copy if have too much data to fit in one copy if (len > 64) { op = EmitCopyLessThan64(op, offset, 60); len -= 60; } // Emit remainder op = EmitCopyLessThan64(op, offset, len); return op; } bool GetUncompressedLength(const char* start, size_t n, size_t* result) { uint32 v = 0; const char* limit = start + n; if (Varint::Parse32WithLimit(start, limit, &v) != NULL) { *result = v; return true; } else { return false; } } namespace internal { uint16* WorkingMemory::GetHashTable(size_t input_size, int* table_size) { // Use smaller hash table when input.size() is smaller, since we // fill the table, incurring O(hash table size) overhead for // compression, and if the input is short, we won't need that // many hash table entries anyway. assert(kMaxHashTableSize >= 256); size_t htsize = 256; while (htsize < kMaxHashTableSize && htsize < input_size) { htsize <<= 1; } uint16* table; if (htsize <= ARRAYSIZE(small_table_)) { table = small_table_; } else { if (large_table_ == NULL) { large_table_ = new uint16[kMaxHashTableSize]; } table = large_table_; } *table_size = htsize; memset(table, 0, htsize * sizeof(*table)); return table; } } // end namespace internal // For 0 <= offset <= 4, GetUint32AtOffset(GetEightBytesAt(p), offset) will // equal UNALIGNED_LOAD32(p + offset). Motivation: On x86-64 hardware we have // empirically found that overlapping loads such as // UNALIGNED_LOAD32(p) ... UNALIGNED_LOAD32(p+1) ... UNALIGNED_LOAD32(p+2) // are slower than UNALIGNED_LOAD64(p) followed by shifts and casts to uint32. // // We have different versions for 64- and 32-bit; ideally we would avoid the // two functions and just inline the UNALIGNED_LOAD64 call into // GetUint32AtOffset, but GCC (at least not as of 4.6) is seemingly not clever // enough to avoid loading the value multiple times then. For 64-bit, the load // is done when GetEightBytesAt() is called, whereas for 32-bit, the load is // done at GetUint32AtOffset() time. #ifdef ARCH_K8 typedef uint64 EightBytesReference; static inline EightBytesReference GetEightBytesAt(const char* ptr) { return UNALIGNED_LOAD64(ptr); } static inline uint32 GetUint32AtOffset(uint64 v, int offset) { assert(offset >= 0); assert(offset <= 4); return v >> (LittleEndian::IsLittleEndian() ? 8 * offset : 32 - 8 * offset); } #else typedef const char* EightBytesReference; static inline EightBytesReference GetEightBytesAt(const char* ptr) { return ptr; } static inline uint32 GetUint32AtOffset(const char* v, int offset) { assert(offset >= 0); assert(offset <= 4); return UNALIGNED_LOAD32(v + offset); } #endif // Flat array compression that does not emit the "uncompressed length" // prefix. Compresses "input" string to the "*op" buffer. // // REQUIRES: "input" is at most "kBlockSize" bytes long. // REQUIRES: "op" points to an array of memory that is at least // "MaxCompressedLength(input.size())" in size. // REQUIRES: All elements in "table[0..table_size-1]" are initialized to zero. // REQUIRES: "table_size" is a power of two // // Returns an "end" pointer into "op" buffer. // "end - op" is the compressed size of "input". namespace internal { char* CompressFragment(const char* input, size_t input_size, char* op, uint16* table, const int table_size) { // "ip" is the input pointer, and "op" is the output pointer. const char* ip = input; assert(input_size <= kBlockSize); assert((table_size & (table_size - 1)) == 0); // table must be power of two const int shift = 32 - Bits::Log2Floor(table_size); assert(static_cast(kuint32max >> shift) == table_size - 1); const char* ip_end = input + input_size; const char* base_ip = ip; // Bytes in [next_emit, ip) will be emitted as literal bytes. Or // [next_emit, ip_end) after the main loop. const char* next_emit = ip; const size_t kInputMarginBytes = 15; if (PREDICT_TRUE(input_size >= kInputMarginBytes)) { const char* ip_limit = input + input_size - kInputMarginBytes; for (uint32 next_hash = Hash(++ip, shift); ; ) { assert(next_emit < ip); // The body of this loop calls EmitLiteral once and then EmitCopy one or // more times. (The exception is that when we're close to exhausting // the input we goto emit_remainder.) // // In the first iteration of this loop we're just starting, so // there's nothing to copy, so calling EmitLiteral once is // necessary. And we only start a new iteration when the // current iteration has determined that a call to EmitLiteral will // precede the next call to EmitCopy (if any). // // Step 1: Scan forward in the input looking for a 4-byte-long match. // If we get close to exhausting the input then goto emit_remainder. // // Heuristic match skipping: If 32 bytes are scanned with no matches // found, start looking only at every other byte. If 32 more bytes are // scanned, look at every third byte, etc.. When a match is found, // immediately go back to looking at every byte. This is a small loss // (~5% performance, ~0.1% density) for compressible data due to more // bookkeeping, but for non-compressible data (such as JPEG) it's a huge // win since the compressor quickly "realizes" the data is incompressible // and doesn't bother looking for matches everywhere. // // The "skip" variable keeps track of how many bytes there are since the // last match; dividing it by 32 (ie. right-shifting by five) gives the // number of bytes to move ahead for each iteration. uint32 skip = 32; const char* next_ip = ip; const char* candidate; do { ip = next_ip; uint32 hash = next_hash; assert(hash == Hash(ip, shift)); uint32 bytes_between_hash_lookups = skip++ >> 5; next_ip = ip + bytes_between_hash_lookups; if (PREDICT_FALSE(next_ip > ip_limit)) { goto emit_remainder; } next_hash = Hash(next_ip, shift); candidate = base_ip + table[hash]; assert(candidate >= base_ip); assert(candidate < ip); table[hash] = ip - base_ip; } while (PREDICT_TRUE(UNALIGNED_LOAD32(ip) != UNALIGNED_LOAD32(candidate))); // Step 2: A 4-byte match has been found. We'll later see if more // than 4 bytes match. But, prior to the match, input // bytes [next_emit, ip) are unmatched. Emit them as "literal bytes." assert(next_emit + 16 <= ip_end); op = EmitLiteral(op, next_emit, ip - next_emit, true); // Step 3: Call EmitCopy, and then see if another EmitCopy could // be our next move. Repeat until we find no match for the // input immediately after what was consumed by the last EmitCopy call. // // If we exit this loop normally then we need to call EmitLiteral next, // though we don't yet know how big the literal will be. We handle that // by proceeding to the next iteration of the main loop. We also can exit // this loop via goto if we get close to exhausting the input. EightBytesReference input_bytes; uint32 candidate_bytes = 0; do { // We have a 4-byte match at ip, and no need to emit any // "literal bytes" prior to ip. const char* base = ip; int matched = 4 + FindMatchLength(candidate + 4, ip + 4, ip_end); ip += matched; size_t offset = base - candidate; assert(0 == memcmp(base, candidate, matched)); op = EmitCopy(op, offset, matched); // We could immediately start working at ip now, but to improve // compression we first update table[Hash(ip - 1, ...)]. const char* insert_tail = ip - 1; next_emit = ip; if (PREDICT_FALSE(ip >= ip_limit)) { goto emit_remainder; } input_bytes = GetEightBytesAt(insert_tail); uint32 prev_hash = HashBytes(GetUint32AtOffset(input_bytes, 0), shift); table[prev_hash] = ip - base_ip - 1; uint32 cur_hash = HashBytes(GetUint32AtOffset(input_bytes, 1), shift); candidate = base_ip + table[cur_hash]; candidate_bytes = UNALIGNED_LOAD32(candidate); table[cur_hash] = ip - base_ip; } while (GetUint32AtOffset(input_bytes, 1) == candidate_bytes); next_hash = HashBytes(GetUint32AtOffset(input_bytes, 2), shift); ++ip; } } emit_remainder: // Emit the remaining bytes as a literal if (next_emit < ip_end) { op = EmitLiteral(op, next_emit, ip_end - next_emit, false); } return op; } } // end namespace internal // Signature of output types needed by decompression code. // The decompression code is templatized on a type that obeys this // signature so that we do not pay virtual function call overhead in // the middle of a tight decompression loop. // // class DecompressionWriter { // public: // // Called before decompression // void SetExpectedLength(size_t length); // // // Called after decompression // bool CheckLength() const; // // // Called repeatedly during decompression // bool Append(const char* ip, size_t length); // bool AppendFromSelf(uint32 offset, size_t length); // // // The rules for how TryFastAppend differs from Append are somewhat // // convoluted: // // // // - TryFastAppend is allowed to decline (return false) at any // // time, for any reason -- just "return false" would be // // a perfectly legal implementation of TryFastAppend. // // The intention is for TryFastAppend to allow a fast path // // in the common case of a small append. // // - TryFastAppend is allowed to read up to bytes // // from the input buffer, whereas Append is allowed to read // // . However, if it returns true, it must leave // // at least five (kMaximumTagLength) bytes in the input buffer // // afterwards, so that there is always enough space to read the // // next tag without checking for a refill. // // - TryFastAppend must always return decline (return false) // // if is 61 or more, as in this case the literal length is not // // decoded fully. In practice, this should not be a big problem, // // as it is unlikely that one would implement a fast path accepting // // this much data. // // // bool TryFastAppend(const char* ip, size_t available, size_t length); // }; // ----------------------------------------------------------------------- // Lookup table for decompression code. Generated by ComputeTable() below. // ----------------------------------------------------------------------- // Mapping from i in range [0,4] to a mask to extract the bottom 8*i bits static const uint32 wordmask[] = { 0u, 0xffu, 0xffffu, 0xffffffu, 0xffffffffu }; // Data stored per entry in lookup table: // Range Bits-used Description // ------------------------------------ // 1..64 0..7 Literal/copy length encoded in opcode byte // 0..7 8..10 Copy offset encoded in opcode byte / 256 // 0..4 11..13 Extra bytes after opcode // // We use eight bits for the length even though 7 would have sufficed // because of efficiency reasons: // (1) Extracting a byte is faster than a bit-field // (2) It properly aligns copy offset so we do not need a <<8 static const uint16 char_table[256] = { 0x0001, 0x0804, 0x1001, 0x2001, 0x0002, 0x0805, 0x1002, 0x2002, 0x0003, 0x0806, 0x1003, 0x2003, 0x0004, 0x0807, 0x1004, 0x2004, 0x0005, 0x0808, 0x1005, 0x2005, 0x0006, 0x0809, 0x1006, 0x2006, 0x0007, 0x080a, 0x1007, 0x2007, 0x0008, 0x080b, 0x1008, 0x2008, 0x0009, 0x0904, 0x1009, 0x2009, 0x000a, 0x0905, 0x100a, 0x200a, 0x000b, 0x0906, 0x100b, 0x200b, 0x000c, 0x0907, 0x100c, 0x200c, 0x000d, 0x0908, 0x100d, 0x200d, 0x000e, 0x0909, 0x100e, 0x200e, 0x000f, 0x090a, 0x100f, 0x200f, 0x0010, 0x090b, 0x1010, 0x2010, 0x0011, 0x0a04, 0x1011, 0x2011, 0x0012, 0x0a05, 0x1012, 0x2012, 0x0013, 0x0a06, 0x1013, 0x2013, 0x0014, 0x0a07, 0x1014, 0x2014, 0x0015, 0x0a08, 0x1015, 0x2015, 0x0016, 0x0a09, 0x1016, 0x2016, 0x0017, 0x0a0a, 0x1017, 0x2017, 0x0018, 0x0a0b, 0x1018, 0x2018, 0x0019, 0x0b04, 0x1019, 0x2019, 0x001a, 0x0b05, 0x101a, 0x201a, 0x001b, 0x0b06, 0x101b, 0x201b, 0x001c, 0x0b07, 0x101c, 0x201c, 0x001d, 0x0b08, 0x101d, 0x201d, 0x001e, 0x0b09, 0x101e, 0x201e, 0x001f, 0x0b0a, 0x101f, 0x201f, 0x0020, 0x0b0b, 0x1020, 0x2020, 0x0021, 0x0c04, 0x1021, 0x2021, 0x0022, 0x0c05, 0x1022, 0x2022, 0x0023, 0x0c06, 0x1023, 0x2023, 0x0024, 0x0c07, 0x1024, 0x2024, 0x0025, 0x0c08, 0x1025, 0x2025, 0x0026, 0x0c09, 0x1026, 0x2026, 0x0027, 0x0c0a, 0x1027, 0x2027, 0x0028, 0x0c0b, 0x1028, 0x2028, 0x0029, 0x0d04, 0x1029, 0x2029, 0x002a, 0x0d05, 0x102a, 0x202a, 0x002b, 0x0d06, 0x102b, 0x202b, 0x002c, 0x0d07, 0x102c, 0x202c, 0x002d, 0x0d08, 0x102d, 0x202d, 0x002e, 0x0d09, 0x102e, 0x202e, 0x002f, 0x0d0a, 0x102f, 0x202f, 0x0030, 0x0d0b, 0x1030, 0x2030, 0x0031, 0x0e04, 0x1031, 0x2031, 0x0032, 0x0e05, 0x1032, 0x2032, 0x0033, 0x0e06, 0x1033, 0x2033, 0x0034, 0x0e07, 0x1034, 0x2034, 0x0035, 0x0e08, 0x1035, 0x2035, 0x0036, 0x0e09, 0x1036, 0x2036, 0x0037, 0x0e0a, 0x1037, 0x2037, 0x0038, 0x0e0b, 0x1038, 0x2038, 0x0039, 0x0f04, 0x1039, 0x2039, 0x003a, 0x0f05, 0x103a, 0x203a, 0x003b, 0x0f06, 0x103b, 0x203b, 0x003c, 0x0f07, 0x103c, 0x203c, 0x0801, 0x0f08, 0x103d, 0x203d, 0x1001, 0x0f09, 0x103e, 0x203e, 0x1801, 0x0f0a, 0x103f, 0x203f, 0x2001, 0x0f0b, 0x1040, 0x2040 }; // In debug mode, allow optional computation of the table at startup. // Also, check that the decompression table is correct. #ifndef NDEBUG DEFINE_bool(snappy_dump_decompression_table, false, "If true, we print the decompression table at startup."); static uint16 MakeEntry(unsigned int extra, unsigned int len, unsigned int copy_offset) { // Check that all of the fields fit within the allocated space assert(extra == (extra & 0x7)); // At most 3 bits assert(copy_offset == (copy_offset & 0x7)); // At most 3 bits assert(len == (len & 0x7f)); // At most 7 bits return len | (copy_offset << 8) | (extra << 11); } static void ComputeTable() { uint16 dst[256]; // Place invalid entries in all places to detect missing initialization int assigned = 0; for (int i = 0; i < 256; i++) { dst[i] = 0xffff; } // Small LITERAL entries. We store (len-1) in the top 6 bits. for (unsigned int len = 1; len <= 60; len++) { dst[LITERAL | ((len-1) << 2)] = MakeEntry(0, len, 0); assigned++; } // Large LITERAL entries. We use 60..63 in the high 6 bits to // encode the number of bytes of length info that follow the opcode. for (unsigned int extra_bytes = 1; extra_bytes <= 4; extra_bytes++) { // We set the length field in the lookup table to 1 because extra // bytes encode len-1. dst[LITERAL | ((extra_bytes+59) << 2)] = MakeEntry(extra_bytes, 1, 0); assigned++; } // COPY_1_BYTE_OFFSET. // // The tag byte in the compressed data stores len-4 in 3 bits, and // offset/256 in 5 bits. offset%256 is stored in the next byte. // // This format is used for length in range [4..11] and offset in // range [0..2047] for (unsigned int len = 4; len < 12; len++) { for (unsigned int offset = 0; offset < 2048; offset += 256) { dst[COPY_1_BYTE_OFFSET | ((len-4)<<2) | ((offset>>8)<<5)] = MakeEntry(1, len, offset>>8); assigned++; } } // COPY_2_BYTE_OFFSET. // Tag contains len-1 in top 6 bits, and offset in next two bytes. for (unsigned int len = 1; len <= 64; len++) { dst[COPY_2_BYTE_OFFSET | ((len-1)<<2)] = MakeEntry(2, len, 0); assigned++; } // COPY_4_BYTE_OFFSET. // Tag contents len-1 in top 6 bits, and offset in next four bytes. for (unsigned int len = 1; len <= 64; len++) { dst[COPY_4_BYTE_OFFSET | ((len-1)<<2)] = MakeEntry(4, len, 0); assigned++; } // Check that each entry was initialized exactly once. if (assigned != 256) { fprintf(stderr, "ComputeTable: assigned only %d of 256\n", assigned); abort(); } for (int i = 0; i < 256; i++) { if (dst[i] == 0xffff) { fprintf(stderr, "ComputeTable: did not assign byte %d\n", i); abort(); } } if (FLAGS_snappy_dump_decompression_table) { printf("static const uint16 char_table[256] = {\n "); for (int i = 0; i < 256; i++) { printf("0x%04x%s", dst[i], ((i == 255) ? "\n" : (((i%8) == 7) ? ",\n " : ", "))); } printf("};\n"); } // Check that computed table matched recorded table for (int i = 0; i < 256; i++) { if (dst[i] != char_table[i]) { fprintf(stderr, "ComputeTable: byte %d: computed (%x), expect (%x)\n", i, static_cast(dst[i]), static_cast(char_table[i])); abort(); } } } #endif /* !NDEBUG */ // Helper class for decompression class SnappyDecompressor { private: Source* reader_; // Underlying source of bytes to decompress const char* ip_; // Points to next buffered byte const char* ip_limit_; // Points just past buffered bytes uint32 peeked_; // Bytes peeked from reader (need to skip) bool eof_; // Hit end of input without an error? char scratch_[kMaximumTagLength]; // See RefillTag(). // Ensure that all of the tag metadata for the next tag is available // in [ip_..ip_limit_-1]. Also ensures that [ip,ip+4] is readable even // if (ip_limit_ - ip_ < 5). // // Returns true on success, false on error or end of input. bool RefillTag(); public: explicit SnappyDecompressor(Source* reader) : reader_(reader), ip_(NULL), ip_limit_(NULL), peeked_(0), eof_(false) { } ~SnappyDecompressor() { // Advance past any bytes we peeked at from the reader reader_->Skip(peeked_); } // Returns true iff we have hit the end of the input without an error. bool eof() const { return eof_; } // Read the uncompressed length stored at the start of the compressed data. // On succcess, stores the length in *result and returns true. // On failure, returns false. bool ReadUncompressedLength(uint32* result) { assert(ip_ == NULL); // Must not have read anything yet // Length is encoded in 1..5 bytes *result = 0; uint32 shift = 0; while (true) { if (shift >= 32) return false; size_t n; const char* ip = reader_->Peek(&n); if (n == 0) return false; const unsigned char c = *(reinterpret_cast(ip)); reader_->Skip(1); *result |= static_cast(c & 0x7f) << shift; if (c < 128) { break; } shift += 7; } return true; } // Process the next item found in the input. // Returns true if successful, false on error or end of input. template void DecompressAllTags(Writer* writer) { const char* ip = ip_; // We could have put this refill fragment only at the beginning of the loop. // However, duplicating it at the end of each branch gives the compiler more // scope to optimize the expression based on the local // context, which overall increases speed. #define MAYBE_REFILL() \ if (ip_limit_ - ip < kMaximumTagLength) { \ ip_ = ip; \ if (!RefillTag()) return; \ ip = ip_; \ } MAYBE_REFILL(); for ( ;; ) { const unsigned char c = *(reinterpret_cast(ip++)); if ((c & 0x3) == LITERAL) { size_t literal_length = (c >> 2) + 1u; if (writer->TryFastAppend(ip, ip_limit_ - ip, literal_length)) { assert(literal_length < 61); ip += literal_length; // NOTE(user): There is no MAYBE_REFILL() here, as TryFastAppend() // will not return true unless there's already at least five spare // bytes in addition to the literal. continue; } if (PREDICT_FALSE(literal_length >= 61)) { // Long literal. const size_t literal_length_length = literal_length - 60; literal_length = (LittleEndian::Load32(ip) & wordmask[literal_length_length]) + 1; ip += literal_length_length; } size_t avail = ip_limit_ - ip; while (avail < literal_length) { if (!writer->Append(ip, avail)) return; literal_length -= avail; reader_->Skip(peeked_); size_t n; ip = reader_->Peek(&n); avail = n; peeked_ = avail; if (avail == 0) return; // Premature end of input ip_limit_ = ip + avail; } if (!writer->Append(ip, literal_length)) { return; } ip += literal_length; MAYBE_REFILL(); } else { const uint32 entry = char_table[c]; const uint32 trailer = LittleEndian::Load32(ip) & wordmask[entry >> 11]; const uint32 length = entry & 0xff; ip += entry >> 11; // copy_offset/256 is encoded in bits 8..10. By just fetching // those bits, we get copy_offset (since the bit-field starts at // bit 8). const uint32 copy_offset = entry & 0x700; if (!writer->AppendFromSelf(copy_offset + trailer, length)) { return; } MAYBE_REFILL(); } } #undef MAYBE_REFILL } }; bool SnappyDecompressor::RefillTag() { const char* ip = ip_; if (ip == ip_limit_) { // Fetch a new fragment from the reader reader_->Skip(peeked_); // All peeked bytes are used up size_t n; ip = reader_->Peek(&n); peeked_ = n; if (n == 0) { eof_ = true; return false; } ip_limit_ = ip + n; } // Read the tag character assert(ip < ip_limit_); const unsigned char c = *(reinterpret_cast(ip)); const uint32 entry = char_table[c]; const uint32 needed = (entry >> 11) + 1; // +1 byte for 'c' assert(needed <= sizeof(scratch_)); // Read more bytes from reader if needed uint32 nbuf = ip_limit_ - ip; if (nbuf < needed) { // Stitch together bytes from ip and reader to form the word // contents. We store the needed bytes in "scratch_". They // will be consumed immediately by the caller since we do not // read more than we need. memmove(scratch_, ip, nbuf); reader_->Skip(peeked_); // All peeked bytes are used up peeked_ = 0; while (nbuf < needed) { size_t length; const char* src = reader_->Peek(&length); if (length == 0) return false; uint32 to_add = min(needed - nbuf, length); memcpy(scratch_ + nbuf, src, to_add); nbuf += to_add; reader_->Skip(to_add); } assert(nbuf == needed); ip_ = scratch_; ip_limit_ = scratch_ + needed; } else if (nbuf < kMaximumTagLength) { // Have enough bytes, but move into scratch_ so that we do not // read past end of input memmove(scratch_, ip, nbuf); reader_->Skip(peeked_); // All peeked bytes are used up peeked_ = 0; ip_ = scratch_; ip_limit_ = scratch_ + nbuf; } else { // Pass pointer to buffer returned by reader_. ip_ = ip; } return true; } template static bool InternalUncompress(Source* r, Writer* writer) { // Read the uncompressed length from the front of the compressed input SnappyDecompressor decompressor(r); uint32 uncompressed_len = 0; if (!decompressor.ReadUncompressedLength(&uncompressed_len)) return false; return InternalUncompressAllTags(&decompressor, writer, uncompressed_len); } template static bool InternalUncompressAllTags(SnappyDecompressor* decompressor, Writer* writer, uint32 uncompressed_len) { writer->SetExpectedLength(uncompressed_len); // Process the entire input decompressor->DecompressAllTags(writer); return (decompressor->eof() && writer->CheckLength()); } bool GetUncompressedLength(Source* source, uint32* result) { SnappyDecompressor decompressor(source); return decompressor.ReadUncompressedLength(result); } size_t Compress(Source* reader, Sink* writer) { size_t written = 0; size_t N = reader->Available(); char ulength[Varint::kMax32]; char* p = Varint::Encode32(ulength, N); writer->Append(ulength, p-ulength); written += (p - ulength); internal::WorkingMemory wmem; char* scratch = NULL; char* scratch_output = NULL; while (N > 0) { // Get next block to compress (without copying if possible) size_t fragment_size; const char* fragment = reader->Peek(&fragment_size); assert(fragment_size != 0); // premature end of input const size_t num_to_read = min(N, kBlockSize); size_t bytes_read = fragment_size; size_t pending_advance = 0; if (bytes_read >= num_to_read) { // Buffer returned by reader is large enough pending_advance = num_to_read; fragment_size = num_to_read; } else { // Read into scratch buffer if (scratch == NULL) { // If this is the last iteration, we want to allocate N bytes // of space, otherwise the max possible kBlockSize space. // num_to_read contains exactly the correct value scratch = new char[num_to_read]; } memcpy(scratch, fragment, bytes_read); reader->Skip(bytes_read); while (bytes_read < num_to_read) { fragment = reader->Peek(&fragment_size); size_t n = min(fragment_size, num_to_read - bytes_read); memcpy(scratch + bytes_read, fragment, n); bytes_read += n; reader->Skip(n); } assert(bytes_read == num_to_read); fragment = scratch; fragment_size = num_to_read; } assert(fragment_size == num_to_read); // Get encoding table for compression int table_size; uint16* table = wmem.GetHashTable(num_to_read, &table_size); // Compress input_fragment and append to dest const int max_output = MaxCompressedLength(num_to_read); // Need a scratch buffer for the output, in case the byte sink doesn't // have room for us directly. if (scratch_output == NULL) { scratch_output = new char[max_output]; } else { // Since we encode kBlockSize regions followed by a region // which is <= kBlockSize in length, a previously allocated // scratch_output[] region is big enough for this iteration. } char* dest = writer->GetAppendBuffer(max_output, scratch_output); char* end = internal::CompressFragment(fragment, fragment_size, dest, table, table_size); writer->Append(dest, end - dest); written += (end - dest); N -= num_to_read; reader->Skip(pending_advance); } delete[] scratch; delete[] scratch_output; return written; } // ----------------------------------------------------------------------- // IOVec interfaces // ----------------------------------------------------------------------- // A type that writes to an iovec. // Note that this is not a "ByteSink", but a type that matches the // Writer template argument to SnappyDecompressor::DecompressAllTags(). class SnappyIOVecWriter { private: const struct iovec* output_iov_; const size_t output_iov_count_; // We are currently writing into output_iov_[curr_iov_index_]. int curr_iov_index_; // Bytes written to output_iov_[curr_iov_index_] so far. size_t curr_iov_written_; // Total bytes decompressed into output_iov_ so far. size_t total_written_; // Maximum number of bytes that will be decompressed into output_iov_. size_t output_limit_; inline char* GetIOVecPointer(int index, size_t offset) { return reinterpret_cast(output_iov_[index].iov_base) + offset; } public: // Does not take ownership of iov. iov must be valid during the // entire lifetime of the SnappyIOVecWriter. inline SnappyIOVecWriter(const struct iovec* iov, size_t iov_count) : output_iov_(iov), output_iov_count_(iov_count), curr_iov_index_(0), curr_iov_written_(0), total_written_(0), output_limit_(-1) { } inline void SetExpectedLength(size_t len) { output_limit_ = len; } inline bool CheckLength() const { return total_written_ == output_limit_; } inline bool Append(const char* ip, size_t len) { if (total_written_ + len > output_limit_) { return false; } while (len > 0) { assert(curr_iov_written_ <= output_iov_[curr_iov_index_].iov_len); if (curr_iov_written_ >= output_iov_[curr_iov_index_].iov_len) { // This iovec is full. Go to the next one. if (curr_iov_index_ + 1 >= output_iov_count_) { return false; } curr_iov_written_ = 0; ++curr_iov_index_; } const size_t to_write = std::min( len, output_iov_[curr_iov_index_].iov_len - curr_iov_written_); memcpy(GetIOVecPointer(curr_iov_index_, curr_iov_written_), ip, to_write); curr_iov_written_ += to_write; total_written_ += to_write; ip += to_write; len -= to_write; } return true; } inline bool TryFastAppend(const char* ip, size_t available, size_t len) { const size_t space_left = output_limit_ - total_written_; if (len <= 16 && available >= 16 + kMaximumTagLength && space_left >= 16 && output_iov_[curr_iov_index_].iov_len - curr_iov_written_ >= 16) { // Fast path, used for the majority (about 95%) of invocations. char* ptr = GetIOVecPointer(curr_iov_index_, curr_iov_written_); UnalignedCopy64(ip, ptr); UnalignedCopy64(ip + 8, ptr + 8); curr_iov_written_ += len; total_written_ += len; return true; } return false; } inline bool AppendFromSelf(size_t offset, size_t len) { if (offset > total_written_ || offset == 0) { return false; } const size_t space_left = output_limit_ - total_written_; if (len > space_left) { return false; } // Locate the iovec from which we need to start the copy. int from_iov_index = curr_iov_index_; size_t from_iov_offset = curr_iov_written_; while (offset > 0) { if (from_iov_offset >= offset) { from_iov_offset -= offset; break; } offset -= from_iov_offset; --from_iov_index; assert(from_iov_index >= 0); from_iov_offset = output_iov_[from_iov_index].iov_len; } // Copy bytes starting from the iovec pointed to by from_iov_index to // the current iovec. while (len > 0) { assert(from_iov_index <= curr_iov_index_); if (from_iov_index != curr_iov_index_) { const size_t to_copy = std::min( output_iov_[from_iov_index].iov_len - from_iov_offset, len); Append(GetIOVecPointer(from_iov_index, from_iov_offset), to_copy); len -= to_copy; if (len > 0) { ++from_iov_index; from_iov_offset = 0; } } else { assert(curr_iov_written_ <= output_iov_[curr_iov_index_].iov_len); size_t to_copy = std::min(output_iov_[curr_iov_index_].iov_len - curr_iov_written_, len); if (to_copy == 0) { // This iovec is full. Go to the next one. if (curr_iov_index_ + 1 >= output_iov_count_) { return false; } ++curr_iov_index_; curr_iov_written_ = 0; continue; } if (to_copy > len) { to_copy = len; } IncrementalCopy(GetIOVecPointer(from_iov_index, from_iov_offset), GetIOVecPointer(curr_iov_index_, curr_iov_written_), to_copy); curr_iov_written_ += to_copy; from_iov_offset += to_copy; total_written_ += to_copy; len -= to_copy; } } return true; } }; bool RawUncompressToIOVec(const char* compressed, size_t compressed_length, const struct iovec* iov, size_t iov_cnt) { ByteArraySource reader(compressed, compressed_length); return RawUncompressToIOVec(&reader, iov, iov_cnt); } bool RawUncompressToIOVec(Source* compressed, const struct iovec* iov, size_t iov_cnt) { SnappyIOVecWriter output(iov, iov_cnt); return InternalUncompress(compressed, &output); } // ----------------------------------------------------------------------- // Flat array interfaces // ----------------------------------------------------------------------- // A type that writes to a flat array. // Note that this is not a "ByteSink", but a type that matches the // Writer template argument to SnappyDecompressor::DecompressAllTags(). class SnappyArrayWriter { private: char* base_; char* op_; char* op_limit_; public: inline explicit SnappyArrayWriter(char* dst) : base_(dst), op_(dst) { } inline void SetExpectedLength(size_t len) { op_limit_ = op_ + len; } inline bool CheckLength() const { return op_ == op_limit_; } inline bool Append(const char* ip, size_t len) { char* op = op_; const size_t space_left = op_limit_ - op; if (space_left < len) { return false; } memcpy(op, ip, len); op_ = op + len; return true; } inline bool TryFastAppend(const char* ip, size_t available, size_t len) { char* op = op_; const size_t space_left = op_limit_ - op; if (len <= 16 && available >= 16 + kMaximumTagLength && space_left >= 16) { // Fast path, used for the majority (about 95%) of invocations. UnalignedCopy64(ip, op); UnalignedCopy64(ip + 8, op + 8); op_ = op + len; return true; } else { return false; } } inline bool AppendFromSelf(size_t offset, size_t len) { char* op = op_; const size_t space_left = op_limit_ - op; // Check if we try to append from before the start of the buffer. // Normally this would just be a check for "produced < offset", // but "produced <= offset - 1u" is equivalent for every case // except the one where offset==0, where the right side will wrap around // to a very big number. This is convenient, as offset==0 is another // invalid case that we also want to catch, so that we do not go // into an infinite loop. assert(op >= base_); size_t produced = op - base_; if (produced <= offset - 1u) { return false; } if (len <= 16 && offset >= 8 && space_left >= 16) { // Fast path, used for the majority (70-80%) of dynamic invocations. UnalignedCopy64(op - offset, op); UnalignedCopy64(op - offset + 8, op + 8); } else { if (space_left >= len + kMaxIncrementCopyOverflow) { IncrementalCopyFastPath(op - offset, op, len); } else { if (space_left < len) { return false; } IncrementalCopy(op - offset, op, len); } } op_ = op + len; return true; } }; bool RawUncompress(const char* compressed, size_t n, char* uncompressed) { ByteArraySource reader(compressed, n); return RawUncompress(&reader, uncompressed); } bool RawUncompress(Source* compressed, char* uncompressed) { SnappyArrayWriter output(uncompressed); return InternalUncompress(compressed, &output); } bool Uncompress(const char* compressed, size_t n, string* uncompressed) { size_t ulength; if (!GetUncompressedLength(compressed, n, &ulength)) { return false; } // On 32-bit builds: max_size() < kuint32max. Check for that instead // of crashing (e.g., consider externally specified compressed data). if (ulength > uncompressed->max_size()) { return false; } STLStringResizeUninitialized(uncompressed, ulength); return RawUncompress(compressed, n, string_as_array(uncompressed)); } // A Writer that drops everything on the floor and just does validation class SnappyDecompressionValidator { private: size_t expected_; size_t produced_; public: inline SnappyDecompressionValidator() : produced_(0) { } inline void SetExpectedLength(size_t len) { expected_ = len; } inline bool CheckLength() const { return expected_ == produced_; } inline bool Append(const char* ip, size_t len) { produced_ += len; return produced_ <= expected_; } inline bool TryFastAppend(const char* ip, size_t available, size_t length) { return false; } inline bool AppendFromSelf(size_t offset, size_t len) { // See SnappyArrayWriter::AppendFromSelf for an explanation of // the "offset - 1u" trick. if (produced_ <= offset - 1u) return false; produced_ += len; return produced_ <= expected_; } }; bool IsValidCompressedBuffer(const char* compressed, size_t n) { ByteArraySource reader(compressed, n); SnappyDecompressionValidator writer; return InternalUncompress(&reader, &writer); } void RawCompress(const char* input, size_t input_length, char* compressed, size_t* compressed_length) { ByteArraySource reader(input, input_length); UncheckedByteArraySink writer(compressed); Compress(&reader, &writer); // Compute how many bytes were added *compressed_length = (writer.CurrentDestination() - compressed); } size_t Compress(const char* input, size_t input_length, string* compressed) { // Pre-grow the buffer to the max length of the compressed output compressed->resize(MaxCompressedLength(input_length)); size_t compressed_length; RawCompress(input, input_length, string_as_array(compressed), &compressed_length); compressed->resize(compressed_length); return compressed_length; } } // end namespace snappy c-blosc-1.7.0/internal-complibs/snappy-1.1.1/snappy.h000066400000000000000000000212641254622165100221250ustar00rootroot00000000000000// Copyright 2005 and onwards Google Inc. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following disclaimer // in the documentation and/or other materials provided with the // distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived from // this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // // A light-weight compression algorithm. It is designed for speed of // compression and decompression, rather than for the utmost in space // savings. // // For getting better compression ratios when you are compressing data // with long repeated sequences or compressing data that is similar to // other data, while still compressing fast, you might look at first // using BMDiff and then compressing the output of BMDiff with // Snappy. #ifndef UTIL_SNAPPY_SNAPPY_H__ #define UTIL_SNAPPY_SNAPPY_H__ #include #include #include "snappy-stubs-public.h" // Windows does not define ssize_t by default. This is a workaround. // Please note that this is only defined in the Blosc sources of Snappy. #if defined(_WIN32) && !defined(__MINGW32__) #include typedef SSIZE_T ssize_t; #endif namespace snappy { class Source; class Sink; // ------------------------------------------------------------------------ // Generic compression/decompression routines. // ------------------------------------------------------------------------ // Compress the bytes read from "*source" and append to "*sink". Return the // number of bytes written. size_t Compress(Source* source, Sink* sink); // Find the uncompressed length of the given stream, as given by the header. // Note that the true length could deviate from this; the stream could e.g. // be truncated. // // Also note that this leaves "*source" in a state that is unsuitable for // further operations, such as RawUncompress(). You will need to rewind // or recreate the source yourself before attempting any further calls. bool GetUncompressedLength(Source* source, uint32* result); // ------------------------------------------------------------------------ // Higher-level string based routines (should be sufficient for most users) // ------------------------------------------------------------------------ // Sets "*output" to the compressed version of "input[0,input_length-1]". // Original contents of *output are lost. // // REQUIRES: "input[]" is not an alias of "*output". size_t Compress(const char* input, size_t input_length, string* output); // Decompresses "compressed[0,compressed_length-1]" to "*uncompressed". // Original contents of "*uncompressed" are lost. // // REQUIRES: "compressed[]" is not an alias of "*uncompressed". // // returns false if the message is corrupted and could not be decompressed bool Uncompress(const char* compressed, size_t compressed_length, string* uncompressed); // ------------------------------------------------------------------------ // Lower-level character array based routines. May be useful for // efficiency reasons in certain circumstances. // ------------------------------------------------------------------------ // REQUIRES: "compressed" must point to an area of memory that is at // least "MaxCompressedLength(input_length)" bytes in length. // // Takes the data stored in "input[0..input_length]" and stores // it in the array pointed to by "compressed". // // "*compressed_length" is set to the length of the compressed output. // // Example: // char* output = new char[snappy::MaxCompressedLength(input_length)]; // size_t output_length; // RawCompress(input, input_length, output, &output_length); // ... Process(output, output_length) ... // delete [] output; void RawCompress(const char* input, size_t input_length, char* compressed, size_t* compressed_length); // Given data in "compressed[0..compressed_length-1]" generated by // calling the Snappy::Compress routine, this routine // stores the uncompressed data to // uncompressed[0..GetUncompressedLength(compressed)-1] // returns false if the message is corrupted and could not be decrypted bool RawUncompress(const char* compressed, size_t compressed_length, char* uncompressed); // Given data from the byte source 'compressed' generated by calling // the Snappy::Compress routine, this routine stores the uncompressed // data to // uncompressed[0..GetUncompressedLength(compressed,compressed_length)-1] // returns false if the message is corrupted and could not be decrypted bool RawUncompress(Source* compressed, char* uncompressed); // Given data in "compressed[0..compressed_length-1]" generated by // calling the Snappy::Compress routine, this routine // stores the uncompressed data to the iovec "iov". The number of physical // buffers in "iov" is given by iov_cnt and their cumulative size // must be at least GetUncompressedLength(compressed). The individual buffers // in "iov" must not overlap with each other. // // returns false if the message is corrupted and could not be decrypted bool RawUncompressToIOVec(const char* compressed, size_t compressed_length, const struct iovec* iov, size_t iov_cnt); // Given data from the byte source 'compressed' generated by calling // the Snappy::Compress routine, this routine stores the uncompressed // data to the iovec "iov". The number of physical // buffers in "iov" is given by iov_cnt and their cumulative size // must be at least GetUncompressedLength(compressed). The individual buffers // in "iov" must not overlap with each other. // // returns false if the message is corrupted and could not be decrypted bool RawUncompressToIOVec(Source* compressed, const struct iovec* iov, size_t iov_cnt); // Returns the maximal size of the compressed representation of // input data that is "source_bytes" bytes in length; size_t MaxCompressedLength(size_t source_bytes); // REQUIRES: "compressed[]" was produced by RawCompress() or Compress() // Returns true and stores the length of the uncompressed data in // *result normally. Returns false on parsing error. // This operation takes O(1) time. bool GetUncompressedLength(const char* compressed, size_t compressed_length, size_t* result); // Returns true iff the contents of "compressed[]" can be uncompressed // successfully. Does not return the uncompressed data. Takes // time proportional to compressed_length, but is usually at least // a factor of four faster than actual decompression. bool IsValidCompressedBuffer(const char* compressed, size_t compressed_length); // The size of a compression block. Note that many parts of the compression // code assumes that kBlockSize <= 65536; in particular, the hash table // can only store 16-bit offsets, and EmitCopy() also assumes the offset // is 65535 bytes or less. Note also that if you change this, it will // affect the framing format (see framing_format.txt). // // Note that there might be older data around that is compressed with larger // block sizes, so the decompression code should not rely on the // non-existence of long backreferences. static const int kBlockLog = 16; static const size_t kBlockSize = 1 << kBlockLog; static const int kMaxHashTableBits = 14; static const size_t kMaxHashTableSize = 1 << kMaxHashTableBits; } // end namespace snappy #endif // UTIL_SNAPPY_SNAPPY_H__ c-blosc-1.7.0/internal-complibs/zlib-1.2.8/000077500000000000000000000000001254622165100200735ustar00rootroot00000000000000c-blosc-1.7.0/internal-complibs/zlib-1.2.8/adler32.c000066400000000000000000000115501254622165100214750ustar00rootroot00000000000000/* adler32.c -- compute the Adler-32 checksum of a data stream * Copyright (C) 1995-2011 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ /* @(#) $Id$ */ #include "zutil.h" #define local static local uLong adler32_combine_ OF((uLong adler1, uLong adler2, z_off64_t len2)); #define BASE 65521 /* largest prime smaller than 65536 */ #define NMAX 5552 /* NMAX is the largest n such that 255n(n+1)/2 + (n+1)(BASE-1) <= 2^32-1 */ #define DO1(buf,i) {adler += (buf)[i]; sum2 += adler;} #define DO2(buf,i) DO1(buf,i); DO1(buf,i+1); #define DO4(buf,i) DO2(buf,i); DO2(buf,i+2); #define DO8(buf,i) DO4(buf,i); DO4(buf,i+4); #define DO16(buf) DO8(buf,0); DO8(buf,8); /* use NO_DIVIDE if your processor does not do division in hardware -- try it both ways to see which is faster */ #ifdef NO_DIVIDE /* note that this assumes BASE is 65521, where 65536 % 65521 == 15 (thank you to John Reiser for pointing this out) */ # define CHOP(a) \ do { \ unsigned long tmp = a >> 16; \ a &= 0xffffUL; \ a += (tmp << 4) - tmp; \ } while (0) # define MOD28(a) \ do { \ CHOP(a); \ if (a >= BASE) a -= BASE; \ } while (0) # define MOD(a) \ do { \ CHOP(a); \ MOD28(a); \ } while (0) # define MOD63(a) \ do { /* this assumes a is not negative */ \ z_off64_t tmp = a >> 32; \ a &= 0xffffffffL; \ a += (tmp << 8) - (tmp << 5) + tmp; \ tmp = a >> 16; \ a &= 0xffffL; \ a += (tmp << 4) - tmp; \ tmp = a >> 16; \ a &= 0xffffL; \ a += (tmp << 4) - tmp; \ if (a >= BASE) a -= BASE; \ } while (0) #else # define MOD(a) a %= BASE # define MOD28(a) a %= BASE # define MOD63(a) a %= BASE #endif /* ========================================================================= */ uLong ZEXPORT adler32(adler, buf, len) uLong adler; const Bytef *buf; uInt len; { unsigned long sum2; unsigned n; /* split Adler-32 into component sums */ sum2 = (adler >> 16) & 0xffff; adler &= 0xffff; /* in case user likes doing a byte at a time, keep it fast */ if (len == 1) { adler += buf[0]; if (adler >= BASE) adler -= BASE; sum2 += adler; if (sum2 >= BASE) sum2 -= BASE; return adler | (sum2 << 16); } /* initial Adler-32 value (deferred check for len == 1 speed) */ if (buf == Z_NULL) return 1L; /* in case short lengths are provided, keep it somewhat fast */ if (len < 16) { while (len--) { adler += *buf++; sum2 += adler; } if (adler >= BASE) adler -= BASE; MOD28(sum2); /* only added so many BASE's */ return adler | (sum2 << 16); } /* do length NMAX blocks -- requires just one modulo operation */ while (len >= NMAX) { len -= NMAX; n = NMAX / 16; /* NMAX is divisible by 16 */ do { DO16(buf); /* 16 sums unrolled */ buf += 16; } while (--n); MOD(adler); MOD(sum2); } /* do remaining bytes (less than NMAX, still just one modulo) */ if (len) { /* avoid modulos if none remaining */ while (len >= 16) { len -= 16; DO16(buf); buf += 16; } while (len--) { adler += *buf++; sum2 += adler; } MOD(adler); MOD(sum2); } /* return recombined sums */ return adler | (sum2 << 16); } /* ========================================================================= */ local uLong adler32_combine_(adler1, adler2, len2) uLong adler1; uLong adler2; z_off64_t len2; { unsigned long sum1; unsigned long sum2; unsigned rem; /* for negative len, return invalid adler32 as a clue for debugging */ if (len2 < 0) return 0xffffffffUL; /* the derivation of this formula is left as an exercise for the reader */ MOD63(len2); /* assumes len2 >= 0 */ rem = (unsigned)len2; sum1 = adler1 & 0xffff; sum2 = rem * sum1; MOD(sum2); sum1 += (adler2 & 0xffff) + BASE - 1; sum2 += ((adler1 >> 16) & 0xffff) + ((adler2 >> 16) & 0xffff) + BASE - rem; if (sum1 >= BASE) sum1 -= BASE; if (sum1 >= BASE) sum1 -= BASE; if (sum2 >= (BASE << 1)) sum2 -= (BASE << 1); if (sum2 >= BASE) sum2 -= BASE; return sum1 | (sum2 << 16); } /* ========================================================================= */ uLong ZEXPORT adler32_combine(adler1, adler2, len2) uLong adler1; uLong adler2; z_off_t len2; { return adler32_combine_(adler1, adler2, len2); } uLong ZEXPORT adler32_combine64(adler1, adler2, len2) uLong adler1; uLong adler2; z_off64_t len2; { return adler32_combine_(adler1, adler2, len2); } c-blosc-1.7.0/internal-complibs/zlib-1.2.8/compress.c000066400000000000000000000047411254622165100221000ustar00rootroot00000000000000/* compress.c -- compress a memory buffer * Copyright (C) 1995-2005 Jean-loup Gailly. * For conditions of distribution and use, see copyright notice in zlib.h */ /* @(#) $Id$ */ #define ZLIB_INTERNAL #include "zlib.h" /* =========================================================================== Compresses the source buffer into the destination buffer. The level parameter has the same meaning as in deflateInit. sourceLen is the byte length of the source buffer. Upon entry, destLen is the total size of the destination buffer, which must be at least 0.1% larger than sourceLen plus 12 bytes. Upon exit, destLen is the actual size of the compressed buffer. compress2 returns Z_OK if success, Z_MEM_ERROR if there was not enough memory, Z_BUF_ERROR if there was not enough room in the output buffer, Z_STREAM_ERROR if the level parameter is invalid. */ int ZEXPORT compress2 (dest, destLen, source, sourceLen, level) Bytef *dest; uLongf *destLen; const Bytef *source; uLong sourceLen; int level; { z_stream stream; int err; stream.next_in = (z_const Bytef *)source; stream.avail_in = (uInt)sourceLen; #ifdef MAXSEG_64K /* Check for source > 64K on 16-bit machine: */ if ((uLong)stream.avail_in != sourceLen) return Z_BUF_ERROR; #endif stream.next_out = dest; stream.avail_out = (uInt)*destLen; if ((uLong)stream.avail_out != *destLen) return Z_BUF_ERROR; stream.zalloc = (alloc_func)0; stream.zfree = (free_func)0; stream.opaque = (voidpf)0; err = deflateInit(&stream, level); if (err != Z_OK) return err; err = deflate(&stream, Z_FINISH); if (err != Z_STREAM_END) { deflateEnd(&stream); return err == Z_OK ? Z_BUF_ERROR : err; } *destLen = stream.total_out; err = deflateEnd(&stream); return err; } /* =========================================================================== */ int ZEXPORT compress (dest, destLen, source, sourceLen) Bytef *dest; uLongf *destLen; const Bytef *source; uLong sourceLen; { return compress2(dest, destLen, source, sourceLen, Z_DEFAULT_COMPRESSION); } /* =========================================================================== If the default memLevel or windowBits for deflateInit() is changed, then this function needs to be updated. */ uLong ZEXPORT compressBound (sourceLen) uLong sourceLen; { return sourceLen + (sourceLen >> 12) + (sourceLen >> 14) + (sourceLen >> 25) + 13; } c-blosc-1.7.0/internal-complibs/zlib-1.2.8/crc32.c000066400000000000000000000315661254622165100211660ustar00rootroot00000000000000/* crc32.c -- compute the CRC-32 of a data stream * Copyright (C) 1995-2006, 2010, 2011, 2012 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h * * Thanks to Rodney Brown for his contribution of faster * CRC methods: exclusive-oring 32 bits of data at a time, and pre-computing * tables for updating the shift register in one step with three exclusive-ors * instead of four steps with four exclusive-ors. This results in about a * factor of two increase in speed on a Power PC G4 (PPC7455) using gcc -O3. */ /* @(#) $Id$ */ /* Note on the use of DYNAMIC_CRC_TABLE: there is no mutex or semaphore protection on the static variables used to control the first-use generation of the crc tables. Therefore, if you #define DYNAMIC_CRC_TABLE, you should first call get_crc_table() to initialize the tables before allowing more than one thread to use crc32(). DYNAMIC_CRC_TABLE and MAKECRCH can be #defined to write out crc32.h. */ #ifdef MAKECRCH # include # ifndef DYNAMIC_CRC_TABLE # define DYNAMIC_CRC_TABLE # endif /* !DYNAMIC_CRC_TABLE */ #endif /* MAKECRCH */ #include "zutil.h" /* for STDC and FAR definitions */ #define local static /* Definitions for doing the crc four data bytes at a time. */ #if !defined(NOBYFOUR) && defined(Z_U4) # define BYFOUR #endif #ifdef BYFOUR local unsigned long crc32_little OF((unsigned long, const unsigned char FAR *, unsigned)); local unsigned long crc32_big OF((unsigned long, const unsigned char FAR *, unsigned)); # define TBLS 8 #else # define TBLS 1 #endif /* BYFOUR */ /* Local functions for crc concatenation */ local unsigned long gf2_matrix_times OF((unsigned long *mat, unsigned long vec)); local void gf2_matrix_square OF((unsigned long *square, unsigned long *mat)); local uLong crc32_combine_ OF((uLong crc1, uLong crc2, z_off64_t len2)); #ifdef DYNAMIC_CRC_TABLE local volatile int crc_table_empty = 1; local z_crc_t FAR crc_table[TBLS][256]; local void make_crc_table OF((void)); #ifdef MAKECRCH local void write_table OF((FILE *, const z_crc_t FAR *)); #endif /* MAKECRCH */ /* Generate tables for a byte-wise 32-bit CRC calculation on the polynomial: x^32+x^26+x^23+x^22+x^16+x^12+x^11+x^10+x^8+x^7+x^5+x^4+x^2+x+1. Polynomials over GF(2) are represented in binary, one bit per coefficient, with the lowest powers in the most significant bit. Then adding polynomials is just exclusive-or, and multiplying a polynomial by x is a right shift by one. If we call the above polynomial p, and represent a byte as the polynomial q, also with the lowest power in the most significant bit (so the byte 0xb1 is the polynomial x^7+x^3+x+1), then the CRC is (q*x^32) mod p, where a mod b means the remainder after dividing a by b. This calculation is done using the shift-register method of multiplying and taking the remainder. The register is initialized to zero, and for each incoming bit, x^32 is added mod p to the register if the bit is a one (where x^32 mod p is p+x^32 = x^26+...+1), and the register is multiplied mod p by x (which is shifting right by one and adding x^32 mod p if the bit shifted out is a one). We start with the highest power (least significant bit) of q and repeat for all eight bits of q. The first table is simply the CRC of all possible eight bit values. This is all the information needed to generate CRCs on data a byte at a time for all combinations of CRC register values and incoming bytes. The remaining tables allow for word-at-a-time CRC calculation for both big-endian and little- endian machines, where a word is four bytes. */ local void make_crc_table() { z_crc_t c; int n, k; z_crc_t poly; /* polynomial exclusive-or pattern */ /* terms of polynomial defining this crc (except x^32): */ static volatile int first = 1; /* flag to limit concurrent making */ static const unsigned char p[] = {0,1,2,4,5,7,8,10,11,12,16,22,23,26}; /* See if another task is already doing this (not thread-safe, but better than nothing -- significantly reduces duration of vulnerability in case the advice about DYNAMIC_CRC_TABLE is ignored) */ if (first) { first = 0; /* make exclusive-or pattern from polynomial (0xedb88320UL) */ poly = 0; for (n = 0; n < (int)(sizeof(p)/sizeof(unsigned char)); n++) poly |= (z_crc_t)1 << (31 - p[n]); /* generate a crc for every 8-bit value */ for (n = 0; n < 256; n++) { c = (z_crc_t)n; for (k = 0; k < 8; k++) c = c & 1 ? poly ^ (c >> 1) : c >> 1; crc_table[0][n] = c; } #ifdef BYFOUR /* generate crc for each value followed by one, two, and three zeros, and then the byte reversal of those as well as the first table */ for (n = 0; n < 256; n++) { c = crc_table[0][n]; crc_table[4][n] = ZSWAP32(c); for (k = 1; k < 4; k++) { c = crc_table[0][c & 0xff] ^ (c >> 8); crc_table[k][n] = c; crc_table[k + 4][n] = ZSWAP32(c); } } #endif /* BYFOUR */ crc_table_empty = 0; } else { /* not first */ /* wait for the other guy to finish (not efficient, but rare) */ while (crc_table_empty) ; } #ifdef MAKECRCH /* write out CRC tables to crc32.h */ { FILE *out; out = fopen("crc32.h", "w"); if (out == NULL) return; fprintf(out, "/* crc32.h -- tables for rapid CRC calculation\n"); fprintf(out, " * Generated automatically by crc32.c\n */\n\n"); fprintf(out, "local const z_crc_t FAR "); fprintf(out, "crc_table[TBLS][256] =\n{\n {\n"); write_table(out, crc_table[0]); # ifdef BYFOUR fprintf(out, "#ifdef BYFOUR\n"); for (k = 1; k < 8; k++) { fprintf(out, " },\n {\n"); write_table(out, crc_table[k]); } fprintf(out, "#endif\n"); # endif /* BYFOUR */ fprintf(out, " }\n};\n"); fclose(out); } #endif /* MAKECRCH */ } #ifdef MAKECRCH local void write_table(out, table) FILE *out; const z_crc_t FAR *table; { int n; for (n = 0; n < 256; n++) fprintf(out, "%s0x%08lxUL%s", n % 5 ? "" : " ", (unsigned long)(table[n]), n == 255 ? "\n" : (n % 5 == 4 ? ",\n" : ", ")); } #endif /* MAKECRCH */ #else /* !DYNAMIC_CRC_TABLE */ /* ======================================================================== * Tables of CRC-32s of all single-byte values, made by make_crc_table(). */ #include "crc32.h" #endif /* DYNAMIC_CRC_TABLE */ /* ========================================================================= * This function can be used by asm versions of crc32() */ const z_crc_t FAR * ZEXPORT get_crc_table() { #ifdef DYNAMIC_CRC_TABLE if (crc_table_empty) make_crc_table(); #endif /* DYNAMIC_CRC_TABLE */ return (const z_crc_t FAR *)crc_table; } /* ========================================================================= */ #define DO1 crc = crc_table[0][((int)crc ^ (*buf++)) & 0xff] ^ (crc >> 8) #define DO8 DO1; DO1; DO1; DO1; DO1; DO1; DO1; DO1 /* ========================================================================= */ unsigned long ZEXPORT crc32(crc, buf, len) unsigned long crc; const unsigned char FAR *buf; uInt len; { if (buf == Z_NULL) return 0UL; #ifdef DYNAMIC_CRC_TABLE if (crc_table_empty) make_crc_table(); #endif /* DYNAMIC_CRC_TABLE */ #ifdef BYFOUR if (sizeof(void *) == sizeof(ptrdiff_t)) { z_crc_t endian; endian = 1; if (*((unsigned char *)(&endian))) return crc32_little(crc, buf, len); else return crc32_big(crc, buf, len); } #endif /* BYFOUR */ crc = crc ^ 0xffffffffUL; while (len >= 8) { DO8; len -= 8; } if (len) do { DO1; } while (--len); return crc ^ 0xffffffffUL; } #ifdef BYFOUR /* ========================================================================= */ #define DOLIT4 c ^= *buf4++; \ c = crc_table[3][c & 0xff] ^ crc_table[2][(c >> 8) & 0xff] ^ \ crc_table[1][(c >> 16) & 0xff] ^ crc_table[0][c >> 24] #define DOLIT32 DOLIT4; DOLIT4; DOLIT4; DOLIT4; DOLIT4; DOLIT4; DOLIT4; DOLIT4 /* ========================================================================= */ local unsigned long crc32_little(crc, buf, len) unsigned long crc; const unsigned char FAR *buf; unsigned len; { register z_crc_t c; register const z_crc_t FAR *buf4; c = (z_crc_t)crc; c = ~c; while (len && ((ptrdiff_t)buf & 3)) { c = crc_table[0][(c ^ *buf++) & 0xff] ^ (c >> 8); len--; } buf4 = (const z_crc_t FAR *)(const void FAR *)buf; while (len >= 32) { DOLIT32; len -= 32; } while (len >= 4) { DOLIT4; len -= 4; } buf = (const unsigned char FAR *)buf4; if (len) do { c = crc_table[0][(c ^ *buf++) & 0xff] ^ (c >> 8); } while (--len); c = ~c; return (unsigned long)c; } /* ========================================================================= */ #define DOBIG4 c ^= *++buf4; \ c = crc_table[4][c & 0xff] ^ crc_table[5][(c >> 8) & 0xff] ^ \ crc_table[6][(c >> 16) & 0xff] ^ crc_table[7][c >> 24] #define DOBIG32 DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4 /* ========================================================================= */ local unsigned long crc32_big(crc, buf, len) unsigned long crc; const unsigned char FAR *buf; unsigned len; { register z_crc_t c; register const z_crc_t FAR *buf4; c = ZSWAP32((z_crc_t)crc); c = ~c; while (len && ((ptrdiff_t)buf & 3)) { c = crc_table[4][(c >> 24) ^ *buf++] ^ (c << 8); len--; } buf4 = (const z_crc_t FAR *)(const void FAR *)buf; buf4--; while (len >= 32) { DOBIG32; len -= 32; } while (len >= 4) { DOBIG4; len -= 4; } buf4++; buf = (const unsigned char FAR *)buf4; if (len) do { c = crc_table[4][(c >> 24) ^ *buf++] ^ (c << 8); } while (--len); c = ~c; return (unsigned long)(ZSWAP32(c)); } #endif /* BYFOUR */ #define GF2_DIM 32 /* dimension of GF(2) vectors (length of CRC) */ /* ========================================================================= */ local unsigned long gf2_matrix_times(mat, vec) unsigned long *mat; unsigned long vec; { unsigned long sum; sum = 0; while (vec) { if (vec & 1) sum ^= *mat; vec >>= 1; mat++; } return sum; } /* ========================================================================= */ local void gf2_matrix_square(square, mat) unsigned long *square; unsigned long *mat; { int n; for (n = 0; n < GF2_DIM; n++) square[n] = gf2_matrix_times(mat, mat[n]); } /* ========================================================================= */ local uLong crc32_combine_(crc1, crc2, len2) uLong crc1; uLong crc2; z_off64_t len2; { int n; unsigned long row; unsigned long even[GF2_DIM]; /* even-power-of-two zeros operator */ unsigned long odd[GF2_DIM]; /* odd-power-of-two zeros operator */ /* degenerate case (also disallow negative lengths) */ if (len2 <= 0) return crc1; /* put operator for one zero bit in odd */ odd[0] = 0xedb88320UL; /* CRC-32 polynomial */ row = 1; for (n = 1; n < GF2_DIM; n++) { odd[n] = row; row <<= 1; } /* put operator for two zero bits in even */ gf2_matrix_square(even, odd); /* put operator for four zero bits in odd */ gf2_matrix_square(odd, even); /* apply len2 zeros to crc1 (first square will put the operator for one zero byte, eight zero bits, in even) */ do { /* apply zeros operator for this bit of len2 */ gf2_matrix_square(even, odd); if (len2 & 1) crc1 = gf2_matrix_times(even, crc1); len2 >>= 1; /* if no more bits set, then done */ if (len2 == 0) break; /* another iteration of the loop with odd and even swapped */ gf2_matrix_square(odd, even); if (len2 & 1) crc1 = gf2_matrix_times(odd, crc1); len2 >>= 1; /* if no more bits set, then done */ } while (len2 != 0); /* return combined crc */ crc1 ^= crc2; return crc1; } /* ========================================================================= */ uLong ZEXPORT crc32_combine(crc1, crc2, len2) uLong crc1; uLong crc2; z_off_t len2; { return crc32_combine_(crc1, crc2, len2); } uLong ZEXPORT crc32_combine64(crc1, crc2, len2) uLong crc1; uLong crc2; z_off64_t len2; { return crc32_combine_(crc1, crc2, len2); } c-blosc-1.7.0/internal-complibs/zlib-1.2.8/crc32.h000066400000000000000000000735421254622165100211730ustar00rootroot00000000000000/* crc32.h -- tables for rapid CRC calculation * Generated automatically by crc32.c */ local const z_crc_t FAR crc_table[TBLS][256] = { { 0x00000000UL, 0x77073096UL, 0xee0e612cUL, 0x990951baUL, 0x076dc419UL, 0x706af48fUL, 0xe963a535UL, 0x9e6495a3UL, 0x0edb8832UL, 0x79dcb8a4UL, 0xe0d5e91eUL, 0x97d2d988UL, 0x09b64c2bUL, 0x7eb17cbdUL, 0xe7b82d07UL, 0x90bf1d91UL, 0x1db71064UL, 0x6ab020f2UL, 0xf3b97148UL, 0x84be41deUL, 0x1adad47dUL, 0x6ddde4ebUL, 0xf4d4b551UL, 0x83d385c7UL, 0x136c9856UL, 0x646ba8c0UL, 0xfd62f97aUL, 0x8a65c9ecUL, 0x14015c4fUL, 0x63066cd9UL, 0xfa0f3d63UL, 0x8d080df5UL, 0x3b6e20c8UL, 0x4c69105eUL, 0xd56041e4UL, 0xa2677172UL, 0x3c03e4d1UL, 0x4b04d447UL, 0xd20d85fdUL, 0xa50ab56bUL, 0x35b5a8faUL, 0x42b2986cUL, 0xdbbbc9d6UL, 0xacbcf940UL, 0x32d86ce3UL, 0x45df5c75UL, 0xdcd60dcfUL, 0xabd13d59UL, 0x26d930acUL, 0x51de003aUL, 0xc8d75180UL, 0xbfd06116UL, 0x21b4f4b5UL, 0x56b3c423UL, 0xcfba9599UL, 0xb8bda50fUL, 0x2802b89eUL, 0x5f058808UL, 0xc60cd9b2UL, 0xb10be924UL, 0x2f6f7c87UL, 0x58684c11UL, 0xc1611dabUL, 0xb6662d3dUL, 0x76dc4190UL, 0x01db7106UL, 0x98d220bcUL, 0xefd5102aUL, 0x71b18589UL, 0x06b6b51fUL, 0x9fbfe4a5UL, 0xe8b8d433UL, 0x7807c9a2UL, 0x0f00f934UL, 0x9609a88eUL, 0xe10e9818UL, 0x7f6a0dbbUL, 0x086d3d2dUL, 0x91646c97UL, 0xe6635c01UL, 0x6b6b51f4UL, 0x1c6c6162UL, 0x856530d8UL, 0xf262004eUL, 0x6c0695edUL, 0x1b01a57bUL, 0x8208f4c1UL, 0xf50fc457UL, 0x65b0d9c6UL, 0x12b7e950UL, 0x8bbeb8eaUL, 0xfcb9887cUL, 0x62dd1ddfUL, 0x15da2d49UL, 0x8cd37cf3UL, 0xfbd44c65UL, 0x4db26158UL, 0x3ab551ceUL, 0xa3bc0074UL, 0xd4bb30e2UL, 0x4adfa541UL, 0x3dd895d7UL, 0xa4d1c46dUL, 0xd3d6f4fbUL, 0x4369e96aUL, 0x346ed9fcUL, 0xad678846UL, 0xda60b8d0UL, 0x44042d73UL, 0x33031de5UL, 0xaa0a4c5fUL, 0xdd0d7cc9UL, 0x5005713cUL, 0x270241aaUL, 0xbe0b1010UL, 0xc90c2086UL, 0x5768b525UL, 0x206f85b3UL, 0xb966d409UL, 0xce61e49fUL, 0x5edef90eUL, 0x29d9c998UL, 0xb0d09822UL, 0xc7d7a8b4UL, 0x59b33d17UL, 0x2eb40d81UL, 0xb7bd5c3bUL, 0xc0ba6cadUL, 0xedb88320UL, 0x9abfb3b6UL, 0x03b6e20cUL, 0x74b1d29aUL, 0xead54739UL, 0x9dd277afUL, 0x04db2615UL, 0x73dc1683UL, 0xe3630b12UL, 0x94643b84UL, 0x0d6d6a3eUL, 0x7a6a5aa8UL, 0xe40ecf0bUL, 0x9309ff9dUL, 0x0a00ae27UL, 0x7d079eb1UL, 0xf00f9344UL, 0x8708a3d2UL, 0x1e01f268UL, 0x6906c2feUL, 0xf762575dUL, 0x806567cbUL, 0x196c3671UL, 0x6e6b06e7UL, 0xfed41b76UL, 0x89d32be0UL, 0x10da7a5aUL, 0x67dd4accUL, 0xf9b9df6fUL, 0x8ebeeff9UL, 0x17b7be43UL, 0x60b08ed5UL, 0xd6d6a3e8UL, 0xa1d1937eUL, 0x38d8c2c4UL, 0x4fdff252UL, 0xd1bb67f1UL, 0xa6bc5767UL, 0x3fb506ddUL, 0x48b2364bUL, 0xd80d2bdaUL, 0xaf0a1b4cUL, 0x36034af6UL, 0x41047a60UL, 0xdf60efc3UL, 0xa867df55UL, 0x316e8eefUL, 0x4669be79UL, 0xcb61b38cUL, 0xbc66831aUL, 0x256fd2a0UL, 0x5268e236UL, 0xcc0c7795UL, 0xbb0b4703UL, 0x220216b9UL, 0x5505262fUL, 0xc5ba3bbeUL, 0xb2bd0b28UL, 0x2bb45a92UL, 0x5cb36a04UL, 0xc2d7ffa7UL, 0xb5d0cf31UL, 0x2cd99e8bUL, 0x5bdeae1dUL, 0x9b64c2b0UL, 0xec63f226UL, 0x756aa39cUL, 0x026d930aUL, 0x9c0906a9UL, 0xeb0e363fUL, 0x72076785UL, 0x05005713UL, 0x95bf4a82UL, 0xe2b87a14UL, 0x7bb12baeUL, 0x0cb61b38UL, 0x92d28e9bUL, 0xe5d5be0dUL, 0x7cdcefb7UL, 0x0bdbdf21UL, 0x86d3d2d4UL, 0xf1d4e242UL, 0x68ddb3f8UL, 0x1fda836eUL, 0x81be16cdUL, 0xf6b9265bUL, 0x6fb077e1UL, 0x18b74777UL, 0x88085ae6UL, 0xff0f6a70UL, 0x66063bcaUL, 0x11010b5cUL, 0x8f659effUL, 0xf862ae69UL, 0x616bffd3UL, 0x166ccf45UL, 0xa00ae278UL, 0xd70dd2eeUL, 0x4e048354UL, 0x3903b3c2UL, 0xa7672661UL, 0xd06016f7UL, 0x4969474dUL, 0x3e6e77dbUL, 0xaed16a4aUL, 0xd9d65adcUL, 0x40df0b66UL, 0x37d83bf0UL, 0xa9bcae53UL, 0xdebb9ec5UL, 0x47b2cf7fUL, 0x30b5ffe9UL, 0xbdbdf21cUL, 0xcabac28aUL, 0x53b39330UL, 0x24b4a3a6UL, 0xbad03605UL, 0xcdd70693UL, 0x54de5729UL, 0x23d967bfUL, 0xb3667a2eUL, 0xc4614ab8UL, 0x5d681b02UL, 0x2a6f2b94UL, 0xb40bbe37UL, 0xc30c8ea1UL, 0x5a05df1bUL, 0x2d02ef8dUL #ifdef BYFOUR }, { 0x00000000UL, 0x191b3141UL, 0x32366282UL, 0x2b2d53c3UL, 0x646cc504UL, 0x7d77f445UL, 0x565aa786UL, 0x4f4196c7UL, 0xc8d98a08UL, 0xd1c2bb49UL, 0xfaefe88aUL, 0xe3f4d9cbUL, 0xacb54f0cUL, 0xb5ae7e4dUL, 0x9e832d8eUL, 0x87981ccfUL, 0x4ac21251UL, 0x53d92310UL, 0x78f470d3UL, 0x61ef4192UL, 0x2eaed755UL, 0x37b5e614UL, 0x1c98b5d7UL, 0x05838496UL, 0x821b9859UL, 0x9b00a918UL, 0xb02dfadbUL, 0xa936cb9aUL, 0xe6775d5dUL, 0xff6c6c1cUL, 0xd4413fdfUL, 0xcd5a0e9eUL, 0x958424a2UL, 0x8c9f15e3UL, 0xa7b24620UL, 0xbea97761UL, 0xf1e8e1a6UL, 0xe8f3d0e7UL, 0xc3de8324UL, 0xdac5b265UL, 0x5d5daeaaUL, 0x44469febUL, 0x6f6bcc28UL, 0x7670fd69UL, 0x39316baeUL, 0x202a5aefUL, 0x0b07092cUL, 0x121c386dUL, 0xdf4636f3UL, 0xc65d07b2UL, 0xed705471UL, 0xf46b6530UL, 0xbb2af3f7UL, 0xa231c2b6UL, 0x891c9175UL, 0x9007a034UL, 0x179fbcfbUL, 0x0e848dbaUL, 0x25a9de79UL, 0x3cb2ef38UL, 0x73f379ffUL, 0x6ae848beUL, 0x41c51b7dUL, 0x58de2a3cUL, 0xf0794f05UL, 0xe9627e44UL, 0xc24f2d87UL, 0xdb541cc6UL, 0x94158a01UL, 0x8d0ebb40UL, 0xa623e883UL, 0xbf38d9c2UL, 0x38a0c50dUL, 0x21bbf44cUL, 0x0a96a78fUL, 0x138d96ceUL, 0x5ccc0009UL, 0x45d73148UL, 0x6efa628bUL, 0x77e153caUL, 0xbabb5d54UL, 0xa3a06c15UL, 0x888d3fd6UL, 0x91960e97UL, 0xded79850UL, 0xc7cca911UL, 0xece1fad2UL, 0xf5facb93UL, 0x7262d75cUL, 0x6b79e61dUL, 0x4054b5deUL, 0x594f849fUL, 0x160e1258UL, 0x0f152319UL, 0x243870daUL, 0x3d23419bUL, 0x65fd6ba7UL, 0x7ce65ae6UL, 0x57cb0925UL, 0x4ed03864UL, 0x0191aea3UL, 0x188a9fe2UL, 0x33a7cc21UL, 0x2abcfd60UL, 0xad24e1afUL, 0xb43fd0eeUL, 0x9f12832dUL, 0x8609b26cUL, 0xc94824abUL, 0xd05315eaUL, 0xfb7e4629UL, 0xe2657768UL, 0x2f3f79f6UL, 0x362448b7UL, 0x1d091b74UL, 0x04122a35UL, 0x4b53bcf2UL, 0x52488db3UL, 0x7965de70UL, 0x607eef31UL, 0xe7e6f3feUL, 0xfefdc2bfUL, 0xd5d0917cUL, 0xcccba03dUL, 0x838a36faUL, 0x9a9107bbUL, 0xb1bc5478UL, 0xa8a76539UL, 0x3b83984bUL, 0x2298a90aUL, 0x09b5fac9UL, 0x10aecb88UL, 0x5fef5d4fUL, 0x46f46c0eUL, 0x6dd93fcdUL, 0x74c20e8cUL, 0xf35a1243UL, 0xea412302UL, 0xc16c70c1UL, 0xd8774180UL, 0x9736d747UL, 0x8e2de606UL, 0xa500b5c5UL, 0xbc1b8484UL, 0x71418a1aUL, 0x685abb5bUL, 0x4377e898UL, 0x5a6cd9d9UL, 0x152d4f1eUL, 0x0c367e5fUL, 0x271b2d9cUL, 0x3e001cddUL, 0xb9980012UL, 0xa0833153UL, 0x8bae6290UL, 0x92b553d1UL, 0xddf4c516UL, 0xc4eff457UL, 0xefc2a794UL, 0xf6d996d5UL, 0xae07bce9UL, 0xb71c8da8UL, 0x9c31de6bUL, 0x852aef2aUL, 0xca6b79edUL, 0xd37048acUL, 0xf85d1b6fUL, 0xe1462a2eUL, 0x66de36e1UL, 0x7fc507a0UL, 0x54e85463UL, 0x4df36522UL, 0x02b2f3e5UL, 0x1ba9c2a4UL, 0x30849167UL, 0x299fa026UL, 0xe4c5aeb8UL, 0xfdde9ff9UL, 0xd6f3cc3aUL, 0xcfe8fd7bUL, 0x80a96bbcUL, 0x99b25afdUL, 0xb29f093eUL, 0xab84387fUL, 0x2c1c24b0UL, 0x350715f1UL, 0x1e2a4632UL, 0x07317773UL, 0x4870e1b4UL, 0x516bd0f5UL, 0x7a468336UL, 0x635db277UL, 0xcbfad74eUL, 0xd2e1e60fUL, 0xf9ccb5ccUL, 0xe0d7848dUL, 0xaf96124aUL, 0xb68d230bUL, 0x9da070c8UL, 0x84bb4189UL, 0x03235d46UL, 0x1a386c07UL, 0x31153fc4UL, 0x280e0e85UL, 0x674f9842UL, 0x7e54a903UL, 0x5579fac0UL, 0x4c62cb81UL, 0x8138c51fUL, 0x9823f45eUL, 0xb30ea79dUL, 0xaa1596dcUL, 0xe554001bUL, 0xfc4f315aUL, 0xd7626299UL, 0xce7953d8UL, 0x49e14f17UL, 0x50fa7e56UL, 0x7bd72d95UL, 0x62cc1cd4UL, 0x2d8d8a13UL, 0x3496bb52UL, 0x1fbbe891UL, 0x06a0d9d0UL, 0x5e7ef3ecUL, 0x4765c2adUL, 0x6c48916eUL, 0x7553a02fUL, 0x3a1236e8UL, 0x230907a9UL, 0x0824546aUL, 0x113f652bUL, 0x96a779e4UL, 0x8fbc48a5UL, 0xa4911b66UL, 0xbd8a2a27UL, 0xf2cbbce0UL, 0xebd08da1UL, 0xc0fdde62UL, 0xd9e6ef23UL, 0x14bce1bdUL, 0x0da7d0fcUL, 0x268a833fUL, 0x3f91b27eUL, 0x70d024b9UL, 0x69cb15f8UL, 0x42e6463bUL, 0x5bfd777aUL, 0xdc656bb5UL, 0xc57e5af4UL, 0xee530937UL, 0xf7483876UL, 0xb809aeb1UL, 0xa1129ff0UL, 0x8a3fcc33UL, 0x9324fd72UL }, { 0x00000000UL, 0x01c26a37UL, 0x0384d46eUL, 0x0246be59UL, 0x0709a8dcUL, 0x06cbc2ebUL, 0x048d7cb2UL, 0x054f1685UL, 0x0e1351b8UL, 0x0fd13b8fUL, 0x0d9785d6UL, 0x0c55efe1UL, 0x091af964UL, 0x08d89353UL, 0x0a9e2d0aUL, 0x0b5c473dUL, 0x1c26a370UL, 0x1de4c947UL, 0x1fa2771eUL, 0x1e601d29UL, 0x1b2f0bacUL, 0x1aed619bUL, 0x18abdfc2UL, 0x1969b5f5UL, 0x1235f2c8UL, 0x13f798ffUL, 0x11b126a6UL, 0x10734c91UL, 0x153c5a14UL, 0x14fe3023UL, 0x16b88e7aUL, 0x177ae44dUL, 0x384d46e0UL, 0x398f2cd7UL, 0x3bc9928eUL, 0x3a0bf8b9UL, 0x3f44ee3cUL, 0x3e86840bUL, 0x3cc03a52UL, 0x3d025065UL, 0x365e1758UL, 0x379c7d6fUL, 0x35dac336UL, 0x3418a901UL, 0x3157bf84UL, 0x3095d5b3UL, 0x32d36beaUL, 0x331101ddUL, 0x246be590UL, 0x25a98fa7UL, 0x27ef31feUL, 0x262d5bc9UL, 0x23624d4cUL, 0x22a0277bUL, 0x20e69922UL, 0x2124f315UL, 0x2a78b428UL, 0x2bbade1fUL, 0x29fc6046UL, 0x283e0a71UL, 0x2d711cf4UL, 0x2cb376c3UL, 0x2ef5c89aUL, 0x2f37a2adUL, 0x709a8dc0UL, 0x7158e7f7UL, 0x731e59aeUL, 0x72dc3399UL, 0x7793251cUL, 0x76514f2bUL, 0x7417f172UL, 0x75d59b45UL, 0x7e89dc78UL, 0x7f4bb64fUL, 0x7d0d0816UL, 0x7ccf6221UL, 0x798074a4UL, 0x78421e93UL, 0x7a04a0caUL, 0x7bc6cafdUL, 0x6cbc2eb0UL, 0x6d7e4487UL, 0x6f38fadeUL, 0x6efa90e9UL, 0x6bb5866cUL, 0x6a77ec5bUL, 0x68315202UL, 0x69f33835UL, 0x62af7f08UL, 0x636d153fUL, 0x612bab66UL, 0x60e9c151UL, 0x65a6d7d4UL, 0x6464bde3UL, 0x662203baUL, 0x67e0698dUL, 0x48d7cb20UL, 0x4915a117UL, 0x4b531f4eUL, 0x4a917579UL, 0x4fde63fcUL, 0x4e1c09cbUL, 0x4c5ab792UL, 0x4d98dda5UL, 0x46c49a98UL, 0x4706f0afUL, 0x45404ef6UL, 0x448224c1UL, 0x41cd3244UL, 0x400f5873UL, 0x4249e62aUL, 0x438b8c1dUL, 0x54f16850UL, 0x55330267UL, 0x5775bc3eUL, 0x56b7d609UL, 0x53f8c08cUL, 0x523aaabbUL, 0x507c14e2UL, 0x51be7ed5UL, 0x5ae239e8UL, 0x5b2053dfUL, 0x5966ed86UL, 0x58a487b1UL, 0x5deb9134UL, 0x5c29fb03UL, 0x5e6f455aUL, 0x5fad2f6dUL, 0xe1351b80UL, 0xe0f771b7UL, 0xe2b1cfeeUL, 0xe373a5d9UL, 0xe63cb35cUL, 0xe7fed96bUL, 0xe5b86732UL, 0xe47a0d05UL, 0xef264a38UL, 0xeee4200fUL, 0xeca29e56UL, 0xed60f461UL, 0xe82fe2e4UL, 0xe9ed88d3UL, 0xebab368aUL, 0xea695cbdUL, 0xfd13b8f0UL, 0xfcd1d2c7UL, 0xfe976c9eUL, 0xff5506a9UL, 0xfa1a102cUL, 0xfbd87a1bUL, 0xf99ec442UL, 0xf85cae75UL, 0xf300e948UL, 0xf2c2837fUL, 0xf0843d26UL, 0xf1465711UL, 0xf4094194UL, 0xf5cb2ba3UL, 0xf78d95faUL, 0xf64fffcdUL, 0xd9785d60UL, 0xd8ba3757UL, 0xdafc890eUL, 0xdb3ee339UL, 0xde71f5bcUL, 0xdfb39f8bUL, 0xddf521d2UL, 0xdc374be5UL, 0xd76b0cd8UL, 0xd6a966efUL, 0xd4efd8b6UL, 0xd52db281UL, 0xd062a404UL, 0xd1a0ce33UL, 0xd3e6706aUL, 0xd2241a5dUL, 0xc55efe10UL, 0xc49c9427UL, 0xc6da2a7eUL, 0xc7184049UL, 0xc25756ccUL, 0xc3953cfbUL, 0xc1d382a2UL, 0xc011e895UL, 0xcb4dafa8UL, 0xca8fc59fUL, 0xc8c97bc6UL, 0xc90b11f1UL, 0xcc440774UL, 0xcd866d43UL, 0xcfc0d31aUL, 0xce02b92dUL, 0x91af9640UL, 0x906dfc77UL, 0x922b422eUL, 0x93e92819UL, 0x96a63e9cUL, 0x976454abUL, 0x9522eaf2UL, 0x94e080c5UL, 0x9fbcc7f8UL, 0x9e7eadcfUL, 0x9c381396UL, 0x9dfa79a1UL, 0x98b56f24UL, 0x99770513UL, 0x9b31bb4aUL, 0x9af3d17dUL, 0x8d893530UL, 0x8c4b5f07UL, 0x8e0de15eUL, 0x8fcf8b69UL, 0x8a809decUL, 0x8b42f7dbUL, 0x89044982UL, 0x88c623b5UL, 0x839a6488UL, 0x82580ebfUL, 0x801eb0e6UL, 0x81dcdad1UL, 0x8493cc54UL, 0x8551a663UL, 0x8717183aUL, 0x86d5720dUL, 0xa9e2d0a0UL, 0xa820ba97UL, 0xaa6604ceUL, 0xaba46ef9UL, 0xaeeb787cUL, 0xaf29124bUL, 0xad6fac12UL, 0xacadc625UL, 0xa7f18118UL, 0xa633eb2fUL, 0xa4755576UL, 0xa5b73f41UL, 0xa0f829c4UL, 0xa13a43f3UL, 0xa37cfdaaUL, 0xa2be979dUL, 0xb5c473d0UL, 0xb40619e7UL, 0xb640a7beUL, 0xb782cd89UL, 0xb2cddb0cUL, 0xb30fb13bUL, 0xb1490f62UL, 0xb08b6555UL, 0xbbd72268UL, 0xba15485fUL, 0xb853f606UL, 0xb9919c31UL, 0xbcde8ab4UL, 0xbd1ce083UL, 0xbf5a5edaUL, 0xbe9834edUL }, { 0x00000000UL, 0xb8bc6765UL, 0xaa09c88bUL, 0x12b5afeeUL, 0x8f629757UL, 0x37def032UL, 0x256b5fdcUL, 0x9dd738b9UL, 0xc5b428efUL, 0x7d084f8aUL, 0x6fbde064UL, 0xd7018701UL, 0x4ad6bfb8UL, 0xf26ad8ddUL, 0xe0df7733UL, 0x58631056UL, 0x5019579fUL, 0xe8a530faUL, 0xfa109f14UL, 0x42acf871UL, 0xdf7bc0c8UL, 0x67c7a7adUL, 0x75720843UL, 0xcdce6f26UL, 0x95ad7f70UL, 0x2d111815UL, 0x3fa4b7fbUL, 0x8718d09eUL, 0x1acfe827UL, 0xa2738f42UL, 0xb0c620acUL, 0x087a47c9UL, 0xa032af3eUL, 0x188ec85bUL, 0x0a3b67b5UL, 0xb28700d0UL, 0x2f503869UL, 0x97ec5f0cUL, 0x8559f0e2UL, 0x3de59787UL, 0x658687d1UL, 0xdd3ae0b4UL, 0xcf8f4f5aUL, 0x7733283fUL, 0xeae41086UL, 0x525877e3UL, 0x40edd80dUL, 0xf851bf68UL, 0xf02bf8a1UL, 0x48979fc4UL, 0x5a22302aUL, 0xe29e574fUL, 0x7f496ff6UL, 0xc7f50893UL, 0xd540a77dUL, 0x6dfcc018UL, 0x359fd04eUL, 0x8d23b72bUL, 0x9f9618c5UL, 0x272a7fa0UL, 0xbafd4719UL, 0x0241207cUL, 0x10f48f92UL, 0xa848e8f7UL, 0x9b14583dUL, 0x23a83f58UL, 0x311d90b6UL, 0x89a1f7d3UL, 0x1476cf6aUL, 0xaccaa80fUL, 0xbe7f07e1UL, 0x06c36084UL, 0x5ea070d2UL, 0xe61c17b7UL, 0xf4a9b859UL, 0x4c15df3cUL, 0xd1c2e785UL, 0x697e80e0UL, 0x7bcb2f0eUL, 0xc377486bUL, 0xcb0d0fa2UL, 0x73b168c7UL, 0x6104c729UL, 0xd9b8a04cUL, 0x446f98f5UL, 0xfcd3ff90UL, 0xee66507eUL, 0x56da371bUL, 0x0eb9274dUL, 0xb6054028UL, 0xa4b0efc6UL, 0x1c0c88a3UL, 0x81dbb01aUL, 0x3967d77fUL, 0x2bd27891UL, 0x936e1ff4UL, 0x3b26f703UL, 0x839a9066UL, 0x912f3f88UL, 0x299358edUL, 0xb4446054UL, 0x0cf80731UL, 0x1e4da8dfUL, 0xa6f1cfbaUL, 0xfe92dfecUL, 0x462eb889UL, 0x549b1767UL, 0xec277002UL, 0x71f048bbUL, 0xc94c2fdeUL, 0xdbf98030UL, 0x6345e755UL, 0x6b3fa09cUL, 0xd383c7f9UL, 0xc1366817UL, 0x798a0f72UL, 0xe45d37cbUL, 0x5ce150aeUL, 0x4e54ff40UL, 0xf6e89825UL, 0xae8b8873UL, 0x1637ef16UL, 0x048240f8UL, 0xbc3e279dUL, 0x21e91f24UL, 0x99557841UL, 0x8be0d7afUL, 0x335cb0caUL, 0xed59b63bUL, 0x55e5d15eUL, 0x47507eb0UL, 0xffec19d5UL, 0x623b216cUL, 0xda874609UL, 0xc832e9e7UL, 0x708e8e82UL, 0x28ed9ed4UL, 0x9051f9b1UL, 0x82e4565fUL, 0x3a58313aUL, 0xa78f0983UL, 0x1f336ee6UL, 0x0d86c108UL, 0xb53aa66dUL, 0xbd40e1a4UL, 0x05fc86c1UL, 0x1749292fUL, 0xaff54e4aUL, 0x322276f3UL, 0x8a9e1196UL, 0x982bbe78UL, 0x2097d91dUL, 0x78f4c94bUL, 0xc048ae2eUL, 0xd2fd01c0UL, 0x6a4166a5UL, 0xf7965e1cUL, 0x4f2a3979UL, 0x5d9f9697UL, 0xe523f1f2UL, 0x4d6b1905UL, 0xf5d77e60UL, 0xe762d18eUL, 0x5fdeb6ebUL, 0xc2098e52UL, 0x7ab5e937UL, 0x680046d9UL, 0xd0bc21bcUL, 0x88df31eaUL, 0x3063568fUL, 0x22d6f961UL, 0x9a6a9e04UL, 0x07bda6bdUL, 0xbf01c1d8UL, 0xadb46e36UL, 0x15080953UL, 0x1d724e9aUL, 0xa5ce29ffUL, 0xb77b8611UL, 0x0fc7e174UL, 0x9210d9cdUL, 0x2aacbea8UL, 0x38191146UL, 0x80a57623UL, 0xd8c66675UL, 0x607a0110UL, 0x72cfaefeUL, 0xca73c99bUL, 0x57a4f122UL, 0xef189647UL, 0xfdad39a9UL, 0x45115eccUL, 0x764dee06UL, 0xcef18963UL, 0xdc44268dUL, 0x64f841e8UL, 0xf92f7951UL, 0x41931e34UL, 0x5326b1daUL, 0xeb9ad6bfUL, 0xb3f9c6e9UL, 0x0b45a18cUL, 0x19f00e62UL, 0xa14c6907UL, 0x3c9b51beUL, 0x842736dbUL, 0x96929935UL, 0x2e2efe50UL, 0x2654b999UL, 0x9ee8defcUL, 0x8c5d7112UL, 0x34e11677UL, 0xa9362eceUL, 0x118a49abUL, 0x033fe645UL, 0xbb838120UL, 0xe3e09176UL, 0x5b5cf613UL, 0x49e959fdUL, 0xf1553e98UL, 0x6c820621UL, 0xd43e6144UL, 0xc68bceaaUL, 0x7e37a9cfUL, 0xd67f4138UL, 0x6ec3265dUL, 0x7c7689b3UL, 0xc4caeed6UL, 0x591dd66fUL, 0xe1a1b10aUL, 0xf3141ee4UL, 0x4ba87981UL, 0x13cb69d7UL, 0xab770eb2UL, 0xb9c2a15cUL, 0x017ec639UL, 0x9ca9fe80UL, 0x241599e5UL, 0x36a0360bUL, 0x8e1c516eUL, 0x866616a7UL, 0x3eda71c2UL, 0x2c6fde2cUL, 0x94d3b949UL, 0x090481f0UL, 0xb1b8e695UL, 0xa30d497bUL, 0x1bb12e1eUL, 0x43d23e48UL, 0xfb6e592dUL, 0xe9dbf6c3UL, 0x516791a6UL, 0xccb0a91fUL, 0x740cce7aUL, 0x66b96194UL, 0xde0506f1UL }, { 0x00000000UL, 0x96300777UL, 0x2c610eeeUL, 0xba510999UL, 0x19c46d07UL, 0x8ff46a70UL, 0x35a563e9UL, 0xa395649eUL, 0x3288db0eUL, 0xa4b8dc79UL, 0x1ee9d5e0UL, 0x88d9d297UL, 0x2b4cb609UL, 0xbd7cb17eUL, 0x072db8e7UL, 0x911dbf90UL, 0x6410b71dUL, 0xf220b06aUL, 0x4871b9f3UL, 0xde41be84UL, 0x7dd4da1aUL, 0xebe4dd6dUL, 0x51b5d4f4UL, 0xc785d383UL, 0x56986c13UL, 0xc0a86b64UL, 0x7af962fdUL, 0xecc9658aUL, 0x4f5c0114UL, 0xd96c0663UL, 0x633d0ffaUL, 0xf50d088dUL, 0xc8206e3bUL, 0x5e10694cUL, 0xe44160d5UL, 0x727167a2UL, 0xd1e4033cUL, 0x47d4044bUL, 0xfd850dd2UL, 0x6bb50aa5UL, 0xfaa8b535UL, 0x6c98b242UL, 0xd6c9bbdbUL, 0x40f9bcacUL, 0xe36cd832UL, 0x755cdf45UL, 0xcf0dd6dcUL, 0x593dd1abUL, 0xac30d926UL, 0x3a00de51UL, 0x8051d7c8UL, 0x1661d0bfUL, 0xb5f4b421UL, 0x23c4b356UL, 0x9995bacfUL, 0x0fa5bdb8UL, 0x9eb80228UL, 0x0888055fUL, 0xb2d90cc6UL, 0x24e90bb1UL, 0x877c6f2fUL, 0x114c6858UL, 0xab1d61c1UL, 0x3d2d66b6UL, 0x9041dc76UL, 0x0671db01UL, 0xbc20d298UL, 0x2a10d5efUL, 0x8985b171UL, 0x1fb5b606UL, 0xa5e4bf9fUL, 0x33d4b8e8UL, 0xa2c90778UL, 0x34f9000fUL, 0x8ea80996UL, 0x18980ee1UL, 0xbb0d6a7fUL, 0x2d3d6d08UL, 0x976c6491UL, 0x015c63e6UL, 0xf4516b6bUL, 0x62616c1cUL, 0xd8306585UL, 0x4e0062f2UL, 0xed95066cUL, 0x7ba5011bUL, 0xc1f40882UL, 0x57c40ff5UL, 0xc6d9b065UL, 0x50e9b712UL, 0xeab8be8bUL, 0x7c88b9fcUL, 0xdf1ddd62UL, 0x492dda15UL, 0xf37cd38cUL, 0x654cd4fbUL, 0x5861b24dUL, 0xce51b53aUL, 0x7400bca3UL, 0xe230bbd4UL, 0x41a5df4aUL, 0xd795d83dUL, 0x6dc4d1a4UL, 0xfbf4d6d3UL, 0x6ae96943UL, 0xfcd96e34UL, 0x468867adUL, 0xd0b860daUL, 0x732d0444UL, 0xe51d0333UL, 0x5f4c0aaaUL, 0xc97c0dddUL, 0x3c710550UL, 0xaa410227UL, 0x10100bbeUL, 0x86200cc9UL, 0x25b56857UL, 0xb3856f20UL, 0x09d466b9UL, 0x9fe461ceUL, 0x0ef9de5eUL, 0x98c9d929UL, 0x2298d0b0UL, 0xb4a8d7c7UL, 0x173db359UL, 0x810db42eUL, 0x3b5cbdb7UL, 0xad6cbac0UL, 0x2083b8edUL, 0xb6b3bf9aUL, 0x0ce2b603UL, 0x9ad2b174UL, 0x3947d5eaUL, 0xaf77d29dUL, 0x1526db04UL, 0x8316dc73UL, 0x120b63e3UL, 0x843b6494UL, 0x3e6a6d0dUL, 0xa85a6a7aUL, 0x0bcf0ee4UL, 0x9dff0993UL, 0x27ae000aUL, 0xb19e077dUL, 0x44930ff0UL, 0xd2a30887UL, 0x68f2011eUL, 0xfec20669UL, 0x5d5762f7UL, 0xcb676580UL, 0x71366c19UL, 0xe7066b6eUL, 0x761bd4feUL, 0xe02bd389UL, 0x5a7ada10UL, 0xcc4add67UL, 0x6fdfb9f9UL, 0xf9efbe8eUL, 0x43beb717UL, 0xd58eb060UL, 0xe8a3d6d6UL, 0x7e93d1a1UL, 0xc4c2d838UL, 0x52f2df4fUL, 0xf167bbd1UL, 0x6757bca6UL, 0xdd06b53fUL, 0x4b36b248UL, 0xda2b0dd8UL, 0x4c1b0aafUL, 0xf64a0336UL, 0x607a0441UL, 0xc3ef60dfUL, 0x55df67a8UL, 0xef8e6e31UL, 0x79be6946UL, 0x8cb361cbUL, 0x1a8366bcUL, 0xa0d26f25UL, 0x36e26852UL, 0x95770cccUL, 0x03470bbbUL, 0xb9160222UL, 0x2f260555UL, 0xbe3bbac5UL, 0x280bbdb2UL, 0x925ab42bUL, 0x046ab35cUL, 0xa7ffd7c2UL, 0x31cfd0b5UL, 0x8b9ed92cUL, 0x1daede5bUL, 0xb0c2649bUL, 0x26f263ecUL, 0x9ca36a75UL, 0x0a936d02UL, 0xa906099cUL, 0x3f360eebUL, 0x85670772UL, 0x13570005UL, 0x824abf95UL, 0x147ab8e2UL, 0xae2bb17bUL, 0x381bb60cUL, 0x9b8ed292UL, 0x0dbed5e5UL, 0xb7efdc7cUL, 0x21dfdb0bUL, 0xd4d2d386UL, 0x42e2d4f1UL, 0xf8b3dd68UL, 0x6e83da1fUL, 0xcd16be81UL, 0x5b26b9f6UL, 0xe177b06fUL, 0x7747b718UL, 0xe65a0888UL, 0x706a0fffUL, 0xca3b0666UL, 0x5c0b0111UL, 0xff9e658fUL, 0x69ae62f8UL, 0xd3ff6b61UL, 0x45cf6c16UL, 0x78e20aa0UL, 0xeed20dd7UL, 0x5483044eUL, 0xc2b30339UL, 0x612667a7UL, 0xf71660d0UL, 0x4d476949UL, 0xdb776e3eUL, 0x4a6ad1aeUL, 0xdc5ad6d9UL, 0x660bdf40UL, 0xf03bd837UL, 0x53aebca9UL, 0xc59ebbdeUL, 0x7fcfb247UL, 0xe9ffb530UL, 0x1cf2bdbdUL, 0x8ac2bacaUL, 0x3093b353UL, 0xa6a3b424UL, 0x0536d0baUL, 0x9306d7cdUL, 0x2957de54UL, 0xbf67d923UL, 0x2e7a66b3UL, 0xb84a61c4UL, 0x021b685dUL, 0x942b6f2aUL, 0x37be0bb4UL, 0xa18e0cc3UL, 0x1bdf055aUL, 0x8def022dUL }, { 0x00000000UL, 0x41311b19UL, 0x82623632UL, 0xc3532d2bUL, 0x04c56c64UL, 0x45f4777dUL, 0x86a75a56UL, 0xc796414fUL, 0x088ad9c8UL, 0x49bbc2d1UL, 0x8ae8effaUL, 0xcbd9f4e3UL, 0x0c4fb5acUL, 0x4d7eaeb5UL, 0x8e2d839eUL, 0xcf1c9887UL, 0x5112c24aUL, 0x1023d953UL, 0xd370f478UL, 0x9241ef61UL, 0x55d7ae2eUL, 0x14e6b537UL, 0xd7b5981cUL, 0x96848305UL, 0x59981b82UL, 0x18a9009bUL, 0xdbfa2db0UL, 0x9acb36a9UL, 0x5d5d77e6UL, 0x1c6c6cffUL, 0xdf3f41d4UL, 0x9e0e5acdUL, 0xa2248495UL, 0xe3159f8cUL, 0x2046b2a7UL, 0x6177a9beUL, 0xa6e1e8f1UL, 0xe7d0f3e8UL, 0x2483dec3UL, 0x65b2c5daUL, 0xaaae5d5dUL, 0xeb9f4644UL, 0x28cc6b6fUL, 0x69fd7076UL, 0xae6b3139UL, 0xef5a2a20UL, 0x2c09070bUL, 0x6d381c12UL, 0xf33646dfUL, 0xb2075dc6UL, 0x715470edUL, 0x30656bf4UL, 0xf7f32abbUL, 0xb6c231a2UL, 0x75911c89UL, 0x34a00790UL, 0xfbbc9f17UL, 0xba8d840eUL, 0x79dea925UL, 0x38efb23cUL, 0xff79f373UL, 0xbe48e86aUL, 0x7d1bc541UL, 0x3c2ade58UL, 0x054f79f0UL, 0x447e62e9UL, 0x872d4fc2UL, 0xc61c54dbUL, 0x018a1594UL, 0x40bb0e8dUL, 0x83e823a6UL, 0xc2d938bfUL, 0x0dc5a038UL, 0x4cf4bb21UL, 0x8fa7960aUL, 0xce968d13UL, 0x0900cc5cUL, 0x4831d745UL, 0x8b62fa6eUL, 0xca53e177UL, 0x545dbbbaUL, 0x156ca0a3UL, 0xd63f8d88UL, 0x970e9691UL, 0x5098d7deUL, 0x11a9ccc7UL, 0xd2fae1ecUL, 0x93cbfaf5UL, 0x5cd76272UL, 0x1de6796bUL, 0xdeb55440UL, 0x9f844f59UL, 0x58120e16UL, 0x1923150fUL, 0xda703824UL, 0x9b41233dUL, 0xa76bfd65UL, 0xe65ae67cUL, 0x2509cb57UL, 0x6438d04eUL, 0xa3ae9101UL, 0xe29f8a18UL, 0x21cca733UL, 0x60fdbc2aUL, 0xafe124adUL, 0xeed03fb4UL, 0x2d83129fUL, 0x6cb20986UL, 0xab2448c9UL, 0xea1553d0UL, 0x29467efbUL, 0x687765e2UL, 0xf6793f2fUL, 0xb7482436UL, 0x741b091dUL, 0x352a1204UL, 0xf2bc534bUL, 0xb38d4852UL, 0x70de6579UL, 0x31ef7e60UL, 0xfef3e6e7UL, 0xbfc2fdfeUL, 0x7c91d0d5UL, 0x3da0cbccUL, 0xfa368a83UL, 0xbb07919aUL, 0x7854bcb1UL, 0x3965a7a8UL, 0x4b98833bUL, 0x0aa99822UL, 0xc9fab509UL, 0x88cbae10UL, 0x4f5def5fUL, 0x0e6cf446UL, 0xcd3fd96dUL, 0x8c0ec274UL, 0x43125af3UL, 0x022341eaUL, 0xc1706cc1UL, 0x804177d8UL, 0x47d73697UL, 0x06e62d8eUL, 0xc5b500a5UL, 0x84841bbcUL, 0x1a8a4171UL, 0x5bbb5a68UL, 0x98e87743UL, 0xd9d96c5aUL, 0x1e4f2d15UL, 0x5f7e360cUL, 0x9c2d1b27UL, 0xdd1c003eUL, 0x120098b9UL, 0x533183a0UL, 0x9062ae8bUL, 0xd153b592UL, 0x16c5f4ddUL, 0x57f4efc4UL, 0x94a7c2efUL, 0xd596d9f6UL, 0xe9bc07aeUL, 0xa88d1cb7UL, 0x6bde319cUL, 0x2aef2a85UL, 0xed796bcaUL, 0xac4870d3UL, 0x6f1b5df8UL, 0x2e2a46e1UL, 0xe136de66UL, 0xa007c57fUL, 0x6354e854UL, 0x2265f34dUL, 0xe5f3b202UL, 0xa4c2a91bUL, 0x67918430UL, 0x26a09f29UL, 0xb8aec5e4UL, 0xf99fdefdUL, 0x3accf3d6UL, 0x7bfde8cfUL, 0xbc6ba980UL, 0xfd5ab299UL, 0x3e099fb2UL, 0x7f3884abUL, 0xb0241c2cUL, 0xf1150735UL, 0x32462a1eUL, 0x73773107UL, 0xb4e17048UL, 0xf5d06b51UL, 0x3683467aUL, 0x77b25d63UL, 0x4ed7facbUL, 0x0fe6e1d2UL, 0xccb5ccf9UL, 0x8d84d7e0UL, 0x4a1296afUL, 0x0b238db6UL, 0xc870a09dUL, 0x8941bb84UL, 0x465d2303UL, 0x076c381aUL, 0xc43f1531UL, 0x850e0e28UL, 0x42984f67UL, 0x03a9547eUL, 0xc0fa7955UL, 0x81cb624cUL, 0x1fc53881UL, 0x5ef42398UL, 0x9da70eb3UL, 0xdc9615aaUL, 0x1b0054e5UL, 0x5a314ffcUL, 0x996262d7UL, 0xd85379ceUL, 0x174fe149UL, 0x567efa50UL, 0x952dd77bUL, 0xd41ccc62UL, 0x138a8d2dUL, 0x52bb9634UL, 0x91e8bb1fUL, 0xd0d9a006UL, 0xecf37e5eUL, 0xadc26547UL, 0x6e91486cUL, 0x2fa05375UL, 0xe836123aUL, 0xa9070923UL, 0x6a542408UL, 0x2b653f11UL, 0xe479a796UL, 0xa548bc8fUL, 0x661b91a4UL, 0x272a8abdUL, 0xe0bccbf2UL, 0xa18dd0ebUL, 0x62defdc0UL, 0x23efe6d9UL, 0xbde1bc14UL, 0xfcd0a70dUL, 0x3f838a26UL, 0x7eb2913fUL, 0xb924d070UL, 0xf815cb69UL, 0x3b46e642UL, 0x7a77fd5bUL, 0xb56b65dcUL, 0xf45a7ec5UL, 0x370953eeUL, 0x763848f7UL, 0xb1ae09b8UL, 0xf09f12a1UL, 0x33cc3f8aUL, 0x72fd2493UL }, { 0x00000000UL, 0x376ac201UL, 0x6ed48403UL, 0x59be4602UL, 0xdca80907UL, 0xebc2cb06UL, 0xb27c8d04UL, 0x85164f05UL, 0xb851130eUL, 0x8f3bd10fUL, 0xd685970dUL, 0xe1ef550cUL, 0x64f91a09UL, 0x5393d808UL, 0x0a2d9e0aUL, 0x3d475c0bUL, 0x70a3261cUL, 0x47c9e41dUL, 0x1e77a21fUL, 0x291d601eUL, 0xac0b2f1bUL, 0x9b61ed1aUL, 0xc2dfab18UL, 0xf5b56919UL, 0xc8f23512UL, 0xff98f713UL, 0xa626b111UL, 0x914c7310UL, 0x145a3c15UL, 0x2330fe14UL, 0x7a8eb816UL, 0x4de47a17UL, 0xe0464d38UL, 0xd72c8f39UL, 0x8e92c93bUL, 0xb9f80b3aUL, 0x3cee443fUL, 0x0b84863eUL, 0x523ac03cUL, 0x6550023dUL, 0x58175e36UL, 0x6f7d9c37UL, 0x36c3da35UL, 0x01a91834UL, 0x84bf5731UL, 0xb3d59530UL, 0xea6bd332UL, 0xdd011133UL, 0x90e56b24UL, 0xa78fa925UL, 0xfe31ef27UL, 0xc95b2d26UL, 0x4c4d6223UL, 0x7b27a022UL, 0x2299e620UL, 0x15f32421UL, 0x28b4782aUL, 0x1fdeba2bUL, 0x4660fc29UL, 0x710a3e28UL, 0xf41c712dUL, 0xc376b32cUL, 0x9ac8f52eUL, 0xada2372fUL, 0xc08d9a70UL, 0xf7e75871UL, 0xae591e73UL, 0x9933dc72UL, 0x1c259377UL, 0x2b4f5176UL, 0x72f11774UL, 0x459bd575UL, 0x78dc897eUL, 0x4fb64b7fUL, 0x16080d7dUL, 0x2162cf7cUL, 0xa4748079UL, 0x931e4278UL, 0xcaa0047aUL, 0xfdcac67bUL, 0xb02ebc6cUL, 0x87447e6dUL, 0xdefa386fUL, 0xe990fa6eUL, 0x6c86b56bUL, 0x5bec776aUL, 0x02523168UL, 0x3538f369UL, 0x087faf62UL, 0x3f156d63UL, 0x66ab2b61UL, 0x51c1e960UL, 0xd4d7a665UL, 0xe3bd6464UL, 0xba032266UL, 0x8d69e067UL, 0x20cbd748UL, 0x17a11549UL, 0x4e1f534bUL, 0x7975914aUL, 0xfc63de4fUL, 0xcb091c4eUL, 0x92b75a4cUL, 0xa5dd984dUL, 0x989ac446UL, 0xaff00647UL, 0xf64e4045UL, 0xc1248244UL, 0x4432cd41UL, 0x73580f40UL, 0x2ae64942UL, 0x1d8c8b43UL, 0x5068f154UL, 0x67023355UL, 0x3ebc7557UL, 0x09d6b756UL, 0x8cc0f853UL, 0xbbaa3a52UL, 0xe2147c50UL, 0xd57ebe51UL, 0xe839e25aUL, 0xdf53205bUL, 0x86ed6659UL, 0xb187a458UL, 0x3491eb5dUL, 0x03fb295cUL, 0x5a456f5eUL, 0x6d2fad5fUL, 0x801b35e1UL, 0xb771f7e0UL, 0xeecfb1e2UL, 0xd9a573e3UL, 0x5cb33ce6UL, 0x6bd9fee7UL, 0x3267b8e5UL, 0x050d7ae4UL, 0x384a26efUL, 0x0f20e4eeUL, 0x569ea2ecUL, 0x61f460edUL, 0xe4e22fe8UL, 0xd388ede9UL, 0x8a36abebUL, 0xbd5c69eaUL, 0xf0b813fdUL, 0xc7d2d1fcUL, 0x9e6c97feUL, 0xa90655ffUL, 0x2c101afaUL, 0x1b7ad8fbUL, 0x42c49ef9UL, 0x75ae5cf8UL, 0x48e900f3UL, 0x7f83c2f2UL, 0x263d84f0UL, 0x115746f1UL, 0x944109f4UL, 0xa32bcbf5UL, 0xfa958df7UL, 0xcdff4ff6UL, 0x605d78d9UL, 0x5737bad8UL, 0x0e89fcdaUL, 0x39e33edbUL, 0xbcf571deUL, 0x8b9fb3dfUL, 0xd221f5ddUL, 0xe54b37dcUL, 0xd80c6bd7UL, 0xef66a9d6UL, 0xb6d8efd4UL, 0x81b22dd5UL, 0x04a462d0UL, 0x33cea0d1UL, 0x6a70e6d3UL, 0x5d1a24d2UL, 0x10fe5ec5UL, 0x27949cc4UL, 0x7e2adac6UL, 0x494018c7UL, 0xcc5657c2UL, 0xfb3c95c3UL, 0xa282d3c1UL, 0x95e811c0UL, 0xa8af4dcbUL, 0x9fc58fcaUL, 0xc67bc9c8UL, 0xf1110bc9UL, 0x740744ccUL, 0x436d86cdUL, 0x1ad3c0cfUL, 0x2db902ceUL, 0x4096af91UL, 0x77fc6d90UL, 0x2e422b92UL, 0x1928e993UL, 0x9c3ea696UL, 0xab546497UL, 0xf2ea2295UL, 0xc580e094UL, 0xf8c7bc9fUL, 0xcfad7e9eUL, 0x9613389cUL, 0xa179fa9dUL, 0x246fb598UL, 0x13057799UL, 0x4abb319bUL, 0x7dd1f39aUL, 0x3035898dUL, 0x075f4b8cUL, 0x5ee10d8eUL, 0x698bcf8fUL, 0xec9d808aUL, 0xdbf7428bUL, 0x82490489UL, 0xb523c688UL, 0x88649a83UL, 0xbf0e5882UL, 0xe6b01e80UL, 0xd1dadc81UL, 0x54cc9384UL, 0x63a65185UL, 0x3a181787UL, 0x0d72d586UL, 0xa0d0e2a9UL, 0x97ba20a8UL, 0xce0466aaUL, 0xf96ea4abUL, 0x7c78ebaeUL, 0x4b1229afUL, 0x12ac6fadUL, 0x25c6adacUL, 0x1881f1a7UL, 0x2feb33a6UL, 0x765575a4UL, 0x413fb7a5UL, 0xc429f8a0UL, 0xf3433aa1UL, 0xaafd7ca3UL, 0x9d97bea2UL, 0xd073c4b5UL, 0xe71906b4UL, 0xbea740b6UL, 0x89cd82b7UL, 0x0cdbcdb2UL, 0x3bb10fb3UL, 0x620f49b1UL, 0x55658bb0UL, 0x6822d7bbUL, 0x5f4815baUL, 0x06f653b8UL, 0x319c91b9UL, 0xb48adebcUL, 0x83e01cbdUL, 0xda5e5abfUL, 0xed3498beUL }, { 0x00000000UL, 0x6567bcb8UL, 0x8bc809aaUL, 0xeeafb512UL, 0x5797628fUL, 0x32f0de37UL, 0xdc5f6b25UL, 0xb938d79dUL, 0xef28b4c5UL, 0x8a4f087dUL, 0x64e0bd6fUL, 0x018701d7UL, 0xb8bfd64aUL, 0xddd86af2UL, 0x3377dfe0UL, 0x56106358UL, 0x9f571950UL, 0xfa30a5e8UL, 0x149f10faUL, 0x71f8ac42UL, 0xc8c07bdfUL, 0xada7c767UL, 0x43087275UL, 0x266fcecdUL, 0x707fad95UL, 0x1518112dUL, 0xfbb7a43fUL, 0x9ed01887UL, 0x27e8cf1aUL, 0x428f73a2UL, 0xac20c6b0UL, 0xc9477a08UL, 0x3eaf32a0UL, 0x5bc88e18UL, 0xb5673b0aUL, 0xd00087b2UL, 0x6938502fUL, 0x0c5fec97UL, 0xe2f05985UL, 0x8797e53dUL, 0xd1878665UL, 0xb4e03addUL, 0x5a4f8fcfUL, 0x3f283377UL, 0x8610e4eaUL, 0xe3775852UL, 0x0dd8ed40UL, 0x68bf51f8UL, 0xa1f82bf0UL, 0xc49f9748UL, 0x2a30225aUL, 0x4f579ee2UL, 0xf66f497fUL, 0x9308f5c7UL, 0x7da740d5UL, 0x18c0fc6dUL, 0x4ed09f35UL, 0x2bb7238dUL, 0xc518969fUL, 0xa07f2a27UL, 0x1947fdbaUL, 0x7c204102UL, 0x928ff410UL, 0xf7e848a8UL, 0x3d58149bUL, 0x583fa823UL, 0xb6901d31UL, 0xd3f7a189UL, 0x6acf7614UL, 0x0fa8caacUL, 0xe1077fbeUL, 0x8460c306UL, 0xd270a05eUL, 0xb7171ce6UL, 0x59b8a9f4UL, 0x3cdf154cUL, 0x85e7c2d1UL, 0xe0807e69UL, 0x0e2fcb7bUL, 0x6b4877c3UL, 0xa20f0dcbUL, 0xc768b173UL, 0x29c70461UL, 0x4ca0b8d9UL, 0xf5986f44UL, 0x90ffd3fcUL, 0x7e5066eeUL, 0x1b37da56UL, 0x4d27b90eUL, 0x284005b6UL, 0xc6efb0a4UL, 0xa3880c1cUL, 0x1ab0db81UL, 0x7fd76739UL, 0x9178d22bUL, 0xf41f6e93UL, 0x03f7263bUL, 0x66909a83UL, 0x883f2f91UL, 0xed589329UL, 0x546044b4UL, 0x3107f80cUL, 0xdfa84d1eUL, 0xbacff1a6UL, 0xecdf92feUL, 0x89b82e46UL, 0x67179b54UL, 0x027027ecUL, 0xbb48f071UL, 0xde2f4cc9UL, 0x3080f9dbUL, 0x55e74563UL, 0x9ca03f6bUL, 0xf9c783d3UL, 0x176836c1UL, 0x720f8a79UL, 0xcb375de4UL, 0xae50e15cUL, 0x40ff544eUL, 0x2598e8f6UL, 0x73888baeUL, 0x16ef3716UL, 0xf8408204UL, 0x9d273ebcUL, 0x241fe921UL, 0x41785599UL, 0xafd7e08bUL, 0xcab05c33UL, 0x3bb659edUL, 0x5ed1e555UL, 0xb07e5047UL, 0xd519ecffUL, 0x6c213b62UL, 0x094687daUL, 0xe7e932c8UL, 0x828e8e70UL, 0xd49eed28UL, 0xb1f95190UL, 0x5f56e482UL, 0x3a31583aUL, 0x83098fa7UL, 0xe66e331fUL, 0x08c1860dUL, 0x6da63ab5UL, 0xa4e140bdUL, 0xc186fc05UL, 0x2f294917UL, 0x4a4ef5afUL, 0xf3762232UL, 0x96119e8aUL, 0x78be2b98UL, 0x1dd99720UL, 0x4bc9f478UL, 0x2eae48c0UL, 0xc001fdd2UL, 0xa566416aUL, 0x1c5e96f7UL, 0x79392a4fUL, 0x97969f5dUL, 0xf2f123e5UL, 0x05196b4dUL, 0x607ed7f5UL, 0x8ed162e7UL, 0xebb6de5fUL, 0x528e09c2UL, 0x37e9b57aUL, 0xd9460068UL, 0xbc21bcd0UL, 0xea31df88UL, 0x8f566330UL, 0x61f9d622UL, 0x049e6a9aUL, 0xbda6bd07UL, 0xd8c101bfUL, 0x366eb4adUL, 0x53090815UL, 0x9a4e721dUL, 0xff29cea5UL, 0x11867bb7UL, 0x74e1c70fUL, 0xcdd91092UL, 0xa8beac2aUL, 0x46111938UL, 0x2376a580UL, 0x7566c6d8UL, 0x10017a60UL, 0xfeaecf72UL, 0x9bc973caUL, 0x22f1a457UL, 0x479618efUL, 0xa939adfdUL, 0xcc5e1145UL, 0x06ee4d76UL, 0x6389f1ceUL, 0x8d2644dcUL, 0xe841f864UL, 0x51792ff9UL, 0x341e9341UL, 0xdab12653UL, 0xbfd69aebUL, 0xe9c6f9b3UL, 0x8ca1450bUL, 0x620ef019UL, 0x07694ca1UL, 0xbe519b3cUL, 0xdb362784UL, 0x35999296UL, 0x50fe2e2eUL, 0x99b95426UL, 0xfcdee89eUL, 0x12715d8cUL, 0x7716e134UL, 0xce2e36a9UL, 0xab498a11UL, 0x45e63f03UL, 0x208183bbUL, 0x7691e0e3UL, 0x13f65c5bUL, 0xfd59e949UL, 0x983e55f1UL, 0x2106826cUL, 0x44613ed4UL, 0xaace8bc6UL, 0xcfa9377eUL, 0x38417fd6UL, 0x5d26c36eUL, 0xb389767cUL, 0xd6eecac4UL, 0x6fd61d59UL, 0x0ab1a1e1UL, 0xe41e14f3UL, 0x8179a84bUL, 0xd769cb13UL, 0xb20e77abUL, 0x5ca1c2b9UL, 0x39c67e01UL, 0x80fea99cUL, 0xe5991524UL, 0x0b36a036UL, 0x6e511c8eUL, 0xa7166686UL, 0xc271da3eUL, 0x2cde6f2cUL, 0x49b9d394UL, 0xf0810409UL, 0x95e6b8b1UL, 0x7b490da3UL, 0x1e2eb11bUL, 0x483ed243UL, 0x2d596efbUL, 0xc3f6dbe9UL, 0xa6916751UL, 0x1fa9b0ccUL, 0x7ace0c74UL, 0x9461b966UL, 0xf10605deUL #endif } }; c-blosc-1.7.0/internal-complibs/zlib-1.2.8/deflate.c000066400000000000000000002134641254622165100216550ustar00rootroot00000000000000/* deflate.c -- compress data using the deflation algorithm * Copyright (C) 1995-2013 Jean-loup Gailly and Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ /* * ALGORITHM * * The "deflation" process depends on being able to identify portions * of the input text which are identical to earlier input (within a * sliding window trailing behind the input currently being processed). * * The most straightforward technique turns out to be the fastest for * most input files: try all possible matches and select the longest. * The key feature of this algorithm is that insertions into the string * dictionary are very simple and thus fast, and deletions are avoided * completely. Insertions are performed at each input character, whereas * string matches are performed only when the previous match ends. So it * is preferable to spend more time in matches to allow very fast string * insertions and avoid deletions. The matching algorithm for small * strings is inspired from that of Rabin & Karp. A brute force approach * is used to find longer strings when a small match has been found. * A similar algorithm is used in comic (by Jan-Mark Wams) and freeze * (by Leonid Broukhis). * A previous version of this file used a more sophisticated algorithm * (by Fiala and Greene) which is guaranteed to run in linear amortized * time, but has a larger average cost, uses more memory and is patented. * However the F&G algorithm may be faster for some highly redundant * files if the parameter max_chain_length (described below) is too large. * * ACKNOWLEDGEMENTS * * The idea of lazy evaluation of matches is due to Jan-Mark Wams, and * I found it in 'freeze' written by Leonid Broukhis. * Thanks to many people for bug reports and testing. * * REFERENCES * * Deutsch, L.P.,"DEFLATE Compressed Data Format Specification". * Available in http://tools.ietf.org/html/rfc1951 * * A description of the Rabin and Karp algorithm is given in the book * "Algorithms" by R. Sedgewick, Addison-Wesley, p252. * * Fiala,E.R., and Greene,D.H. * Data Compression with Finite Windows, Comm.ACM, 32,4 (1989) 490-595 * */ /* @(#) $Id$ */ #include "deflate.h" const char deflate_copyright[] = " deflate 1.2.8 Copyright 1995-2013 Jean-loup Gailly and Mark Adler "; /* If you use the zlib library in a product, an acknowledgment is welcome in the documentation of your product. If for some reason you cannot include such an acknowledgment, I would appreciate that you keep this copyright string in the executable of your product. */ /* =========================================================================== * Function prototypes. */ typedef enum { need_more, /* block not completed, need more input or more output */ block_done, /* block flush performed */ finish_started, /* finish started, need only more output at next deflate */ finish_done /* finish done, accept no more input or output */ } block_state; typedef block_state (*compress_func) OF((deflate_state *s, int flush)); /* Compression function. Returns the block state after the call. */ local void fill_window OF((deflate_state *s)); local block_state deflate_stored OF((deflate_state *s, int flush)); local block_state deflate_fast OF((deflate_state *s, int flush)); #ifndef FASTEST local block_state deflate_slow OF((deflate_state *s, int flush)); #endif local block_state deflate_rle OF((deflate_state *s, int flush)); local block_state deflate_huff OF((deflate_state *s, int flush)); local void lm_init OF((deflate_state *s)); local void putShortMSB OF((deflate_state *s, uInt b)); local void flush_pending OF((z_streamp strm)); local int read_buf OF((z_streamp strm, Bytef *buf, unsigned size)); #ifdef ASMV void match_init OF((void)); /* asm code initialization */ uInt longest_match OF((deflate_state *s, IPos cur_match)); #else local uInt longest_match OF((deflate_state *s, IPos cur_match)); #endif #ifdef DEBUG local void check_match OF((deflate_state *s, IPos start, IPos match, int length)); #endif /* =========================================================================== * Local data */ #define NIL 0 /* Tail of hash chains */ #ifndef TOO_FAR # define TOO_FAR 4096 #endif /* Matches of length 3 are discarded if their distance exceeds TOO_FAR */ /* Values for max_lazy_match, good_match and max_chain_length, depending on * the desired pack level (0..9). The values given below have been tuned to * exclude worst case performance for pathological files. Better values may be * found for specific files. */ typedef struct config_s { ush good_length; /* reduce lazy search above this match length */ ush max_lazy; /* do not perform lazy search above this match length */ ush nice_length; /* quit search above this match length */ ush max_chain; compress_func func; } config; #ifdef FASTEST local const config configuration_table[2] = { /* good lazy nice chain */ /* 0 */ {0, 0, 0, 0, deflate_stored}, /* store only */ /* 1 */ {4, 4, 8, 4, deflate_fast}}; /* max speed, no lazy matches */ #else local const config configuration_table[10] = { /* good lazy nice chain */ /* 0 */ {0, 0, 0, 0, deflate_stored}, /* store only */ /* 1 */ {4, 4, 8, 4, deflate_fast}, /* max speed, no lazy matches */ /* 2 */ {4, 5, 16, 8, deflate_fast}, /* 3 */ {4, 6, 32, 32, deflate_fast}, /* 4 */ {4, 4, 16, 16, deflate_slow}, /* lazy matches */ /* 5 */ {8, 16, 32, 32, deflate_slow}, /* 6 */ {8, 16, 128, 128, deflate_slow}, /* 7 */ {8, 32, 128, 256, deflate_slow}, /* 8 */ {32, 128, 258, 1024, deflate_slow}, /* 9 */ {32, 258, 258, 4096, deflate_slow}}; /* max compression */ #endif /* Note: the deflate() code requires max_lazy >= MIN_MATCH and max_chain >= 4 * For deflate_fast() (levels <= 3) good is ignored and lazy has a different * meaning. */ #define EQUAL 0 /* result of memcmp for equal strings */ #ifndef NO_DUMMY_DECL struct static_tree_desc_s {int dummy;}; /* for buggy compilers */ #endif /* rank Z_BLOCK between Z_NO_FLUSH and Z_PARTIAL_FLUSH */ #define RANK(f) (((f) << 1) - ((f) > 4 ? 9 : 0)) /* =========================================================================== * Update a hash value with the given input byte * IN assertion: all calls to to UPDATE_HASH are made with consecutive * input characters, so that a running hash key can be computed from the * previous key instead of complete recalculation each time. */ #define UPDATE_HASH(s,h,c) (h = (((h)<hash_shift) ^ (c)) & s->hash_mask) /* =========================================================================== * Insert string str in the dictionary and set match_head to the previous head * of the hash chain (the most recent string with same hash key). Return * the previous length of the hash chain. * If this file is compiled with -DFASTEST, the compression level is forced * to 1, and no hash chains are maintained. * IN assertion: all calls to to INSERT_STRING are made with consecutive * input characters and the first MIN_MATCH bytes of str are valid * (except for the last MIN_MATCH-1 bytes of the input file). */ #ifdef FASTEST #define INSERT_STRING(s, str, match_head) \ (UPDATE_HASH(s, s->ins_h, s->window[(str) + (MIN_MATCH-1)]), \ match_head = s->head[s->ins_h], \ s->head[s->ins_h] = (Pos)(str)) #else #define INSERT_STRING(s, str, match_head) \ (UPDATE_HASH(s, s->ins_h, s->window[(str) + (MIN_MATCH-1)]), \ match_head = s->prev[(str) & s->w_mask] = s->head[s->ins_h], \ s->head[s->ins_h] = (Pos)(str)) #endif /* =========================================================================== * Initialize the hash table (avoiding 64K overflow for 16 bit systems). * prev[] will be initialized on the fly. */ #define CLEAR_HASH(s) \ s->head[s->hash_size-1] = NIL; \ zmemzero((Bytef *)s->head, (unsigned)(s->hash_size-1)*sizeof(*s->head)); /* ========================================================================= */ int ZEXPORT deflateInit_(strm, level, version, stream_size) z_streamp strm; int level; const char *version; int stream_size; { return deflateInit2_(strm, level, Z_DEFLATED, MAX_WBITS, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY, version, stream_size); /* To do: ignore strm->next_in if we use it as window */ } /* ========================================================================= */ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy, version, stream_size) z_streamp strm; int level; int method; int windowBits; int memLevel; int strategy; const char *version; int stream_size; { deflate_state *s; int wrap = 1; static const char my_version[] = ZLIB_VERSION; ushf *overlay; /* We overlay pending_buf and d_buf+l_buf. This works since the average * output size for (length,distance) codes is <= 24 bits. */ if (version == Z_NULL || version[0] != my_version[0] || stream_size != sizeof(z_stream)) { return Z_VERSION_ERROR; } if (strm == Z_NULL) return Z_STREAM_ERROR; strm->msg = Z_NULL; if (strm->zalloc == (alloc_func)0) { #ifdef Z_SOLO return Z_STREAM_ERROR; #else strm->zalloc = zcalloc; strm->opaque = (voidpf)0; #endif } if (strm->zfree == (free_func)0) #ifdef Z_SOLO return Z_STREAM_ERROR; #else strm->zfree = zcfree; #endif #ifdef FASTEST if (level != 0) level = 1; #else if (level == Z_DEFAULT_COMPRESSION) level = 6; #endif if (windowBits < 0) { /* suppress zlib wrapper */ wrap = 0; windowBits = -windowBits; } #ifdef GZIP else if (windowBits > 15) { wrap = 2; /* write gzip wrapper instead */ windowBits -= 16; } #endif if (memLevel < 1 || memLevel > MAX_MEM_LEVEL || method != Z_DEFLATED || windowBits < 8 || windowBits > 15 || level < 0 || level > 9 || strategy < 0 || strategy > Z_FIXED) { return Z_STREAM_ERROR; } if (windowBits == 8) windowBits = 9; /* until 256-byte window bug fixed */ s = (deflate_state *) ZALLOC(strm, 1, sizeof(deflate_state)); if (s == Z_NULL) return Z_MEM_ERROR; strm->state = (struct internal_state FAR *)s; s->strm = strm; s->wrap = wrap; s->gzhead = Z_NULL; s->w_bits = windowBits; s->w_size = 1 << s->w_bits; s->w_mask = s->w_size - 1; s->hash_bits = memLevel + 7; s->hash_size = 1 << s->hash_bits; s->hash_mask = s->hash_size - 1; s->hash_shift = ((s->hash_bits+MIN_MATCH-1)/MIN_MATCH); s->window = (Bytef *) ZALLOC(strm, s->w_size, 2*sizeof(Byte)); s->prev = (Posf *) ZALLOC(strm, s->w_size, sizeof(Pos)); s->head = (Posf *) ZALLOC(strm, s->hash_size, sizeof(Pos)); s->high_water = 0; /* nothing written to s->window yet */ s->lit_bufsize = 1 << (memLevel + 6); /* 16K elements by default */ overlay = (ushf *) ZALLOC(strm, s->lit_bufsize, sizeof(ush)+2); s->pending_buf = (uchf *) overlay; s->pending_buf_size = (ulg)s->lit_bufsize * (sizeof(ush)+2L); if (s->window == Z_NULL || s->prev == Z_NULL || s->head == Z_NULL || s->pending_buf == Z_NULL) { s->status = FINISH_STATE; strm->msg = ERR_MSG(Z_MEM_ERROR); deflateEnd (strm); return Z_MEM_ERROR; } s->d_buf = overlay + s->lit_bufsize/sizeof(ush); s->l_buf = s->pending_buf + (1+sizeof(ush))*s->lit_bufsize; s->level = level; s->strategy = strategy; s->method = (Byte)method; return deflateReset(strm); } /* ========================================================================= */ int ZEXPORT deflateSetDictionary (strm, dictionary, dictLength) z_streamp strm; const Bytef *dictionary; uInt dictLength; { deflate_state *s; uInt str, n; int wrap; unsigned avail; z_const unsigned char *next; if (strm == Z_NULL || strm->state == Z_NULL || dictionary == Z_NULL) return Z_STREAM_ERROR; s = strm->state; wrap = s->wrap; if (wrap == 2 || (wrap == 1 && s->status != INIT_STATE) || s->lookahead) return Z_STREAM_ERROR; /* when using zlib wrappers, compute Adler-32 for provided dictionary */ if (wrap == 1) strm->adler = adler32(strm->adler, dictionary, dictLength); s->wrap = 0; /* avoid computing Adler-32 in read_buf */ /* if dictionary would fill window, just replace the history */ if (dictLength >= s->w_size) { if (wrap == 0) { /* already empty otherwise */ CLEAR_HASH(s); s->strstart = 0; s->block_start = 0L; s->insert = 0; } dictionary += dictLength - s->w_size; /* use the tail */ dictLength = s->w_size; } /* insert dictionary into window and hash */ avail = strm->avail_in; next = strm->next_in; strm->avail_in = dictLength; strm->next_in = (z_const Bytef *)dictionary; fill_window(s); while (s->lookahead >= MIN_MATCH) { str = s->strstart; n = s->lookahead - (MIN_MATCH-1); do { UPDATE_HASH(s, s->ins_h, s->window[str + MIN_MATCH-1]); #ifndef FASTEST s->prev[str & s->w_mask] = s->head[s->ins_h]; #endif s->head[s->ins_h] = (Pos)str; str++; } while (--n); s->strstart = str; s->lookahead = MIN_MATCH-1; fill_window(s); } s->strstart += s->lookahead; s->block_start = (long)s->strstart; s->insert = s->lookahead; s->lookahead = 0; s->match_length = s->prev_length = MIN_MATCH-1; s->match_available = 0; strm->next_in = next; strm->avail_in = avail; s->wrap = wrap; return Z_OK; } /* ========================================================================= */ int ZEXPORT deflateResetKeep (strm) z_streamp strm; { deflate_state *s; if (strm == Z_NULL || strm->state == Z_NULL || strm->zalloc == (alloc_func)0 || strm->zfree == (free_func)0) { return Z_STREAM_ERROR; } strm->total_in = strm->total_out = 0; strm->msg = Z_NULL; /* use zfree if we ever allocate msg dynamically */ strm->data_type = Z_UNKNOWN; s = (deflate_state *)strm->state; s->pending = 0; s->pending_out = s->pending_buf; if (s->wrap < 0) { s->wrap = -s->wrap; /* was made negative by deflate(..., Z_FINISH); */ } s->status = s->wrap ? INIT_STATE : BUSY_STATE; strm->adler = #ifdef GZIP s->wrap == 2 ? crc32(0L, Z_NULL, 0) : #endif adler32(0L, Z_NULL, 0); s->last_flush = Z_NO_FLUSH; _tr_init(s); return Z_OK; } /* ========================================================================= */ int ZEXPORT deflateReset (strm) z_streamp strm; { int ret; ret = deflateResetKeep(strm); if (ret == Z_OK) lm_init(strm->state); return ret; } /* ========================================================================= */ int ZEXPORT deflateSetHeader (strm, head) z_streamp strm; gz_headerp head; { if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; if (strm->state->wrap != 2) return Z_STREAM_ERROR; strm->state->gzhead = head; return Z_OK; } /* ========================================================================= */ int ZEXPORT deflatePending (strm, pending, bits) unsigned *pending; int *bits; z_streamp strm; { if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; if (pending != Z_NULL) *pending = strm->state->pending; if (bits != Z_NULL) *bits = strm->state->bi_valid; return Z_OK; } /* ========================================================================= */ int ZEXPORT deflatePrime (strm, bits, value) z_streamp strm; int bits; int value; { deflate_state *s; int put; if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; s = strm->state; if ((Bytef *)(s->d_buf) < s->pending_out + ((Buf_size + 7) >> 3)) return Z_BUF_ERROR; do { put = Buf_size - s->bi_valid; if (put > bits) put = bits; s->bi_buf |= (ush)((value & ((1 << put) - 1)) << s->bi_valid); s->bi_valid += put; _tr_flush_bits(s); value >>= put; bits -= put; } while (bits); return Z_OK; } /* ========================================================================= */ int ZEXPORT deflateParams(strm, level, strategy) z_streamp strm; int level; int strategy; { deflate_state *s; compress_func func; int err = Z_OK; if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; s = strm->state; #ifdef FASTEST if (level != 0) level = 1; #else if (level == Z_DEFAULT_COMPRESSION) level = 6; #endif if (level < 0 || level > 9 || strategy < 0 || strategy > Z_FIXED) { return Z_STREAM_ERROR; } func = configuration_table[s->level].func; if ((strategy != s->strategy || func != configuration_table[level].func) && strm->total_in != 0) { /* Flush the last buffer: */ err = deflate(strm, Z_BLOCK); if (err == Z_BUF_ERROR && s->pending == 0) err = Z_OK; } if (s->level != level) { s->level = level; s->max_lazy_match = configuration_table[level].max_lazy; s->good_match = configuration_table[level].good_length; s->nice_match = configuration_table[level].nice_length; s->max_chain_length = configuration_table[level].max_chain; } s->strategy = strategy; return err; } /* ========================================================================= */ int ZEXPORT deflateTune(strm, good_length, max_lazy, nice_length, max_chain) z_streamp strm; int good_length; int max_lazy; int nice_length; int max_chain; { deflate_state *s; if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; s = strm->state; s->good_match = good_length; s->max_lazy_match = max_lazy; s->nice_match = nice_length; s->max_chain_length = max_chain; return Z_OK; } /* ========================================================================= * For the default windowBits of 15 and memLevel of 8, this function returns * a close to exact, as well as small, upper bound on the compressed size. * They are coded as constants here for a reason--if the #define's are * changed, then this function needs to be changed as well. The return * value for 15 and 8 only works for those exact settings. * * For any setting other than those defaults for windowBits and memLevel, * the value returned is a conservative worst case for the maximum expansion * resulting from using fixed blocks instead of stored blocks, which deflate * can emit on compressed data for some combinations of the parameters. * * This function could be more sophisticated to provide closer upper bounds for * every combination of windowBits and memLevel. But even the conservative * upper bound of about 14% expansion does not seem onerous for output buffer * allocation. */ uLong ZEXPORT deflateBound(strm, sourceLen) z_streamp strm; uLong sourceLen; { deflate_state *s; uLong complen, wraplen; Bytef *str; /* conservative upper bound for compressed data */ complen = sourceLen + ((sourceLen + 7) >> 3) + ((sourceLen + 63) >> 6) + 5; /* if can't get parameters, return conservative bound plus zlib wrapper */ if (strm == Z_NULL || strm->state == Z_NULL) return complen + 6; /* compute wrapper length */ s = strm->state; switch (s->wrap) { case 0: /* raw deflate */ wraplen = 0; break; case 1: /* zlib wrapper */ wraplen = 6 + (s->strstart ? 4 : 0); break; case 2: /* gzip wrapper */ wraplen = 18; if (s->gzhead != Z_NULL) { /* user-supplied gzip header */ if (s->gzhead->extra != Z_NULL) wraplen += 2 + s->gzhead->extra_len; str = s->gzhead->name; if (str != Z_NULL) do { wraplen++; } while (*str++); str = s->gzhead->comment; if (str != Z_NULL) do { wraplen++; } while (*str++); if (s->gzhead->hcrc) wraplen += 2; } break; default: /* for compiler happiness */ wraplen = 6; } /* if not default parameters, return conservative bound */ if (s->w_bits != 15 || s->hash_bits != 8 + 7) return complen + wraplen; /* default settings: return tight bound for that case */ return sourceLen + (sourceLen >> 12) + (sourceLen >> 14) + (sourceLen >> 25) + 13 - 6 + wraplen; } /* ========================================================================= * Put a short in the pending buffer. The 16-bit value is put in MSB order. * IN assertion: the stream state is correct and there is enough room in * pending_buf. */ local void putShortMSB (s, b) deflate_state *s; uInt b; { put_byte(s, (Byte)(b >> 8)); put_byte(s, (Byte)(b & 0xff)); } /* ========================================================================= * Flush as much pending output as possible. All deflate() output goes * through this function so some applications may wish to modify it * to avoid allocating a large strm->next_out buffer and copying into it. * (See also read_buf()). */ local void flush_pending(strm) z_streamp strm; { unsigned len; deflate_state *s = strm->state; _tr_flush_bits(s); len = s->pending; if (len > strm->avail_out) len = strm->avail_out; if (len == 0) return; zmemcpy(strm->next_out, s->pending_out, len); strm->next_out += len; s->pending_out += len; strm->total_out += len; strm->avail_out -= len; s->pending -= len; if (s->pending == 0) { s->pending_out = s->pending_buf; } } /* ========================================================================= */ int ZEXPORT deflate (strm, flush) z_streamp strm; int flush; { int old_flush; /* value of flush param for previous deflate call */ deflate_state *s; if (strm == Z_NULL || strm->state == Z_NULL || flush > Z_BLOCK || flush < 0) { return Z_STREAM_ERROR; } s = strm->state; if (strm->next_out == Z_NULL || (strm->next_in == Z_NULL && strm->avail_in != 0) || (s->status == FINISH_STATE && flush != Z_FINISH)) { ERR_RETURN(strm, Z_STREAM_ERROR); } if (strm->avail_out == 0) ERR_RETURN(strm, Z_BUF_ERROR); s->strm = strm; /* just in case */ old_flush = s->last_flush; s->last_flush = flush; /* Write the header */ if (s->status == INIT_STATE) { #ifdef GZIP if (s->wrap == 2) { strm->adler = crc32(0L, Z_NULL, 0); put_byte(s, 31); put_byte(s, 139); put_byte(s, 8); if (s->gzhead == Z_NULL) { put_byte(s, 0); put_byte(s, 0); put_byte(s, 0); put_byte(s, 0); put_byte(s, 0); put_byte(s, s->level == 9 ? 2 : (s->strategy >= Z_HUFFMAN_ONLY || s->level < 2 ? 4 : 0)); put_byte(s, OS_CODE); s->status = BUSY_STATE; } else { put_byte(s, (s->gzhead->text ? 1 : 0) + (s->gzhead->hcrc ? 2 : 0) + (s->gzhead->extra == Z_NULL ? 0 : 4) + (s->gzhead->name == Z_NULL ? 0 : 8) + (s->gzhead->comment == Z_NULL ? 0 : 16) ); put_byte(s, (Byte)(s->gzhead->time & 0xff)); put_byte(s, (Byte)((s->gzhead->time >> 8) & 0xff)); put_byte(s, (Byte)((s->gzhead->time >> 16) & 0xff)); put_byte(s, (Byte)((s->gzhead->time >> 24) & 0xff)); put_byte(s, s->level == 9 ? 2 : (s->strategy >= Z_HUFFMAN_ONLY || s->level < 2 ? 4 : 0)); put_byte(s, s->gzhead->os & 0xff); if (s->gzhead->extra != Z_NULL) { put_byte(s, s->gzhead->extra_len & 0xff); put_byte(s, (s->gzhead->extra_len >> 8) & 0xff); } if (s->gzhead->hcrc) strm->adler = crc32(strm->adler, s->pending_buf, s->pending); s->gzindex = 0; s->status = EXTRA_STATE; } } else #endif { uInt header = (Z_DEFLATED + ((s->w_bits-8)<<4)) << 8; uInt level_flags; if (s->strategy >= Z_HUFFMAN_ONLY || s->level < 2) level_flags = 0; else if (s->level < 6) level_flags = 1; else if (s->level == 6) level_flags = 2; else level_flags = 3; header |= (level_flags << 6); if (s->strstart != 0) header |= PRESET_DICT; header += 31 - (header % 31); s->status = BUSY_STATE; putShortMSB(s, header); /* Save the adler32 of the preset dictionary: */ if (s->strstart != 0) { putShortMSB(s, (uInt)(strm->adler >> 16)); putShortMSB(s, (uInt)(strm->adler & 0xffff)); } strm->adler = adler32(0L, Z_NULL, 0); } } #ifdef GZIP if (s->status == EXTRA_STATE) { if (s->gzhead->extra != Z_NULL) { uInt beg = s->pending; /* start of bytes to update crc */ while (s->gzindex < (s->gzhead->extra_len & 0xffff)) { if (s->pending == s->pending_buf_size) { if (s->gzhead->hcrc && s->pending > beg) strm->adler = crc32(strm->adler, s->pending_buf + beg, s->pending - beg); flush_pending(strm); beg = s->pending; if (s->pending == s->pending_buf_size) break; } put_byte(s, s->gzhead->extra[s->gzindex]); s->gzindex++; } if (s->gzhead->hcrc && s->pending > beg) strm->adler = crc32(strm->adler, s->pending_buf + beg, s->pending - beg); if (s->gzindex == s->gzhead->extra_len) { s->gzindex = 0; s->status = NAME_STATE; } } else s->status = NAME_STATE; } if (s->status == NAME_STATE) { if (s->gzhead->name != Z_NULL) { uInt beg = s->pending; /* start of bytes to update crc */ int val; do { if (s->pending == s->pending_buf_size) { if (s->gzhead->hcrc && s->pending > beg) strm->adler = crc32(strm->adler, s->pending_buf + beg, s->pending - beg); flush_pending(strm); beg = s->pending; if (s->pending == s->pending_buf_size) { val = 1; break; } } val = s->gzhead->name[s->gzindex++]; put_byte(s, val); } while (val != 0); if (s->gzhead->hcrc && s->pending > beg) strm->adler = crc32(strm->adler, s->pending_buf + beg, s->pending - beg); if (val == 0) { s->gzindex = 0; s->status = COMMENT_STATE; } } else s->status = COMMENT_STATE; } if (s->status == COMMENT_STATE) { if (s->gzhead->comment != Z_NULL) { uInt beg = s->pending; /* start of bytes to update crc */ int val; do { if (s->pending == s->pending_buf_size) { if (s->gzhead->hcrc && s->pending > beg) strm->adler = crc32(strm->adler, s->pending_buf + beg, s->pending - beg); flush_pending(strm); beg = s->pending; if (s->pending == s->pending_buf_size) { val = 1; break; } } val = s->gzhead->comment[s->gzindex++]; put_byte(s, val); } while (val != 0); if (s->gzhead->hcrc && s->pending > beg) strm->adler = crc32(strm->adler, s->pending_buf + beg, s->pending - beg); if (val == 0) s->status = HCRC_STATE; } else s->status = HCRC_STATE; } if (s->status == HCRC_STATE) { if (s->gzhead->hcrc) { if (s->pending + 2 > s->pending_buf_size) flush_pending(strm); if (s->pending + 2 <= s->pending_buf_size) { put_byte(s, (Byte)(strm->adler & 0xff)); put_byte(s, (Byte)((strm->adler >> 8) & 0xff)); strm->adler = crc32(0L, Z_NULL, 0); s->status = BUSY_STATE; } } else s->status = BUSY_STATE; } #endif /* Flush as much pending output as possible */ if (s->pending != 0) { flush_pending(strm); if (strm->avail_out == 0) { /* Since avail_out is 0, deflate will be called again with * more output space, but possibly with both pending and * avail_in equal to zero. There won't be anything to do, * but this is not an error situation so make sure we * return OK instead of BUF_ERROR at next call of deflate: */ s->last_flush = -1; return Z_OK; } /* Make sure there is something to do and avoid duplicate consecutive * flushes. For repeated and useless calls with Z_FINISH, we keep * returning Z_STREAM_END instead of Z_BUF_ERROR. */ } else if (strm->avail_in == 0 && RANK(flush) <= RANK(old_flush) && flush != Z_FINISH) { ERR_RETURN(strm, Z_BUF_ERROR); } /* User must not provide more input after the first FINISH: */ if (s->status == FINISH_STATE && strm->avail_in != 0) { ERR_RETURN(strm, Z_BUF_ERROR); } /* Start a new block or continue the current one. */ if (strm->avail_in != 0 || s->lookahead != 0 || (flush != Z_NO_FLUSH && s->status != FINISH_STATE)) { block_state bstate; bstate = s->strategy == Z_HUFFMAN_ONLY ? deflate_huff(s, flush) : (s->strategy == Z_RLE ? deflate_rle(s, flush) : (*(configuration_table[s->level].func))(s, flush)); if (bstate == finish_started || bstate == finish_done) { s->status = FINISH_STATE; } if (bstate == need_more || bstate == finish_started) { if (strm->avail_out == 0) { s->last_flush = -1; /* avoid BUF_ERROR next call, see above */ } return Z_OK; /* If flush != Z_NO_FLUSH && avail_out == 0, the next call * of deflate should use the same flush parameter to make sure * that the flush is complete. So we don't have to output an * empty block here, this will be done at next call. This also * ensures that for a very small output buffer, we emit at most * one empty block. */ } if (bstate == block_done) { if (flush == Z_PARTIAL_FLUSH) { _tr_align(s); } else if (flush != Z_BLOCK) { /* FULL_FLUSH or SYNC_FLUSH */ _tr_stored_block(s, (char*)0, 0L, 0); /* For a full flush, this empty block will be recognized * as a special marker by inflate_sync(). */ if (flush == Z_FULL_FLUSH) { CLEAR_HASH(s); /* forget history */ if (s->lookahead == 0) { s->strstart = 0; s->block_start = 0L; s->insert = 0; } } } flush_pending(strm); if (strm->avail_out == 0) { s->last_flush = -1; /* avoid BUF_ERROR at next call, see above */ return Z_OK; } } } Assert(strm->avail_out > 0, "bug2"); if (flush != Z_FINISH) return Z_OK; if (s->wrap <= 0) return Z_STREAM_END; /* Write the trailer */ #ifdef GZIP if (s->wrap == 2) { put_byte(s, (Byte)(strm->adler & 0xff)); put_byte(s, (Byte)((strm->adler >> 8) & 0xff)); put_byte(s, (Byte)((strm->adler >> 16) & 0xff)); put_byte(s, (Byte)((strm->adler >> 24) & 0xff)); put_byte(s, (Byte)(strm->total_in & 0xff)); put_byte(s, (Byte)((strm->total_in >> 8) & 0xff)); put_byte(s, (Byte)((strm->total_in >> 16) & 0xff)); put_byte(s, (Byte)((strm->total_in >> 24) & 0xff)); } else #endif { putShortMSB(s, (uInt)(strm->adler >> 16)); putShortMSB(s, (uInt)(strm->adler & 0xffff)); } flush_pending(strm); /* If avail_out is zero, the application will call deflate again * to flush the rest. */ if (s->wrap > 0) s->wrap = -s->wrap; /* write the trailer only once! */ return s->pending != 0 ? Z_OK : Z_STREAM_END; } /* ========================================================================= */ int ZEXPORT deflateEnd (strm) z_streamp strm; { int status; if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; status = strm->state->status; if (status != INIT_STATE && status != EXTRA_STATE && status != NAME_STATE && status != COMMENT_STATE && status != HCRC_STATE && status != BUSY_STATE && status != FINISH_STATE) { return Z_STREAM_ERROR; } /* Deallocate in reverse order of allocations: */ TRY_FREE(strm, strm->state->pending_buf); TRY_FREE(strm, strm->state->head); TRY_FREE(strm, strm->state->prev); TRY_FREE(strm, strm->state->window); ZFREE(strm, strm->state); strm->state = Z_NULL; return status == BUSY_STATE ? Z_DATA_ERROR : Z_OK; } /* ========================================================================= * Copy the source state to the destination state. * To simplify the source, this is not supported for 16-bit MSDOS (which * doesn't have enough memory anyway to duplicate compression states). */ int ZEXPORT deflateCopy (dest, source) z_streamp dest; z_streamp source; { #ifdef MAXSEG_64K return Z_STREAM_ERROR; #else deflate_state *ds; deflate_state *ss; ushf *overlay; if (source == Z_NULL || dest == Z_NULL || source->state == Z_NULL) { return Z_STREAM_ERROR; } ss = source->state; zmemcpy((voidpf)dest, (voidpf)source, sizeof(z_stream)); ds = (deflate_state *) ZALLOC(dest, 1, sizeof(deflate_state)); if (ds == Z_NULL) return Z_MEM_ERROR; dest->state = (struct internal_state FAR *) ds; zmemcpy((voidpf)ds, (voidpf)ss, sizeof(deflate_state)); ds->strm = dest; ds->window = (Bytef *) ZALLOC(dest, ds->w_size, 2*sizeof(Byte)); ds->prev = (Posf *) ZALLOC(dest, ds->w_size, sizeof(Pos)); ds->head = (Posf *) ZALLOC(dest, ds->hash_size, sizeof(Pos)); overlay = (ushf *) ZALLOC(dest, ds->lit_bufsize, sizeof(ush)+2); ds->pending_buf = (uchf *) overlay; if (ds->window == Z_NULL || ds->prev == Z_NULL || ds->head == Z_NULL || ds->pending_buf == Z_NULL) { deflateEnd (dest); return Z_MEM_ERROR; } /* following zmemcpy do not work for 16-bit MSDOS */ zmemcpy(ds->window, ss->window, ds->w_size * 2 * sizeof(Byte)); zmemcpy((voidpf)ds->prev, (voidpf)ss->prev, ds->w_size * sizeof(Pos)); zmemcpy((voidpf)ds->head, (voidpf)ss->head, ds->hash_size * sizeof(Pos)); zmemcpy(ds->pending_buf, ss->pending_buf, (uInt)ds->pending_buf_size); ds->pending_out = ds->pending_buf + (ss->pending_out - ss->pending_buf); ds->d_buf = overlay + ds->lit_bufsize/sizeof(ush); ds->l_buf = ds->pending_buf + (1+sizeof(ush))*ds->lit_bufsize; ds->l_desc.dyn_tree = ds->dyn_ltree; ds->d_desc.dyn_tree = ds->dyn_dtree; ds->bl_desc.dyn_tree = ds->bl_tree; return Z_OK; #endif /* MAXSEG_64K */ } /* =========================================================================== * Read a new buffer from the current input stream, update the adler32 * and total number of bytes read. All deflate() input goes through * this function so some applications may wish to modify it to avoid * allocating a large strm->next_in buffer and copying from it. * (See also flush_pending()). */ local int read_buf(strm, buf, size) z_streamp strm; Bytef *buf; unsigned size; { unsigned len = strm->avail_in; if (len > size) len = size; if (len == 0) return 0; strm->avail_in -= len; zmemcpy(buf, strm->next_in, len); if (strm->state->wrap == 1) { strm->adler = adler32(strm->adler, buf, len); } #ifdef GZIP else if (strm->state->wrap == 2) { strm->adler = crc32(strm->adler, buf, len); } #endif strm->next_in += len; strm->total_in += len; return (int)len; } /* =========================================================================== * Initialize the "longest match" routines for a new zlib stream */ local void lm_init (s) deflate_state *s; { s->window_size = (ulg)2L*s->w_size; CLEAR_HASH(s); /* Set the default configuration parameters: */ s->max_lazy_match = configuration_table[s->level].max_lazy; s->good_match = configuration_table[s->level].good_length; s->nice_match = configuration_table[s->level].nice_length; s->max_chain_length = configuration_table[s->level].max_chain; s->strstart = 0; s->block_start = 0L; s->lookahead = 0; s->insert = 0; s->match_length = s->prev_length = MIN_MATCH-1; s->match_available = 0; s->ins_h = 0; #ifndef FASTEST #ifdef ASMV match_init(); /* initialize the asm code */ #endif #endif } #ifndef FASTEST /* =========================================================================== * Set match_start to the longest match starting at the given string and * return its length. Matches shorter or equal to prev_length are discarded, * in which case the result is equal to prev_length and match_start is * garbage. * IN assertions: cur_match is the head of the hash chain for the current * string (strstart) and its distance is <= MAX_DIST, and prev_length >= 1 * OUT assertion: the match length is not greater than s->lookahead. */ #ifndef ASMV /* For 80x86 and 680x0, an optimized version will be provided in match.asm or * match.S. The code will be functionally equivalent. */ local uInt longest_match(s, cur_match) deflate_state *s; IPos cur_match; /* current match */ { unsigned chain_length = s->max_chain_length;/* max hash chain length */ register Bytef *scan = s->window + s->strstart; /* current string */ register Bytef *match; /* matched string */ register int len; /* length of current match */ int best_len = s->prev_length; /* best match length so far */ int nice_match = s->nice_match; /* stop if match long enough */ IPos limit = s->strstart > (IPos)MAX_DIST(s) ? s->strstart - (IPos)MAX_DIST(s) : NIL; /* Stop when cur_match becomes <= limit. To simplify the code, * we prevent matches with the string of window index 0. */ Posf *prev = s->prev; uInt wmask = s->w_mask; #ifdef UNALIGNED_OK /* Compare two bytes at a time. Note: this is not always beneficial. * Try with and without -DUNALIGNED_OK to check. */ register Bytef *strend = s->window + s->strstart + MAX_MATCH - 1; register ush scan_start = *(ushf*)scan; register ush scan_end = *(ushf*)(scan+best_len-1); #else register Bytef *strend = s->window + s->strstart + MAX_MATCH; register Byte scan_end1 = scan[best_len-1]; register Byte scan_end = scan[best_len]; #endif /* The code is optimized for HASH_BITS >= 8 and MAX_MATCH-2 multiple of 16. * It is easy to get rid of this optimization if necessary. */ Assert(s->hash_bits >= 8 && MAX_MATCH == 258, "Code too clever"); /* Do not waste too much time if we already have a good match: */ if (s->prev_length >= s->good_match) { chain_length >>= 2; } /* Do not look for matches beyond the end of the input. This is necessary * to make deflate deterministic. */ if ((uInt)nice_match > s->lookahead) nice_match = s->lookahead; Assert((ulg)s->strstart <= s->window_size-MIN_LOOKAHEAD, "need lookahead"); do { Assert(cur_match < s->strstart, "no future"); match = s->window + cur_match; /* Skip to next match if the match length cannot increase * or if the match length is less than 2. Note that the checks below * for insufficient lookahead only occur occasionally for performance * reasons. Therefore uninitialized memory will be accessed, and * conditional jumps will be made that depend on those values. * However the length of the match is limited to the lookahead, so * the output of deflate is not affected by the uninitialized values. */ #if (defined(UNALIGNED_OK) && MAX_MATCH == 258) /* This code assumes sizeof(unsigned short) == 2. Do not use * UNALIGNED_OK if your compiler uses a different size. */ if (*(ushf*)(match+best_len-1) != scan_end || *(ushf*)match != scan_start) continue; /* It is not necessary to compare scan[2] and match[2] since they are * always equal when the other bytes match, given that the hash keys * are equal and that HASH_BITS >= 8. Compare 2 bytes at a time at * strstart+3, +5, ... up to strstart+257. We check for insufficient * lookahead only every 4th comparison; the 128th check will be made * at strstart+257. If MAX_MATCH-2 is not a multiple of 8, it is * necessary to put more guard bytes at the end of the window, or * to check more often for insufficient lookahead. */ Assert(scan[2] == match[2], "scan[2]?"); scan++, match++; do { } while (*(ushf*)(scan+=2) == *(ushf*)(match+=2) && *(ushf*)(scan+=2) == *(ushf*)(match+=2) && *(ushf*)(scan+=2) == *(ushf*)(match+=2) && *(ushf*)(scan+=2) == *(ushf*)(match+=2) && scan < strend); /* The funny "do {}" generates better code on most compilers */ /* Here, scan <= window+strstart+257 */ Assert(scan <= s->window+(unsigned)(s->window_size-1), "wild scan"); if (*scan == *match) scan++; len = (MAX_MATCH - 1) - (int)(strend-scan); scan = strend - (MAX_MATCH-1); #else /* UNALIGNED_OK */ if (match[best_len] != scan_end || match[best_len-1] != scan_end1 || *match != *scan || *++match != scan[1]) continue; /* The check at best_len-1 can be removed because it will be made * again later. (This heuristic is not always a win.) * It is not necessary to compare scan[2] and match[2] since they * are always equal when the other bytes match, given that * the hash keys are equal and that HASH_BITS >= 8. */ scan += 2, match++; Assert(*scan == *match, "match[2]?"); /* We check for insufficient lookahead only every 8th comparison; * the 256th check will be made at strstart+258. */ do { } while (*++scan == *++match && *++scan == *++match && *++scan == *++match && *++scan == *++match && *++scan == *++match && *++scan == *++match && *++scan == *++match && *++scan == *++match && scan < strend); Assert(scan <= s->window+(unsigned)(s->window_size-1), "wild scan"); len = MAX_MATCH - (int)(strend - scan); scan = strend - MAX_MATCH; #endif /* UNALIGNED_OK */ if (len > best_len) { s->match_start = cur_match; best_len = len; if (len >= nice_match) break; #ifdef UNALIGNED_OK scan_end = *(ushf*)(scan+best_len-1); #else scan_end1 = scan[best_len-1]; scan_end = scan[best_len]; #endif } } while ((cur_match = prev[cur_match & wmask]) > limit && --chain_length != 0); if ((uInt)best_len <= s->lookahead) return (uInt)best_len; return s->lookahead; } #endif /* ASMV */ #else /* FASTEST */ /* --------------------------------------------------------------------------- * Optimized version for FASTEST only */ local uInt longest_match(s, cur_match) deflate_state *s; IPos cur_match; /* current match */ { register Bytef *scan = s->window + s->strstart; /* current string */ register Bytef *match; /* matched string */ register int len; /* length of current match */ register Bytef *strend = s->window + s->strstart + MAX_MATCH; /* The code is optimized for HASH_BITS >= 8 and MAX_MATCH-2 multiple of 16. * It is easy to get rid of this optimization if necessary. */ Assert(s->hash_bits >= 8 && MAX_MATCH == 258, "Code too clever"); Assert((ulg)s->strstart <= s->window_size-MIN_LOOKAHEAD, "need lookahead"); Assert(cur_match < s->strstart, "no future"); match = s->window + cur_match; /* Return failure if the match length is less than 2: */ if (match[0] != scan[0] || match[1] != scan[1]) return MIN_MATCH-1; /* The check at best_len-1 can be removed because it will be made * again later. (This heuristic is not always a win.) * It is not necessary to compare scan[2] and match[2] since they * are always equal when the other bytes match, given that * the hash keys are equal and that HASH_BITS >= 8. */ scan += 2, match += 2; Assert(*scan == *match, "match[2]?"); /* We check for insufficient lookahead only every 8th comparison; * the 256th check will be made at strstart+258. */ do { } while (*++scan == *++match && *++scan == *++match && *++scan == *++match && *++scan == *++match && *++scan == *++match && *++scan == *++match && *++scan == *++match && *++scan == *++match && scan < strend); Assert(scan <= s->window+(unsigned)(s->window_size-1), "wild scan"); len = MAX_MATCH - (int)(strend - scan); if (len < MIN_MATCH) return MIN_MATCH - 1; s->match_start = cur_match; return (uInt)len <= s->lookahead ? (uInt)len : s->lookahead; } #endif /* FASTEST */ #ifdef DEBUG /* =========================================================================== * Check that the match at match_start is indeed a match. */ local void check_match(s, start, match, length) deflate_state *s; IPos start, match; int length; { /* check that the match is indeed a match */ if (zmemcmp(s->window + match, s->window + start, length) != EQUAL) { fprintf(stderr, " start %u, match %u, length %d\n", start, match, length); do { fprintf(stderr, "%c%c", s->window[match++], s->window[start++]); } while (--length != 0); z_error("invalid match"); } if (z_verbose > 1) { fprintf(stderr,"\\[%d,%d]", start-match, length); do { putc(s->window[start++], stderr); } while (--length != 0); } } #else # define check_match(s, start, match, length) #endif /* DEBUG */ /* =========================================================================== * Fill the window when the lookahead becomes insufficient. * Updates strstart and lookahead. * * IN assertion: lookahead < MIN_LOOKAHEAD * OUT assertions: strstart <= window_size-MIN_LOOKAHEAD * At least one byte has been read, or avail_in == 0; reads are * performed for at least two bytes (required for the zip translate_eol * option -- not supported here). */ local void fill_window(s) deflate_state *s; { register unsigned n, m; register Posf *p; unsigned more; /* Amount of free space at the end of the window. */ uInt wsize = s->w_size; Assert(s->lookahead < MIN_LOOKAHEAD, "already enough lookahead"); do { more = (unsigned)(s->window_size -(ulg)s->lookahead -(ulg)s->strstart); /* Deal with !@#$% 64K limit: */ if (sizeof(int) <= 2) { if (more == 0 && s->strstart == 0 && s->lookahead == 0) { more = wsize; } else if (more == (unsigned)(-1)) { /* Very unlikely, but possible on 16 bit machine if * strstart == 0 && lookahead == 1 (input done a byte at time) */ more--; } } /* If the window is almost full and there is insufficient lookahead, * move the upper half to the lower one to make room in the upper half. */ if (s->strstart >= wsize+MAX_DIST(s)) { zmemcpy(s->window, s->window+wsize, (unsigned)wsize); s->match_start -= wsize; s->strstart -= wsize; /* we now have strstart >= MAX_DIST */ s->block_start -= (long) wsize; /* Slide the hash table (could be avoided with 32 bit values at the expense of memory usage). We slide even when level == 0 to keep the hash table consistent if we switch back to level > 0 later. (Using level 0 permanently is not an optimal usage of zlib, so we don't care about this pathological case.) */ n = s->hash_size; p = &s->head[n]; do { m = *--p; *p = (Pos)(m >= wsize ? m-wsize : NIL); } while (--n); n = wsize; #ifndef FASTEST p = &s->prev[n]; do { m = *--p; *p = (Pos)(m >= wsize ? m-wsize : NIL); /* If n is not on any hash chain, prev[n] is garbage but * its value will never be used. */ } while (--n); #endif more += wsize; } if (s->strm->avail_in == 0) break; /* If there was no sliding: * strstart <= WSIZE+MAX_DIST-1 && lookahead <= MIN_LOOKAHEAD - 1 && * more == window_size - lookahead - strstart * => more >= window_size - (MIN_LOOKAHEAD-1 + WSIZE + MAX_DIST-1) * => more >= window_size - 2*WSIZE + 2 * In the BIG_MEM or MMAP case (not yet supported), * window_size == input_size + MIN_LOOKAHEAD && * strstart + s->lookahead <= input_size => more >= MIN_LOOKAHEAD. * Otherwise, window_size == 2*WSIZE so more >= 2. * If there was sliding, more >= WSIZE. So in all cases, more >= 2. */ Assert(more >= 2, "more < 2"); n = read_buf(s->strm, s->window + s->strstart + s->lookahead, more); s->lookahead += n; /* Initialize the hash value now that we have some input: */ if (s->lookahead + s->insert >= MIN_MATCH) { uInt str = s->strstart - s->insert; s->ins_h = s->window[str]; UPDATE_HASH(s, s->ins_h, s->window[str + 1]); #if MIN_MATCH != 3 Call UPDATE_HASH() MIN_MATCH-3 more times #endif while (s->insert) { UPDATE_HASH(s, s->ins_h, s->window[str + MIN_MATCH-1]); #ifndef FASTEST s->prev[str & s->w_mask] = s->head[s->ins_h]; #endif s->head[s->ins_h] = (Pos)str; str++; s->insert--; if (s->lookahead + s->insert < MIN_MATCH) break; } } /* If the whole input has less than MIN_MATCH bytes, ins_h is garbage, * but this is not important since only literal bytes will be emitted. */ } while (s->lookahead < MIN_LOOKAHEAD && s->strm->avail_in != 0); /* If the WIN_INIT bytes after the end of the current data have never been * written, then zero those bytes in order to avoid memory check reports of * the use of uninitialized (or uninitialised as Julian writes) bytes by * the longest match routines. Update the high water mark for the next * time through here. WIN_INIT is set to MAX_MATCH since the longest match * routines allow scanning to strstart + MAX_MATCH, ignoring lookahead. */ if (s->high_water < s->window_size) { ulg curr = s->strstart + (ulg)(s->lookahead); ulg init; if (s->high_water < curr) { /* Previous high water mark below current data -- zero WIN_INIT * bytes or up to end of window, whichever is less. */ init = s->window_size - curr; if (init > WIN_INIT) init = WIN_INIT; zmemzero(s->window + curr, (unsigned)init); s->high_water = curr + init; } else if (s->high_water < (ulg)curr + WIN_INIT) { /* High water mark at or above current data, but below current data * plus WIN_INIT -- zero out to current data plus WIN_INIT, or up * to end of window, whichever is less. */ init = (ulg)curr + WIN_INIT - s->high_water; if (init > s->window_size - s->high_water) init = s->window_size - s->high_water; zmemzero(s->window + s->high_water, (unsigned)init); s->high_water += init; } } Assert((ulg)s->strstart <= s->window_size - MIN_LOOKAHEAD, "not enough room for search"); } /* =========================================================================== * Flush the current block, with given end-of-file flag. * IN assertion: strstart is set to the end of the current match. */ #define FLUSH_BLOCK_ONLY(s, last) { \ _tr_flush_block(s, (s->block_start >= 0L ? \ (charf *)&s->window[(unsigned)s->block_start] : \ (charf *)Z_NULL), \ (ulg)((long)s->strstart - s->block_start), \ (last)); \ s->block_start = s->strstart; \ flush_pending(s->strm); \ Tracev((stderr,"[FLUSH]")); \ } /* Same but force premature exit if necessary. */ #define FLUSH_BLOCK(s, last) { \ FLUSH_BLOCK_ONLY(s, last); \ if (s->strm->avail_out == 0) return (last) ? finish_started : need_more; \ } /* =========================================================================== * Copy without compression as much as possible from the input stream, return * the current block state. * This function does not insert new strings in the dictionary since * uncompressible data is probably not useful. This function is used * only for the level=0 compression option. * NOTE: this function should be optimized to avoid extra copying from * window to pending_buf. */ local block_state deflate_stored(s, flush) deflate_state *s; int flush; { /* Stored blocks are limited to 0xffff bytes, pending_buf is limited * to pending_buf_size, and each stored block has a 5 byte header: */ ulg max_block_size = 0xffff; ulg max_start; if (max_block_size > s->pending_buf_size - 5) { max_block_size = s->pending_buf_size - 5; } /* Copy as much as possible from input to output: */ for (;;) { /* Fill the window as much as possible: */ if (s->lookahead <= 1) { Assert(s->strstart < s->w_size+MAX_DIST(s) || s->block_start >= (long)s->w_size, "slide too late"); fill_window(s); if (s->lookahead == 0 && flush == Z_NO_FLUSH) return need_more; if (s->lookahead == 0) break; /* flush the current block */ } Assert(s->block_start >= 0L, "block gone"); s->strstart += s->lookahead; s->lookahead = 0; /* Emit a stored block if pending_buf will be full: */ max_start = s->block_start + max_block_size; if (s->strstart == 0 || (ulg)s->strstart >= max_start) { /* strstart == 0 is possible when wraparound on 16-bit machine */ s->lookahead = (uInt)(s->strstart - max_start); s->strstart = (uInt)max_start; FLUSH_BLOCK(s, 0); } /* Flush if we may have to slide, otherwise block_start may become * negative and the data will be gone: */ if (s->strstart - (uInt)s->block_start >= MAX_DIST(s)) { FLUSH_BLOCK(s, 0); } } s->insert = 0; if (flush == Z_FINISH) { FLUSH_BLOCK(s, 1); return finish_done; } if ((long)s->strstart > s->block_start) FLUSH_BLOCK(s, 0); return block_done; } /* =========================================================================== * Compress as much as possible from the input stream, return the current * block state. * This function does not perform lazy evaluation of matches and inserts * new strings in the dictionary only for unmatched strings or for short * matches. It is used only for the fast compression options. */ local block_state deflate_fast(s, flush) deflate_state *s; int flush; { IPos hash_head; /* head of the hash chain */ int bflush; /* set if current block must be flushed */ for (;;) { /* Make sure that we always have enough lookahead, except * at the end of the input file. We need MAX_MATCH bytes * for the next match, plus MIN_MATCH bytes to insert the * string following the next match. */ if (s->lookahead < MIN_LOOKAHEAD) { fill_window(s); if (s->lookahead < MIN_LOOKAHEAD && flush == Z_NO_FLUSH) { return need_more; } if (s->lookahead == 0) break; /* flush the current block */ } /* Insert the string window[strstart .. strstart+2] in the * dictionary, and set hash_head to the head of the hash chain: */ hash_head = NIL; if (s->lookahead >= MIN_MATCH) { INSERT_STRING(s, s->strstart, hash_head); } /* Find the longest match, discarding those <= prev_length. * At this point we have always match_length < MIN_MATCH */ if (hash_head != NIL && s->strstart - hash_head <= MAX_DIST(s)) { /* To simplify the code, we prevent matches with the string * of window index 0 (in particular we have to avoid a match * of the string with itself at the start of the input file). */ s->match_length = longest_match (s, hash_head); /* longest_match() sets match_start */ } if (s->match_length >= MIN_MATCH) { check_match(s, s->strstart, s->match_start, s->match_length); _tr_tally_dist(s, s->strstart - s->match_start, s->match_length - MIN_MATCH, bflush); s->lookahead -= s->match_length; /* Insert new strings in the hash table only if the match length * is not too large. This saves time but degrades compression. */ #ifndef FASTEST if (s->match_length <= s->max_insert_length && s->lookahead >= MIN_MATCH) { s->match_length--; /* string at strstart already in table */ do { s->strstart++; INSERT_STRING(s, s->strstart, hash_head); /* strstart never exceeds WSIZE-MAX_MATCH, so there are * always MIN_MATCH bytes ahead. */ } while (--s->match_length != 0); s->strstart++; } else #endif { s->strstart += s->match_length; s->match_length = 0; s->ins_h = s->window[s->strstart]; UPDATE_HASH(s, s->ins_h, s->window[s->strstart+1]); #if MIN_MATCH != 3 Call UPDATE_HASH() MIN_MATCH-3 more times #endif /* If lookahead < MIN_MATCH, ins_h is garbage, but it does not * matter since it will be recomputed at next deflate call. */ } } else { /* No match, output a literal byte */ Tracevv((stderr,"%c", s->window[s->strstart])); _tr_tally_lit (s, s->window[s->strstart], bflush); s->lookahead--; s->strstart++; } if (bflush) FLUSH_BLOCK(s, 0); } s->insert = s->strstart < MIN_MATCH-1 ? s->strstart : MIN_MATCH-1; if (flush == Z_FINISH) { FLUSH_BLOCK(s, 1); return finish_done; } if (s->last_lit) FLUSH_BLOCK(s, 0); return block_done; } #ifndef FASTEST /* =========================================================================== * Same as above, but achieves better compression. We use a lazy * evaluation for matches: a match is finally adopted only if there is * no better match at the next window position. */ local block_state deflate_slow(s, flush) deflate_state *s; int flush; { IPos hash_head; /* head of hash chain */ int bflush; /* set if current block must be flushed */ /* Process the input block. */ for (;;) { /* Make sure that we always have enough lookahead, except * at the end of the input file. We need MAX_MATCH bytes * for the next match, plus MIN_MATCH bytes to insert the * string following the next match. */ if (s->lookahead < MIN_LOOKAHEAD) { fill_window(s); if (s->lookahead < MIN_LOOKAHEAD && flush == Z_NO_FLUSH) { return need_more; } if (s->lookahead == 0) break; /* flush the current block */ } /* Insert the string window[strstart .. strstart+2] in the * dictionary, and set hash_head to the head of the hash chain: */ hash_head = NIL; if (s->lookahead >= MIN_MATCH) { INSERT_STRING(s, s->strstart, hash_head); } /* Find the longest match, discarding those <= prev_length. */ s->prev_length = s->match_length, s->prev_match = s->match_start; s->match_length = MIN_MATCH-1; if (hash_head != NIL && s->prev_length < s->max_lazy_match && s->strstart - hash_head <= MAX_DIST(s)) { /* To simplify the code, we prevent matches with the string * of window index 0 (in particular we have to avoid a match * of the string with itself at the start of the input file). */ s->match_length = longest_match (s, hash_head); /* longest_match() sets match_start */ if (s->match_length <= 5 && (s->strategy == Z_FILTERED #if TOO_FAR <= 32767 || (s->match_length == MIN_MATCH && s->strstart - s->match_start > TOO_FAR) #endif )) { /* If prev_match is also MIN_MATCH, match_start is garbage * but we will ignore the current match anyway. */ s->match_length = MIN_MATCH-1; } } /* If there was a match at the previous step and the current * match is not better, output the previous match: */ if (s->prev_length >= MIN_MATCH && s->match_length <= s->prev_length) { uInt max_insert = s->strstart + s->lookahead - MIN_MATCH; /* Do not insert strings in hash table beyond this. */ check_match(s, s->strstart-1, s->prev_match, s->prev_length); _tr_tally_dist(s, s->strstart -1 - s->prev_match, s->prev_length - MIN_MATCH, bflush); /* Insert in hash table all strings up to the end of the match. * strstart-1 and strstart are already inserted. If there is not * enough lookahead, the last two strings are not inserted in * the hash table. */ s->lookahead -= s->prev_length-1; s->prev_length -= 2; do { if (++s->strstart <= max_insert) { INSERT_STRING(s, s->strstart, hash_head); } } while (--s->prev_length != 0); s->match_available = 0; s->match_length = MIN_MATCH-1; s->strstart++; if (bflush) FLUSH_BLOCK(s, 0); } else if (s->match_available) { /* If there was no match at the previous position, output a * single literal. If there was a match but the current match * is longer, truncate the previous match to a single literal. */ Tracevv((stderr,"%c", s->window[s->strstart-1])); _tr_tally_lit(s, s->window[s->strstart-1], bflush); if (bflush) { FLUSH_BLOCK_ONLY(s, 0); } s->strstart++; s->lookahead--; if (s->strm->avail_out == 0) return need_more; } else { /* There is no previous match to compare with, wait for * the next step to decide. */ s->match_available = 1; s->strstart++; s->lookahead--; } } Assert (flush != Z_NO_FLUSH, "no flush?"); if (s->match_available) { Tracevv((stderr,"%c", s->window[s->strstart-1])); _tr_tally_lit(s, s->window[s->strstart-1], bflush); s->match_available = 0; } s->insert = s->strstart < MIN_MATCH-1 ? s->strstart : MIN_MATCH-1; if (flush == Z_FINISH) { FLUSH_BLOCK(s, 1); return finish_done; } if (s->last_lit) FLUSH_BLOCK(s, 0); return block_done; } #endif /* FASTEST */ /* =========================================================================== * For Z_RLE, simply look for runs of bytes, generate matches only of distance * one. Do not maintain a hash table. (It will be regenerated if this run of * deflate switches away from Z_RLE.) */ local block_state deflate_rle(s, flush) deflate_state *s; int flush; { int bflush; /* set if current block must be flushed */ uInt prev; /* byte at distance one to match */ Bytef *scan, *strend; /* scan goes up to strend for length of run */ for (;;) { /* Make sure that we always have enough lookahead, except * at the end of the input file. We need MAX_MATCH bytes * for the longest run, plus one for the unrolled loop. */ if (s->lookahead <= MAX_MATCH) { fill_window(s); if (s->lookahead <= MAX_MATCH && flush == Z_NO_FLUSH) { return need_more; } if (s->lookahead == 0) break; /* flush the current block */ } /* See how many times the previous byte repeats */ s->match_length = 0; if (s->lookahead >= MIN_MATCH && s->strstart > 0) { scan = s->window + s->strstart - 1; prev = *scan; if (prev == *++scan && prev == *++scan && prev == *++scan) { strend = s->window + s->strstart + MAX_MATCH; do { } while (prev == *++scan && prev == *++scan && prev == *++scan && prev == *++scan && prev == *++scan && prev == *++scan && prev == *++scan && prev == *++scan && scan < strend); s->match_length = MAX_MATCH - (int)(strend - scan); if (s->match_length > s->lookahead) s->match_length = s->lookahead; } Assert(scan <= s->window+(uInt)(s->window_size-1), "wild scan"); } /* Emit match if have run of MIN_MATCH or longer, else emit literal */ if (s->match_length >= MIN_MATCH) { check_match(s, s->strstart, s->strstart - 1, s->match_length); _tr_tally_dist(s, 1, s->match_length - MIN_MATCH, bflush); s->lookahead -= s->match_length; s->strstart += s->match_length; s->match_length = 0; } else { /* No match, output a literal byte */ Tracevv((stderr,"%c", s->window[s->strstart])); _tr_tally_lit (s, s->window[s->strstart], bflush); s->lookahead--; s->strstart++; } if (bflush) FLUSH_BLOCK(s, 0); } s->insert = 0; if (flush == Z_FINISH) { FLUSH_BLOCK(s, 1); return finish_done; } if (s->last_lit) FLUSH_BLOCK(s, 0); return block_done; } /* =========================================================================== * For Z_HUFFMAN_ONLY, do not look for matches. Do not maintain a hash table. * (It will be regenerated if this run of deflate switches away from Huffman.) */ local block_state deflate_huff(s, flush) deflate_state *s; int flush; { int bflush; /* set if current block must be flushed */ for (;;) { /* Make sure that we have a literal to write. */ if (s->lookahead == 0) { fill_window(s); if (s->lookahead == 0) { if (flush == Z_NO_FLUSH) return need_more; break; /* flush the current block */ } } /* Output a literal byte */ s->match_length = 0; Tracevv((stderr,"%c", s->window[s->strstart])); _tr_tally_lit (s, s->window[s->strstart], bflush); s->lookahead--; s->strstart++; if (bflush) FLUSH_BLOCK(s, 0); } s->insert = 0; if (flush == Z_FINISH) { FLUSH_BLOCK(s, 1); return finish_done; } if (s->last_lit) FLUSH_BLOCK(s, 0); return block_done; } c-blosc-1.7.0/internal-complibs/zlib-1.2.8/deflate.h000066400000000000000000000307461254622165100216620ustar00rootroot00000000000000/* deflate.h -- internal compression state * Copyright (C) 1995-2012 Jean-loup Gailly * For conditions of distribution and use, see copyright notice in zlib.h */ /* WARNING: this file should *not* be used by applications. It is part of the implementation of the compression library and is subject to change. Applications should only use zlib.h. */ /* @(#) $Id$ */ #ifndef DEFLATE_H #define DEFLATE_H #include "zutil.h" /* define NO_GZIP when compiling if you want to disable gzip header and trailer creation by deflate(). NO_GZIP would be used to avoid linking in the crc code when it is not needed. For shared libraries, gzip encoding should be left enabled. */ #ifndef NO_GZIP # define GZIP #endif /* =========================================================================== * Internal compression state. */ #define LENGTH_CODES 29 /* number of length codes, not counting the special END_BLOCK code */ #define LITERALS 256 /* number of literal bytes 0..255 */ #define L_CODES (LITERALS+1+LENGTH_CODES) /* number of Literal or Length codes, including the END_BLOCK code */ #define D_CODES 30 /* number of distance codes */ #define BL_CODES 19 /* number of codes used to transfer the bit lengths */ #define HEAP_SIZE (2*L_CODES+1) /* maximum heap size */ #define MAX_BITS 15 /* All codes must not exceed MAX_BITS bits */ #define Buf_size 16 /* size of bit buffer in bi_buf */ #define INIT_STATE 42 #define EXTRA_STATE 69 #define NAME_STATE 73 #define COMMENT_STATE 91 #define HCRC_STATE 103 #define BUSY_STATE 113 #define FINISH_STATE 666 /* Stream status */ /* Data structure describing a single value and its code string. */ typedef struct ct_data_s { union { ush freq; /* frequency count */ ush code; /* bit string */ } fc; union { ush dad; /* father node in Huffman tree */ ush len; /* length of bit string */ } dl; } FAR ct_data; #define Freq fc.freq #define Code fc.code #define Dad dl.dad #define Len dl.len typedef struct static_tree_desc_s static_tree_desc; typedef struct tree_desc_s { ct_data *dyn_tree; /* the dynamic tree */ int max_code; /* largest code with non zero frequency */ static_tree_desc *stat_desc; /* the corresponding static tree */ } FAR tree_desc; typedef ush Pos; typedef Pos FAR Posf; typedef unsigned IPos; /* A Pos is an index in the character window. We use short instead of int to * save space in the various tables. IPos is used only for parameter passing. */ typedef struct internal_state { z_streamp strm; /* pointer back to this zlib stream */ int status; /* as the name implies */ Bytef *pending_buf; /* output still pending */ ulg pending_buf_size; /* size of pending_buf */ Bytef *pending_out; /* next pending byte to output to the stream */ uInt pending; /* nb of bytes in the pending buffer */ int wrap; /* bit 0 true for zlib, bit 1 true for gzip */ gz_headerp gzhead; /* gzip header information to write */ uInt gzindex; /* where in extra, name, or comment */ Byte method; /* can only be DEFLATED */ int last_flush; /* value of flush param for previous deflate call */ /* used by deflate.c: */ uInt w_size; /* LZ77 window size (32K by default) */ uInt w_bits; /* log2(w_size) (8..16) */ uInt w_mask; /* w_size - 1 */ Bytef *window; /* Sliding window. Input bytes are read into the second half of the window, * and move to the first half later to keep a dictionary of at least wSize * bytes. With this organization, matches are limited to a distance of * wSize-MAX_MATCH bytes, but this ensures that IO is always * performed with a length multiple of the block size. Also, it limits * the window size to 64K, which is quite useful on MSDOS. * To do: use the user input buffer as sliding window. */ ulg window_size; /* Actual size of window: 2*wSize, except when the user input buffer * is directly used as sliding window. */ Posf *prev; /* Link to older string with same hash index. To limit the size of this * array to 64K, this link is maintained only for the last 32K strings. * An index in this array is thus a window index modulo 32K. */ Posf *head; /* Heads of the hash chains or NIL. */ uInt ins_h; /* hash index of string to be inserted */ uInt hash_size; /* number of elements in hash table */ uInt hash_bits; /* log2(hash_size) */ uInt hash_mask; /* hash_size-1 */ uInt hash_shift; /* Number of bits by which ins_h must be shifted at each input * step. It must be such that after MIN_MATCH steps, the oldest * byte no longer takes part in the hash key, that is: * hash_shift * MIN_MATCH >= hash_bits */ long block_start; /* Window position at the beginning of the current output block. Gets * negative when the window is moved backwards. */ uInt match_length; /* length of best match */ IPos prev_match; /* previous match */ int match_available; /* set if previous match exists */ uInt strstart; /* start of string to insert */ uInt match_start; /* start of matching string */ uInt lookahead; /* number of valid bytes ahead in window */ uInt prev_length; /* Length of the best match at previous step. Matches not greater than this * are discarded. This is used in the lazy match evaluation. */ uInt max_chain_length; /* To speed up deflation, hash chains are never searched beyond this * length. A higher limit improves compression ratio but degrades the * speed. */ uInt max_lazy_match; /* Attempt to find a better match only when the current match is strictly * smaller than this value. This mechanism is used only for compression * levels >= 4. */ # define max_insert_length max_lazy_match /* Insert new strings in the hash table only if the match length is not * greater than this length. This saves time but degrades compression. * max_insert_length is used only for compression levels <= 3. */ int level; /* compression level (1..9) */ int strategy; /* favor or force Huffman coding*/ uInt good_match; /* Use a faster search when the previous match is longer than this */ int nice_match; /* Stop searching when current match exceeds this */ /* used by trees.c: */ /* Didn't use ct_data typedef below to suppress compiler warning */ struct ct_data_s dyn_ltree[HEAP_SIZE]; /* literal and length tree */ struct ct_data_s dyn_dtree[2*D_CODES+1]; /* distance tree */ struct ct_data_s bl_tree[2*BL_CODES+1]; /* Huffman tree for bit lengths */ struct tree_desc_s l_desc; /* desc. for literal tree */ struct tree_desc_s d_desc; /* desc. for distance tree */ struct tree_desc_s bl_desc; /* desc. for bit length tree */ ush bl_count[MAX_BITS+1]; /* number of codes at each bit length for an optimal tree */ int heap[2*L_CODES+1]; /* heap used to build the Huffman trees */ int heap_len; /* number of elements in the heap */ int heap_max; /* element of largest frequency */ /* The sons of heap[n] are heap[2*n] and heap[2*n+1]. heap[0] is not used. * The same heap array is used to build all trees. */ uch depth[2*L_CODES+1]; /* Depth of each subtree used as tie breaker for trees of equal frequency */ uchf *l_buf; /* buffer for literals or lengths */ uInt lit_bufsize; /* Size of match buffer for literals/lengths. There are 4 reasons for * limiting lit_bufsize to 64K: * - frequencies can be kept in 16 bit counters * - if compression is not successful for the first block, all input * data is still in the window so we can still emit a stored block even * when input comes from standard input. (This can also be done for * all blocks if lit_bufsize is not greater than 32K.) * - if compression is not successful for a file smaller than 64K, we can * even emit a stored file instead of a stored block (saving 5 bytes). * This is applicable only for zip (not gzip or zlib). * - creating new Huffman trees less frequently may not provide fast * adaptation to changes in the input data statistics. (Take for * example a binary file with poorly compressible code followed by * a highly compressible string table.) Smaller buffer sizes give * fast adaptation but have of course the overhead of transmitting * trees more frequently. * - I can't count above 4 */ uInt last_lit; /* running index in l_buf */ ushf *d_buf; /* Buffer for distances. To simplify the code, d_buf and l_buf have * the same number of elements. To use different lengths, an extra flag * array would be necessary. */ ulg opt_len; /* bit length of current block with optimal trees */ ulg static_len; /* bit length of current block with static trees */ uInt matches; /* number of string matches in current block */ uInt insert; /* bytes at end of window left to insert */ #ifdef DEBUG ulg compressed_len; /* total bit length of compressed file mod 2^32 */ ulg bits_sent; /* bit length of compressed data sent mod 2^32 */ #endif ush bi_buf; /* Output buffer. bits are inserted starting at the bottom (least * significant bits). */ int bi_valid; /* Number of valid bits in bi_buf. All bits above the last valid bit * are always zero. */ ulg high_water; /* High water mark offset in window for initialized bytes -- bytes above * this are set to zero in order to avoid memory check warnings when * longest match routines access bytes past the input. This is then * updated to the new high water mark. */ } FAR deflate_state; /* Output a byte on the stream. * IN assertion: there is enough room in pending_buf. */ #define put_byte(s, c) {s->pending_buf[s->pending++] = (c);} #define MIN_LOOKAHEAD (MAX_MATCH+MIN_MATCH+1) /* Minimum amount of lookahead, except at the end of the input file. * See deflate.c for comments about the MIN_MATCH+1. */ #define MAX_DIST(s) ((s)->w_size-MIN_LOOKAHEAD) /* In order to simplify the code, particularly on 16 bit machines, match * distances are limited to MAX_DIST instead of WSIZE. */ #define WIN_INIT MAX_MATCH /* Number of bytes after end of data in window to initialize in order to avoid memory checker errors from longest match routines */ /* in trees.c */ void ZLIB_INTERNAL _tr_init OF((deflate_state *s)); int ZLIB_INTERNAL _tr_tally OF((deflate_state *s, unsigned dist, unsigned lc)); void ZLIB_INTERNAL _tr_flush_block OF((deflate_state *s, charf *buf, ulg stored_len, int last)); void ZLIB_INTERNAL _tr_flush_bits OF((deflate_state *s)); void ZLIB_INTERNAL _tr_align OF((deflate_state *s)); void ZLIB_INTERNAL _tr_stored_block OF((deflate_state *s, charf *buf, ulg stored_len, int last)); #define d_code(dist) \ ((dist) < 256 ? _dist_code[dist] : _dist_code[256+((dist)>>7)]) /* Mapping from a distance to a distance code. dist is the distance - 1 and * must not have side effects. _dist_code[256] and _dist_code[257] are never * used. */ #ifndef DEBUG /* Inline versions of _tr_tally for speed: */ #if defined(GEN_TREES_H) || !defined(STDC) extern uch ZLIB_INTERNAL _length_code[]; extern uch ZLIB_INTERNAL _dist_code[]; #else extern const uch ZLIB_INTERNAL _length_code[]; extern const uch ZLIB_INTERNAL _dist_code[]; #endif # define _tr_tally_lit(s, c, flush) \ { uch cc = (c); \ s->d_buf[s->last_lit] = 0; \ s->l_buf[s->last_lit++] = cc; \ s->dyn_ltree[cc].Freq++; \ flush = (s->last_lit == s->lit_bufsize-1); \ } # define _tr_tally_dist(s, distance, length, flush) \ { uch len = (length); \ ush dist = (distance); \ s->d_buf[s->last_lit] = dist; \ s->l_buf[s->last_lit++] = len; \ dist--; \ s->dyn_ltree[_length_code[len]+LITERALS+1].Freq++; \ s->dyn_dtree[d_code(dist)].Freq++; \ flush = (s->last_lit == s->lit_bufsize-1); \ } #else # define _tr_tally_lit(s, c, flush) flush = _tr_tally(s, 0, c) # define _tr_tally_dist(s, distance, length, flush) \ flush = _tr_tally(s, distance, length) #endif #endif /* DEFLATE_H */ c-blosc-1.7.0/internal-complibs/zlib-1.2.8/gzclose.c000066400000000000000000000012461254622165100217100ustar00rootroot00000000000000/* gzclose.c -- zlib gzclose() function * Copyright (C) 2004, 2010 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ #include "gzguts.h" /* gzclose() is in a separate file so that it is linked in only if it is used. That way the other gzclose functions can be used instead to avoid linking in unneeded compression or decompression routines. */ int ZEXPORT gzclose(file) gzFile file; { #ifndef NO_GZCOMPRESS gz_statep state; if (file == NULL) return Z_STREAM_ERROR; state = (gz_statep)file; return state->mode == GZ_READ ? gzclose_r(file) : gzclose_w(file); #else return gzclose_r(file); #endif } c-blosc-1.7.0/internal-complibs/zlib-1.2.8/gzguts.h000066400000000000000000000146301254622165100215730ustar00rootroot00000000000000/* gzguts.h -- zlib internal header definitions for gz* operations * Copyright (C) 2004, 2005, 2010, 2011, 2012, 2013 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ #ifdef _LARGEFILE64_SOURCE # ifndef _LARGEFILE_SOURCE # define _LARGEFILE_SOURCE 1 # endif # ifdef _FILE_OFFSET_BITS # undef _FILE_OFFSET_BITS # endif #endif #ifdef HAVE_HIDDEN # define ZLIB_INTERNAL __attribute__((visibility ("hidden"))) #else # define ZLIB_INTERNAL #endif #include #include "zlib.h" #ifdef STDC # include # include # include #endif #include #ifdef _WIN32 # include #endif #if defined(__TURBOC__) || defined(_MSC_VER) || defined(_WIN32) # include #endif #ifdef WINAPI_FAMILY # define open _open # define read _read # define write _write # define close _close #endif #ifdef NO_DEFLATE /* for compatibility with old definition */ # define NO_GZCOMPRESS #endif #if defined(STDC99) || (defined(__TURBOC__) && __TURBOC__ >= 0x550) # ifndef HAVE_VSNPRINTF # define HAVE_VSNPRINTF # endif #endif #if defined(__CYGWIN__) # ifndef HAVE_VSNPRINTF # define HAVE_VSNPRINTF # endif #endif #if defined(MSDOS) && defined(__BORLANDC__) && (BORLANDC > 0x410) # ifndef HAVE_VSNPRINTF # define HAVE_VSNPRINTF # endif #endif #ifndef HAVE_VSNPRINTF # ifdef MSDOS /* vsnprintf may exist on some MS-DOS compilers (DJGPP?), but for now we just assume it doesn't. */ # define NO_vsnprintf # endif # ifdef __TURBOC__ # define NO_vsnprintf # endif # ifdef WIN32 /* In Win32, vsnprintf is available as the "non-ANSI" _vsnprintf. */ # if !defined(vsnprintf) && !defined(NO_vsnprintf) # if !defined(_MSC_VER) || ( defined(_MSC_VER) && _MSC_VER < 1500 ) # define vsnprintf _vsnprintf # endif # endif # endif # ifdef __SASC # define NO_vsnprintf # endif # ifdef VMS # define NO_vsnprintf # endif # ifdef __OS400__ # define NO_vsnprintf # endif # ifdef __MVS__ # define NO_vsnprintf # endif #endif /* unlike snprintf (which is required in C99, yet still not supported by Microsoft more than a decade later!), _snprintf does not guarantee null termination of the result -- however this is only used in gzlib.c where the result is assured to fit in the space provided */ #ifdef _MSC_VER # define snprintf _snprintf #endif #ifndef local # define local static #endif /* compile with -Dlocal if your debugger can't find static symbols */ /* gz* functions always use library allocation functions */ #ifndef STDC extern voidp malloc OF((uInt size)); extern void free OF((voidpf ptr)); #endif /* get errno and strerror definition */ #if defined UNDER_CE # include # define zstrerror() gz_strwinerror((DWORD)GetLastError()) #else # ifndef NO_STRERROR # include # define zstrerror() strerror(errno) # else # define zstrerror() "stdio error (consult errno)" # endif #endif /* provide prototypes for these when building zlib without LFS */ #if !defined(_LARGEFILE64_SOURCE) || _LFS64_LARGEFILE-0 == 0 ZEXTERN gzFile ZEXPORT gzopen64 OF((const char *, const char *)); ZEXTERN z_off64_t ZEXPORT gzseek64 OF((gzFile, z_off64_t, int)); ZEXTERN z_off64_t ZEXPORT gztell64 OF((gzFile)); ZEXTERN z_off64_t ZEXPORT gzoffset64 OF((gzFile)); #endif /* default memLevel */ #if MAX_MEM_LEVEL >= 8 # define DEF_MEM_LEVEL 8 #else # define DEF_MEM_LEVEL MAX_MEM_LEVEL #endif /* default i/o buffer size -- double this for output when reading (this and twice this must be able to fit in an unsigned type) */ #define GZBUFSIZE 8192 /* gzip modes, also provide a little integrity check on the passed structure */ #define GZ_NONE 0 #define GZ_READ 7247 #define GZ_WRITE 31153 #define GZ_APPEND 1 /* mode set to GZ_WRITE after the file is opened */ /* values for gz_state how */ #define LOOK 0 /* look for a gzip header */ #define COPY 1 /* copy input directly */ #define GZIP 2 /* decompress a gzip stream */ /* internal gzip file state data structure */ typedef struct { /* exposed contents for gzgetc() macro */ struct gzFile_s x; /* "x" for exposed */ /* x.have: number of bytes available at x.next */ /* x.next: next output data to deliver or write */ /* x.pos: current position in uncompressed data */ /* used for both reading and writing */ int mode; /* see gzip modes above */ int fd; /* file descriptor */ char *path; /* path or fd for error messages */ unsigned size; /* buffer size, zero if not allocated yet */ unsigned want; /* requested buffer size, default is GZBUFSIZE */ unsigned char *in; /* input buffer */ unsigned char *out; /* output buffer (double-sized when reading) */ int direct; /* 0 if processing gzip, 1 if transparent */ /* just for reading */ int how; /* 0: get header, 1: copy, 2: decompress */ z_off64_t start; /* where the gzip data started, for rewinding */ int eof; /* true if end of input file reached */ int past; /* true if read requested past end */ /* just for writing */ int level; /* compression level */ int strategy; /* compression strategy */ /* seek request */ z_off64_t skip; /* amount to skip (already rewound if backwards) */ int seek; /* true if seek request pending */ /* error information */ int err; /* error code */ char *msg; /* error message */ /* zlib inflate or deflate stream */ z_stream strm; /* stream structure in-place (not a pointer) */ } gz_state; typedef gz_state FAR *gz_statep; /* shared functions */ void ZLIB_INTERNAL gz_error OF((gz_statep, int, const char *)); #if defined UNDER_CE char ZLIB_INTERNAL *gz_strwinerror OF((DWORD error)); #endif /* GT_OFF(x), where x is an unsigned value, is true if x > maximum z_off64_t value -- needed when comparing unsigned to z_off64_t, which is signed (possible z_off64_t types off_t, off64_t, and long are all signed) */ #ifdef INT_MAX # define GT_OFF(x) (sizeof(int) == sizeof(z_off64_t) && (x) > INT_MAX) #else unsigned ZLIB_INTERNAL gz_intmax OF((void)); # define GT_OFF(x) (sizeof(int) == sizeof(z_off64_t) && (x) > gz_intmax()) #endif c-blosc-1.7.0/internal-complibs/zlib-1.2.8/gzlib.c000066400000000000000000000400371254622165100213520ustar00rootroot00000000000000/* gzlib.c -- zlib functions common to reading and writing gzip files * Copyright (C) 2004, 2010, 2011, 2012, 2013 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ #include "gzguts.h" #if defined(_WIN32) && !defined(__BORLANDC__) # define LSEEK _lseeki64 #else #if defined(_LARGEFILE64_SOURCE) && _LFS64_LARGEFILE-0 # define LSEEK lseek64 #else # define LSEEK lseek #endif #endif /* Local functions */ local void gz_reset OF((gz_statep)); local gzFile gz_open OF((const void *, int, const char *)); #if defined UNDER_CE /* Map the Windows error number in ERROR to a locale-dependent error message string and return a pointer to it. Typically, the values for ERROR come from GetLastError. The string pointed to shall not be modified by the application, but may be overwritten by a subsequent call to gz_strwinerror The gz_strwinerror function does not change the current setting of GetLastError. */ char ZLIB_INTERNAL *gz_strwinerror (error) DWORD error; { static char buf[1024]; wchar_t *msgbuf; DWORD lasterr = GetLastError(); DWORD chars = FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_ALLOCATE_BUFFER, NULL, error, 0, /* Default language */ (LPVOID)&msgbuf, 0, NULL); if (chars != 0) { /* If there is an \r\n appended, zap it. */ if (chars >= 2 && msgbuf[chars - 2] == '\r' && msgbuf[chars - 1] == '\n') { chars -= 2; msgbuf[chars] = 0; } if (chars > sizeof (buf) - 1) { chars = sizeof (buf) - 1; msgbuf[chars] = 0; } wcstombs(buf, msgbuf, chars + 1); LocalFree(msgbuf); } else { sprintf(buf, "unknown win32 error (%ld)", error); } SetLastError(lasterr); return buf; } #endif /* UNDER_CE */ /* Reset gzip file state */ local void gz_reset(state) gz_statep state; { state->x.have = 0; /* no output data available */ if (state->mode == GZ_READ) { /* for reading ... */ state->eof = 0; /* not at end of file */ state->past = 0; /* have not read past end yet */ state->how = LOOK; /* look for gzip header */ } state->seek = 0; /* no seek request pending */ gz_error(state, Z_OK, NULL); /* clear error */ state->x.pos = 0; /* no uncompressed data yet */ state->strm.avail_in = 0; /* no input data yet */ } /* Open a gzip file either by name or file descriptor. */ local gzFile gz_open(path, fd, mode) const void *path; int fd; const char *mode; { gz_statep state; size_t len; int oflag; #ifdef O_CLOEXEC int cloexec = 0; #endif #ifdef O_EXCL int exclusive = 0; #endif /* check input */ if (path == NULL) return NULL; /* allocate gzFile structure to return */ state = (gz_statep)malloc(sizeof(gz_state)); if (state == NULL) return NULL; state->size = 0; /* no buffers allocated yet */ state->want = GZBUFSIZE; /* requested buffer size */ state->msg = NULL; /* no error message yet */ /* interpret mode */ state->mode = GZ_NONE; state->level = Z_DEFAULT_COMPRESSION; state->strategy = Z_DEFAULT_STRATEGY; state->direct = 0; while (*mode) { if (*mode >= '0' && *mode <= '9') state->level = *mode - '0'; else switch (*mode) { case 'r': state->mode = GZ_READ; break; #ifndef NO_GZCOMPRESS case 'w': state->mode = GZ_WRITE; break; case 'a': state->mode = GZ_APPEND; break; #endif case '+': /* can't read and write at the same time */ free(state); return NULL; case 'b': /* ignore -- will request binary anyway */ break; #ifdef O_CLOEXEC case 'e': cloexec = 1; break; #endif #ifdef O_EXCL case 'x': exclusive = 1; break; #endif case 'f': state->strategy = Z_FILTERED; break; case 'h': state->strategy = Z_HUFFMAN_ONLY; break; case 'R': state->strategy = Z_RLE; break; case 'F': state->strategy = Z_FIXED; break; case 'T': state->direct = 1; break; default: /* could consider as an error, but just ignore */ ; } mode++; } /* must provide an "r", "w", or "a" */ if (state->mode == GZ_NONE) { free(state); return NULL; } /* can't force transparent read */ if (state->mode == GZ_READ) { if (state->direct) { free(state); return NULL; } state->direct = 1; /* for empty file */ } /* save the path name for error messages */ #ifdef _WIN32 if (fd == -2) { len = wcstombs(NULL, path, 0); if (len == (size_t)-1) len = 0; } else #endif len = strlen((const char *)path); state->path = (char *)malloc(len + 1); if (state->path == NULL) { free(state); return NULL; } #ifdef _WIN32 if (fd == -2) if (len) wcstombs(state->path, path, len + 1); else *(state->path) = 0; else #endif #if !defined(NO_snprintf) && !defined(NO_vsnprintf) snprintf(state->path, len + 1, "%s", (const char *)path); #else strcpy(state->path, path); #endif /* compute the flags for open() */ oflag = #ifdef O_LARGEFILE O_LARGEFILE | #endif #ifdef O_BINARY O_BINARY | #endif #ifdef O_CLOEXEC (cloexec ? O_CLOEXEC : 0) | #endif (state->mode == GZ_READ ? O_RDONLY : (O_WRONLY | O_CREAT | #ifdef O_EXCL (exclusive ? O_EXCL : 0) | #endif (state->mode == GZ_WRITE ? O_TRUNC : O_APPEND))); /* open the file with the appropriate flags (or just use fd) */ state->fd = fd > -1 ? fd : ( #ifdef _WIN32 fd == -2 ? _wopen(path, oflag, 0666) : #endif open((const char *)path, oflag, 0666)); if (state->fd == -1) { free(state->path); free(state); return NULL; } if (state->mode == GZ_APPEND) state->mode = GZ_WRITE; /* simplify later checks */ /* save the current position for rewinding (only if reading) */ if (state->mode == GZ_READ) { state->start = LSEEK(state->fd, 0, SEEK_CUR); if (state->start == -1) state->start = 0; } /* initialize stream */ gz_reset(state); /* return stream */ return (gzFile)state; } /* -- see zlib.h -- */ gzFile ZEXPORT gzopen(path, mode) const char *path; const char *mode; { return gz_open(path, -1, mode); } /* -- see zlib.h -- */ gzFile ZEXPORT gzopen64(path, mode) const char *path; const char *mode; { return gz_open(path, -1, mode); } /* -- see zlib.h -- */ gzFile ZEXPORT gzdopen(fd, mode) int fd; const char *mode; { char *path; /* identifier for error messages */ gzFile gz; if (fd == -1 || (path = (char *)malloc(7 + 3 * sizeof(int))) == NULL) return NULL; #if !defined(NO_snprintf) && !defined(NO_vsnprintf) snprintf(path, 7 + 3 * sizeof(int), "", fd); /* for debugging */ #else sprintf(path, "", fd); /* for debugging */ #endif gz = gz_open(path, fd, mode); free(path); return gz; } /* -- see zlib.h -- */ #ifdef _WIN32 gzFile ZEXPORT gzopen_w(path, mode) const wchar_t *path; const char *mode; { return gz_open(path, -2, mode); } #endif /* -- see zlib.h -- */ int ZEXPORT gzbuffer(file, size) gzFile file; unsigned size; { gz_statep state; /* get internal structure and check integrity */ if (file == NULL) return -1; state = (gz_statep)file; if (state->mode != GZ_READ && state->mode != GZ_WRITE) return -1; /* make sure we haven't already allocated memory */ if (state->size != 0) return -1; /* check and set requested size */ if (size < 2) size = 2; /* need two bytes to check magic header */ state->want = size; return 0; } /* -- see zlib.h -- */ int ZEXPORT gzrewind(file) gzFile file; { gz_statep state; /* get internal structure */ if (file == NULL) return -1; state = (gz_statep)file; /* check that we're reading and that there's no error */ if (state->mode != GZ_READ || (state->err != Z_OK && state->err != Z_BUF_ERROR)) return -1; /* back up and start over */ if (LSEEK(state->fd, state->start, SEEK_SET) == -1) return -1; gz_reset(state); return 0; } /* -- see zlib.h -- */ z_off64_t ZEXPORT gzseek64(file, offset, whence) gzFile file; z_off64_t offset; int whence; { unsigned n; z_off64_t ret; gz_statep state; /* get internal structure and check integrity */ if (file == NULL) return -1; state = (gz_statep)file; if (state->mode != GZ_READ && state->mode != GZ_WRITE) return -1; /* check that there's no error */ if (state->err != Z_OK && state->err != Z_BUF_ERROR) return -1; /* can only seek from start or relative to current position */ if (whence != SEEK_SET && whence != SEEK_CUR) return -1; /* normalize offset to a SEEK_CUR specification */ if (whence == SEEK_SET) offset -= state->x.pos; else if (state->seek) offset += state->skip; state->seek = 0; /* if within raw area while reading, just go there */ if (state->mode == GZ_READ && state->how == COPY && state->x.pos + offset >= 0) { ret = LSEEK(state->fd, offset - state->x.have, SEEK_CUR); if (ret == -1) return -1; state->x.have = 0; state->eof = 0; state->past = 0; state->seek = 0; gz_error(state, Z_OK, NULL); state->strm.avail_in = 0; state->x.pos += offset; return state->x.pos; } /* calculate skip amount, rewinding if needed for back seek when reading */ if (offset < 0) { if (state->mode != GZ_READ) /* writing -- can't go backwards */ return -1; offset += state->x.pos; if (offset < 0) /* before start of file! */ return -1; if (gzrewind(file) == -1) /* rewind, then skip to offset */ return -1; } /* if reading, skip what's in output buffer (one less gzgetc() check) */ if (state->mode == GZ_READ) { n = GT_OFF(state->x.have) || (z_off64_t)state->x.have > offset ? (unsigned)offset : state->x.have; state->x.have -= n; state->x.next += n; state->x.pos += n; offset -= n; } /* request skip (if not zero) */ if (offset) { state->seek = 1; state->skip = offset; } return state->x.pos + offset; } /* -- see zlib.h -- */ z_off_t ZEXPORT gzseek(file, offset, whence) gzFile file; z_off_t offset; int whence; { z_off64_t ret; ret = gzseek64(file, (z_off64_t)offset, whence); return ret == (z_off_t)ret ? (z_off_t)ret : -1; } /* -- see zlib.h -- */ z_off64_t ZEXPORT gztell64(file) gzFile file; { gz_statep state; /* get internal structure and check integrity */ if (file == NULL) return -1; state = (gz_statep)file; if (state->mode != GZ_READ && state->mode != GZ_WRITE) return -1; /* return position */ return state->x.pos + (state->seek ? state->skip : 0); } /* -- see zlib.h -- */ z_off_t ZEXPORT gztell(file) gzFile file; { z_off64_t ret; ret = gztell64(file); return ret == (z_off_t)ret ? (z_off_t)ret : -1; } /* -- see zlib.h -- */ z_off64_t ZEXPORT gzoffset64(file) gzFile file; { z_off64_t offset; gz_statep state; /* get internal structure and check integrity */ if (file == NULL) return -1; state = (gz_statep)file; if (state->mode != GZ_READ && state->mode != GZ_WRITE) return -1; /* compute and return effective offset in file */ offset = LSEEK(state->fd, 0, SEEK_CUR); if (offset == -1) return -1; if (state->mode == GZ_READ) /* reading */ offset -= state->strm.avail_in; /* don't count buffered input */ return offset; } /* -- see zlib.h -- */ z_off_t ZEXPORT gzoffset(file) gzFile file; { z_off64_t ret; ret = gzoffset64(file); return ret == (z_off_t)ret ? (z_off_t)ret : -1; } /* -- see zlib.h -- */ int ZEXPORT gzeof(file) gzFile file; { gz_statep state; /* get internal structure and check integrity */ if (file == NULL) return 0; state = (gz_statep)file; if (state->mode != GZ_READ && state->mode != GZ_WRITE) return 0; /* return end-of-file state */ return state->mode == GZ_READ ? state->past : 0; } /* -- see zlib.h -- */ const char * ZEXPORT gzerror(file, errnum) gzFile file; int *errnum; { gz_statep state; /* get internal structure and check integrity */ if (file == NULL) return NULL; state = (gz_statep)file; if (state->mode != GZ_READ && state->mode != GZ_WRITE) return NULL; /* return error information */ if (errnum != NULL) *errnum = state->err; return state->err == Z_MEM_ERROR ? "out of memory" : (state->msg == NULL ? "" : state->msg); } /* -- see zlib.h -- */ void ZEXPORT gzclearerr(file) gzFile file; { gz_statep state; /* get internal structure and check integrity */ if (file == NULL) return; state = (gz_statep)file; if (state->mode != GZ_READ && state->mode != GZ_WRITE) return; /* clear error and end-of-file */ if (state->mode == GZ_READ) { state->eof = 0; state->past = 0; } gz_error(state, Z_OK, NULL); } /* Create an error message in allocated memory and set state->err and state->msg accordingly. Free any previous error message already there. Do not try to free or allocate space if the error is Z_MEM_ERROR (out of memory). Simply save the error message as a static string. If there is an allocation failure constructing the error message, then convert the error to out of memory. */ void ZLIB_INTERNAL gz_error(state, err, msg) gz_statep state; int err; const char *msg; { /* free previously allocated message and clear */ if (state->msg != NULL) { if (state->err != Z_MEM_ERROR) free(state->msg); state->msg = NULL; } /* if fatal, set state->x.have to 0 so that the gzgetc() macro fails */ if (err != Z_OK && err != Z_BUF_ERROR) state->x.have = 0; /* set error code, and if no message, then done */ state->err = err; if (msg == NULL) return; /* for an out of memory error, return literal string when requested */ if (err == Z_MEM_ERROR) return; /* construct error message with path */ if ((state->msg = (char *)malloc(strlen(state->path) + strlen(msg) + 3)) == NULL) { state->err = Z_MEM_ERROR; return; } #if !defined(NO_snprintf) && !defined(NO_vsnprintf) snprintf(state->msg, strlen(state->path) + strlen(msg) + 3, "%s%s%s", state->path, ": ", msg); #else strcpy(state->msg, state->path); strcat(state->msg, ": "); strcat(state->msg, msg); #endif return; } #ifndef INT_MAX /* portably return maximum value for an int (when limits.h presumed not available) -- we need to do this to cover cases where 2's complement not used, since C standard permits 1's complement and sign-bit representations, otherwise we could just use ((unsigned)-1) >> 1 */ unsigned ZLIB_INTERNAL gz_intmax() { unsigned p, q; p = 1; do { q = p; p <<= 1; p++; } while (p > q); return q >> 1; } #endif c-blosc-1.7.0/internal-complibs/zlib-1.2.8/gzread.c000066400000000000000000000444061254622165100215230ustar00rootroot00000000000000/* gzread.c -- zlib functions for reading gzip files * Copyright (C) 2004, 2005, 2010, 2011, 2012, 2013 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ #include "gzguts.h" /* Local functions */ local int gz_load OF((gz_statep, unsigned char *, unsigned, unsigned *)); local int gz_avail OF((gz_statep)); local int gz_look OF((gz_statep)); local int gz_decomp OF((gz_statep)); local int gz_fetch OF((gz_statep)); local int gz_skip OF((gz_statep, z_off64_t)); /* Use read() to load a buffer -- return -1 on error, otherwise 0. Read from state->fd, and update state->eof, state->err, and state->msg as appropriate. This function needs to loop on read(), since read() is not guaranteed to read the number of bytes requested, depending on the type of descriptor. */ local int gz_load(state, buf, len, have) gz_statep state; unsigned char *buf; unsigned len; unsigned *have; { int ret; *have = 0; do { ret = read(state->fd, buf + *have, len - *have); if (ret <= 0) break; *have += ret; } while (*have < len); if (ret < 0) { gz_error(state, Z_ERRNO, zstrerror()); return -1; } if (ret == 0) state->eof = 1; return 0; } /* Load up input buffer and set eof flag if last data loaded -- return -1 on error, 0 otherwise. Note that the eof flag is set when the end of the input file is reached, even though there may be unused data in the buffer. Once that data has been used, no more attempts will be made to read the file. If strm->avail_in != 0, then the current data is moved to the beginning of the input buffer, and then the remainder of the buffer is loaded with the available data from the input file. */ local int gz_avail(state) gz_statep state; { unsigned got; z_streamp strm = &(state->strm); if (state->err != Z_OK && state->err != Z_BUF_ERROR) return -1; if (state->eof == 0) { if (strm->avail_in) { /* copy what's there to the start */ unsigned char *p = state->in; unsigned const char *q = strm->next_in; unsigned n = strm->avail_in; do { *p++ = *q++; } while (--n); } if (gz_load(state, state->in + strm->avail_in, state->size - strm->avail_in, &got) == -1) return -1; strm->avail_in += got; strm->next_in = state->in; } return 0; } /* Look for gzip header, set up for inflate or copy. state->x.have must be 0. If this is the first time in, allocate required memory. state->how will be left unchanged if there is no more input data available, will be set to COPY if there is no gzip header and direct copying will be performed, or it will be set to GZIP for decompression. If direct copying, then leftover input data from the input buffer will be copied to the output buffer. In that case, all further file reads will be directly to either the output buffer or a user buffer. If decompressing, the inflate state will be initialized. gz_look() will return 0 on success or -1 on failure. */ local int gz_look(state) gz_statep state; { z_streamp strm = &(state->strm); /* allocate read buffers and inflate memory */ if (state->size == 0) { /* allocate buffers */ state->in = (unsigned char *)malloc(state->want); state->out = (unsigned char *)malloc(state->want << 1); if (state->in == NULL || state->out == NULL) { if (state->out != NULL) free(state->out); if (state->in != NULL) free(state->in); gz_error(state, Z_MEM_ERROR, "out of memory"); return -1; } state->size = state->want; /* allocate inflate memory */ state->strm.zalloc = Z_NULL; state->strm.zfree = Z_NULL; state->strm.opaque = Z_NULL; state->strm.avail_in = 0; state->strm.next_in = Z_NULL; if (inflateInit2(&(state->strm), 15 + 16) != Z_OK) { /* gunzip */ free(state->out); free(state->in); state->size = 0; gz_error(state, Z_MEM_ERROR, "out of memory"); return -1; } } /* get at least the magic bytes in the input buffer */ if (strm->avail_in < 2) { if (gz_avail(state) == -1) return -1; if (strm->avail_in == 0) return 0; } /* look for gzip magic bytes -- if there, do gzip decoding (note: there is a logical dilemma here when considering the case of a partially written gzip file, to wit, if a single 31 byte is written, then we cannot tell whether this is a single-byte file, or just a partially written gzip file -- for here we assume that if a gzip file is being written, then the header will be written in a single operation, so that reading a single byte is sufficient indication that it is not a gzip file) */ if (strm->avail_in > 1 && strm->next_in[0] == 31 && strm->next_in[1] == 139) { inflateReset(strm); state->how = GZIP; state->direct = 0; return 0; } /* no gzip header -- if we were decoding gzip before, then this is trailing garbage. Ignore the trailing garbage and finish. */ if (state->direct == 0) { strm->avail_in = 0; state->eof = 1; state->x.have = 0; return 0; } /* doing raw i/o, copy any leftover input to output -- this assumes that the output buffer is larger than the input buffer, which also assures space for gzungetc() */ state->x.next = state->out; if (strm->avail_in) { memcpy(state->x.next, strm->next_in, strm->avail_in); state->x.have = strm->avail_in; strm->avail_in = 0; } state->how = COPY; state->direct = 1; return 0; } /* Decompress from input to the provided next_out and avail_out in the state. On return, state->x.have and state->x.next point to the just decompressed data. If the gzip stream completes, state->how is reset to LOOK to look for the next gzip stream or raw data, once state->x.have is depleted. Returns 0 on success, -1 on failure. */ local int gz_decomp(state) gz_statep state; { int ret = Z_OK; unsigned had; z_streamp strm = &(state->strm); /* fill output buffer up to end of deflate stream */ had = strm->avail_out; do { /* get more input for inflate() */ if (strm->avail_in == 0 && gz_avail(state) == -1) return -1; if (strm->avail_in == 0) { gz_error(state, Z_BUF_ERROR, "unexpected end of file"); break; } /* decompress and handle errors */ ret = inflate(strm, Z_NO_FLUSH); if (ret == Z_STREAM_ERROR || ret == Z_NEED_DICT) { gz_error(state, Z_STREAM_ERROR, "internal error: inflate stream corrupt"); return -1; } if (ret == Z_MEM_ERROR) { gz_error(state, Z_MEM_ERROR, "out of memory"); return -1; } if (ret == Z_DATA_ERROR) { /* deflate stream invalid */ gz_error(state, Z_DATA_ERROR, strm->msg == NULL ? "compressed data error" : strm->msg); return -1; } } while (strm->avail_out && ret != Z_STREAM_END); /* update available output */ state->x.have = had - strm->avail_out; state->x.next = strm->next_out - state->x.have; /* if the gzip stream completed successfully, look for another */ if (ret == Z_STREAM_END) state->how = LOOK; /* good decompression */ return 0; } /* Fetch data and put it in the output buffer. Assumes state->x.have is 0. Data is either copied from the input file or decompressed from the input file depending on state->how. If state->how is LOOK, then a gzip header is looked for to determine whether to copy or decompress. Returns -1 on error, otherwise 0. gz_fetch() will leave state->how as COPY or GZIP unless the end of the input file has been reached and all data has been processed. */ local int gz_fetch(state) gz_statep state; { z_streamp strm = &(state->strm); do { switch(state->how) { case LOOK: /* -> LOOK, COPY (only if never GZIP), or GZIP */ if (gz_look(state) == -1) return -1; if (state->how == LOOK) return 0; break; case COPY: /* -> COPY */ if (gz_load(state, state->out, state->size << 1, &(state->x.have)) == -1) return -1; state->x.next = state->out; return 0; case GZIP: /* -> GZIP or LOOK (if end of gzip stream) */ strm->avail_out = state->size << 1; strm->next_out = state->out; if (gz_decomp(state) == -1) return -1; } } while (state->x.have == 0 && (!state->eof || strm->avail_in)); return 0; } /* Skip len uncompressed bytes of output. Return -1 on error, 0 on success. */ local int gz_skip(state, len) gz_statep state; z_off64_t len; { unsigned n; /* skip over len bytes or reach end-of-file, whichever comes first */ while (len) /* skip over whatever is in output buffer */ if (state->x.have) { n = GT_OFF(state->x.have) || (z_off64_t)state->x.have > len ? (unsigned)len : state->x.have; state->x.have -= n; state->x.next += n; state->x.pos += n; len -= n; } /* output buffer empty -- return if we're at the end of the input */ else if (state->eof && state->strm.avail_in == 0) break; /* need more data to skip -- load up output buffer */ else { /* get more output, looking for header if required */ if (gz_fetch(state) == -1) return -1; } return 0; } /* -- see zlib.h -- */ int ZEXPORT gzread(file, buf, len) gzFile file; voidp buf; unsigned len; { unsigned got, n; gz_statep state; z_streamp strm; /* get internal structure */ if (file == NULL) return -1; state = (gz_statep)file; strm = &(state->strm); /* check that we're reading and that there's no (serious) error */ if (state->mode != GZ_READ || (state->err != Z_OK && state->err != Z_BUF_ERROR)) return -1; /* since an int is returned, make sure len fits in one, otherwise return with an error (this avoids the flaw in the interface) */ if ((int)len < 0) { gz_error(state, Z_DATA_ERROR, "requested length does not fit in int"); return -1; } /* if len is zero, avoid unnecessary operations */ if (len == 0) return 0; /* process a skip request */ if (state->seek) { state->seek = 0; if (gz_skip(state, state->skip) == -1) return -1; } /* get len bytes to buf, or less than len if at the end */ got = 0; do { /* first just try copying data from the output buffer */ if (state->x.have) { n = state->x.have > len ? len : state->x.have; memcpy(buf, state->x.next, n); state->x.next += n; state->x.have -= n; } /* output buffer empty -- return if we're at the end of the input */ else if (state->eof && strm->avail_in == 0) { state->past = 1; /* tried to read past end */ break; } /* need output data -- for small len or new stream load up our output buffer */ else if (state->how == LOOK || len < (state->size << 1)) { /* get more output, looking for header if required */ if (gz_fetch(state) == -1) return -1; continue; /* no progress yet -- go back to copy above */ /* the copy above assures that we will leave with space in the output buffer, allowing at least one gzungetc() to succeed */ } /* large len -- read directly into user buffer */ else if (state->how == COPY) { /* read directly */ if (gz_load(state, (unsigned char *)buf, len, &n) == -1) return -1; } /* large len -- decompress directly into user buffer */ else { /* state->how == GZIP */ strm->avail_out = len; strm->next_out = (unsigned char *)buf; if (gz_decomp(state) == -1) return -1; n = state->x.have; state->x.have = 0; } /* update progress */ len -= n; buf = (char *)buf + n; got += n; state->x.pos += n; } while (len); /* return number of bytes read into user buffer (will fit in int) */ return (int)got; } /* -- see zlib.h -- */ #ifdef Z_PREFIX_SET # undef z_gzgetc #else # undef gzgetc #endif int ZEXPORT gzgetc(file) gzFile file; { int ret; unsigned char buf[1]; gz_statep state; /* get internal structure */ if (file == NULL) return -1; state = (gz_statep)file; /* check that we're reading and that there's no (serious) error */ if (state->mode != GZ_READ || (state->err != Z_OK && state->err != Z_BUF_ERROR)) return -1; /* try output buffer (no need to check for skip request) */ if (state->x.have) { state->x.have--; state->x.pos++; return *(state->x.next)++; } /* nothing there -- try gzread() */ ret = gzread(file, buf, 1); return ret < 1 ? -1 : buf[0]; } int ZEXPORT gzgetc_(file) gzFile file; { return gzgetc(file); } /* -- see zlib.h -- */ int ZEXPORT gzungetc(c, file) int c; gzFile file; { gz_statep state; /* get internal structure */ if (file == NULL) return -1; state = (gz_statep)file; /* check that we're reading and that there's no (serious) error */ if (state->mode != GZ_READ || (state->err != Z_OK && state->err != Z_BUF_ERROR)) return -1; /* process a skip request */ if (state->seek) { state->seek = 0; if (gz_skip(state, state->skip) == -1) return -1; } /* can't push EOF */ if (c < 0) return -1; /* if output buffer empty, put byte at end (allows more pushing) */ if (state->x.have == 0) { state->x.have = 1; state->x.next = state->out + (state->size << 1) - 1; state->x.next[0] = c; state->x.pos--; state->past = 0; return c; } /* if no room, give up (must have already done a gzungetc()) */ if (state->x.have == (state->size << 1)) { gz_error(state, Z_DATA_ERROR, "out of room to push characters"); return -1; } /* slide output data if needed and insert byte before existing data */ if (state->x.next == state->out) { unsigned char *src = state->out + state->x.have; unsigned char *dest = state->out + (state->size << 1); while (src > state->out) *--dest = *--src; state->x.next = dest; } state->x.have++; state->x.next--; state->x.next[0] = c; state->x.pos--; state->past = 0; return c; } /* -- see zlib.h -- */ char * ZEXPORT gzgets(file, buf, len) gzFile file; char *buf; int len; { unsigned left, n; char *str; unsigned char *eol; gz_statep state; /* check parameters and get internal structure */ if (file == NULL || buf == NULL || len < 1) return NULL; state = (gz_statep)file; /* check that we're reading and that there's no (serious) error */ if (state->mode != GZ_READ || (state->err != Z_OK && state->err != Z_BUF_ERROR)) return NULL; /* process a skip request */ if (state->seek) { state->seek = 0; if (gz_skip(state, state->skip) == -1) return NULL; } /* copy output bytes up to new line or len - 1, whichever comes first -- append a terminating zero to the string (we don't check for a zero in the contents, let the user worry about that) */ str = buf; left = (unsigned)len - 1; if (left) do { /* assure that something is in the output buffer */ if (state->x.have == 0 && gz_fetch(state) == -1) return NULL; /* error */ if (state->x.have == 0) { /* end of file */ state->past = 1; /* read past end */ break; /* return what we have */ } /* look for end-of-line in current output buffer */ n = state->x.have > left ? left : state->x.have; eol = (unsigned char *)memchr(state->x.next, '\n', n); if (eol != NULL) n = (unsigned)(eol - state->x.next) + 1; /* copy through end-of-line, or remainder if not found */ memcpy(buf, state->x.next, n); state->x.have -= n; state->x.next += n; state->x.pos += n; left -= n; buf += n; } while (left && eol == NULL); /* return terminated string, or if nothing, end of file */ if (buf == str) return NULL; buf[0] = 0; return str; } /* -- see zlib.h -- */ int ZEXPORT gzdirect(file) gzFile file; { gz_statep state; /* get internal structure */ if (file == NULL) return 0; state = (gz_statep)file; /* if the state is not known, but we can find out, then do so (this is mainly for right after a gzopen() or gzdopen()) */ if (state->mode == GZ_READ && state->how == LOOK && state->x.have == 0) (void)gz_look(state); /* return 1 if transparent, 0 if processing a gzip stream */ return state->direct; } /* -- see zlib.h -- */ int ZEXPORT gzclose_r(file) gzFile file; { int ret, err; gz_statep state; /* get internal structure */ if (file == NULL) return Z_STREAM_ERROR; state = (gz_statep)file; /* check that we're reading */ if (state->mode != GZ_READ) return Z_STREAM_ERROR; /* free memory and close file */ if (state->size) { inflateEnd(&(state->strm)); free(state->out); free(state->in); } err = state->err == Z_BUF_ERROR ? Z_BUF_ERROR : Z_OK; gz_error(state, Z_OK, NULL); free(state->path); ret = close(state->fd); free(state); return ret ? Z_ERRNO : err; } c-blosc-1.7.0/internal-complibs/zlib-1.2.8/gzwrite.c000066400000000000000000000375071254622165100217460ustar00rootroot00000000000000/* gzwrite.c -- zlib functions for writing gzip files * Copyright (C) 2004, 2005, 2010, 2011, 2012, 2013 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ #include "gzguts.h" /* Local functions */ local int gz_init OF((gz_statep)); local int gz_comp OF((gz_statep, int)); local int gz_zero OF((gz_statep, z_off64_t)); /* Initialize state for writing a gzip file. Mark initialization by setting state->size to non-zero. Return -1 on failure or 0 on success. */ local int gz_init(state) gz_statep state; { int ret; z_streamp strm = &(state->strm); /* allocate input buffer */ state->in = (unsigned char *)malloc(state->want); if (state->in == NULL) { gz_error(state, Z_MEM_ERROR, "out of memory"); return -1; } /* only need output buffer and deflate state if compressing */ if (!state->direct) { /* allocate output buffer */ state->out = (unsigned char *)malloc(state->want); if (state->out == NULL) { free(state->in); gz_error(state, Z_MEM_ERROR, "out of memory"); return -1; } /* allocate deflate memory, set up for gzip compression */ strm->zalloc = Z_NULL; strm->zfree = Z_NULL; strm->opaque = Z_NULL; ret = deflateInit2(strm, state->level, Z_DEFLATED, MAX_WBITS + 16, DEF_MEM_LEVEL, state->strategy); if (ret != Z_OK) { free(state->out); free(state->in); gz_error(state, Z_MEM_ERROR, "out of memory"); return -1; } } /* mark state as initialized */ state->size = state->want; /* initialize write buffer if compressing */ if (!state->direct) { strm->avail_out = state->size; strm->next_out = state->out; state->x.next = strm->next_out; } return 0; } /* Compress whatever is at avail_in and next_in and write to the output file. Return -1 if there is an error writing to the output file, otherwise 0. flush is assumed to be a valid deflate() flush value. If flush is Z_FINISH, then the deflate() state is reset to start a new gzip stream. If gz->direct is true, then simply write to the output file without compressing, and ignore flush. */ local int gz_comp(state, flush) gz_statep state; int flush; { int ret, got; unsigned have; z_streamp strm = &(state->strm); /* allocate memory if this is the first time through */ if (state->size == 0 && gz_init(state) == -1) return -1; /* write directly if requested */ if (state->direct) { got = write(state->fd, strm->next_in, strm->avail_in); if (got < 0 || (unsigned)got != strm->avail_in) { gz_error(state, Z_ERRNO, zstrerror()); return -1; } strm->avail_in = 0; return 0; } /* run deflate() on provided input until it produces no more output */ ret = Z_OK; do { /* write out current buffer contents if full, or if flushing, but if doing Z_FINISH then don't write until we get to Z_STREAM_END */ if (strm->avail_out == 0 || (flush != Z_NO_FLUSH && (flush != Z_FINISH || ret == Z_STREAM_END))) { have = (unsigned)(strm->next_out - state->x.next); if (have && ((got = write(state->fd, state->x.next, have)) < 0 || (unsigned)got != have)) { gz_error(state, Z_ERRNO, zstrerror()); return -1; } if (strm->avail_out == 0) { strm->avail_out = state->size; strm->next_out = state->out; } state->x.next = strm->next_out; } /* compress */ have = strm->avail_out; ret = deflate(strm, flush); if (ret == Z_STREAM_ERROR) { gz_error(state, Z_STREAM_ERROR, "internal error: deflate stream corrupt"); return -1; } have -= strm->avail_out; } while (have); /* if that completed a deflate stream, allow another to start */ if (flush == Z_FINISH) deflateReset(strm); /* all done, no errors */ return 0; } /* Compress len zeros to output. Return -1 on error, 0 on success. */ local int gz_zero(state, len) gz_statep state; z_off64_t len; { int first; unsigned n; z_streamp strm = &(state->strm); /* consume whatever's left in the input buffer */ if (strm->avail_in && gz_comp(state, Z_NO_FLUSH) == -1) return -1; /* compress len zeros (len guaranteed > 0) */ first = 1; while (len) { n = GT_OFF(state->size) || (z_off64_t)state->size > len ? (unsigned)len : state->size; if (first) { memset(state->in, 0, n); first = 0; } strm->avail_in = n; strm->next_in = state->in; state->x.pos += n; if (gz_comp(state, Z_NO_FLUSH) == -1) return -1; len -= n; } return 0; } /* -- see zlib.h -- */ int ZEXPORT gzwrite(file, buf, len) gzFile file; voidpc buf; unsigned len; { unsigned put = len; gz_statep state; z_streamp strm; /* get internal structure */ if (file == NULL) return 0; state = (gz_statep)file; strm = &(state->strm); /* check that we're writing and that there's no error */ if (state->mode != GZ_WRITE || state->err != Z_OK) return 0; /* since an int is returned, make sure len fits in one, otherwise return with an error (this avoids the flaw in the interface) */ if ((int)len < 0) { gz_error(state, Z_DATA_ERROR, "requested length does not fit in int"); return 0; } /* if len is zero, avoid unnecessary operations */ if (len == 0) return 0; /* allocate memory if this is the first time through */ if (state->size == 0 && gz_init(state) == -1) return 0; /* check for seek request */ if (state->seek) { state->seek = 0; if (gz_zero(state, state->skip) == -1) return 0; } /* for small len, copy to input buffer, otherwise compress directly */ if (len < state->size) { /* copy to input buffer, compress when full */ do { unsigned have, copy; if (strm->avail_in == 0) strm->next_in = state->in; have = (unsigned)((strm->next_in + strm->avail_in) - state->in); copy = state->size - have; if (copy > len) copy = len; memcpy(state->in + have, buf, copy); strm->avail_in += copy; state->x.pos += copy; buf = (const char *)buf + copy; len -= copy; if (len && gz_comp(state, Z_NO_FLUSH) == -1) return 0; } while (len); } else { /* consume whatever's left in the input buffer */ if (strm->avail_in && gz_comp(state, Z_NO_FLUSH) == -1) return 0; /* directly compress user buffer to file */ strm->avail_in = len; strm->next_in = (z_const Bytef *)buf; state->x.pos += len; if (gz_comp(state, Z_NO_FLUSH) == -1) return 0; } /* input was all buffered or compressed (put will fit in int) */ return (int)put; } /* -- see zlib.h -- */ int ZEXPORT gzputc(file, c) gzFile file; int c; { unsigned have; unsigned char buf[1]; gz_statep state; z_streamp strm; /* get internal structure */ if (file == NULL) return -1; state = (gz_statep)file; strm = &(state->strm); /* check that we're writing and that there's no error */ if (state->mode != GZ_WRITE || state->err != Z_OK) return -1; /* check for seek request */ if (state->seek) { state->seek = 0; if (gz_zero(state, state->skip) == -1) return -1; } /* try writing to input buffer for speed (state->size == 0 if buffer not initialized) */ if (state->size) { if (strm->avail_in == 0) strm->next_in = state->in; have = (unsigned)((strm->next_in + strm->avail_in) - state->in); if (have < state->size) { state->in[have] = c; strm->avail_in++; state->x.pos++; return c & 0xff; } } /* no room in buffer or not initialized, use gz_write() */ buf[0] = c; if (gzwrite(file, buf, 1) != 1) return -1; return c & 0xff; } /* -- see zlib.h -- */ int ZEXPORT gzputs(file, str) gzFile file; const char *str; { int ret; unsigned len; /* write string */ len = (unsigned)strlen(str); ret = gzwrite(file, str, len); return ret == 0 && len != 0 ? -1 : ret; } #if defined(STDC) || defined(Z_HAVE_STDARG_H) #include /* -- see zlib.h -- */ int ZEXPORTVA gzvprintf(gzFile file, const char *format, va_list va) { int size, len; gz_statep state; z_streamp strm; /* get internal structure */ if (file == NULL) return -1; state = (gz_statep)file; strm = &(state->strm); /* check that we're writing and that there's no error */ if (state->mode != GZ_WRITE || state->err != Z_OK) return 0; /* make sure we have some buffer space */ if (state->size == 0 && gz_init(state) == -1) return 0; /* check for seek request */ if (state->seek) { state->seek = 0; if (gz_zero(state, state->skip) == -1) return 0; } /* consume whatever's left in the input buffer */ if (strm->avail_in && gz_comp(state, Z_NO_FLUSH) == -1) return 0; /* do the printf() into the input buffer, put length in len */ size = (int)(state->size); state->in[size - 1] = 0; #ifdef NO_vsnprintf # ifdef HAS_vsprintf_void (void)vsprintf((char *)(state->in), format, va); for (len = 0; len < size; len++) if (state->in[len] == 0) break; # else len = vsprintf((char *)(state->in), format, va); # endif #else # ifdef HAS_vsnprintf_void (void)vsnprintf((char *)(state->in), size, format, va); len = strlen((char *)(state->in)); # else len = vsnprintf((char *)(state->in), size, format, va); # endif #endif /* check that printf() results fit in buffer */ if (len <= 0 || len >= (int)size || state->in[size - 1] != 0) return 0; /* update buffer and position, defer compression until needed */ strm->avail_in = (unsigned)len; strm->next_in = state->in; state->x.pos += len; return len; } int ZEXPORTVA gzprintf(gzFile file, const char *format, ...) { va_list va; int ret; va_start(va, format); ret = gzvprintf(file, format, va); va_end(va); return ret; } #else /* !STDC && !Z_HAVE_STDARG_H */ /* -- see zlib.h -- */ int ZEXPORTVA gzprintf (file, format, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20) gzFile file; const char *format; int a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20; { int size, len; gz_statep state; z_streamp strm; /* get internal structure */ if (file == NULL) return -1; state = (gz_statep)file; strm = &(state->strm); /* check that can really pass pointer in ints */ if (sizeof(int) != sizeof(void *)) return 0; /* check that we're writing and that there's no error */ if (state->mode != GZ_WRITE || state->err != Z_OK) return 0; /* make sure we have some buffer space */ if (state->size == 0 && gz_init(state) == -1) return 0; /* check for seek request */ if (state->seek) { state->seek = 0; if (gz_zero(state, state->skip) == -1) return 0; } /* consume whatever's left in the input buffer */ if (strm->avail_in && gz_comp(state, Z_NO_FLUSH) == -1) return 0; /* do the printf() into the input buffer, put length in len */ size = (int)(state->size); state->in[size - 1] = 0; #ifdef NO_snprintf # ifdef HAS_sprintf_void sprintf((char *)(state->in), format, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20); for (len = 0; len < size; len++) if (state->in[len] == 0) break; # else len = sprintf((char *)(state->in), format, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20); # endif #else # ifdef HAS_snprintf_void snprintf((char *)(state->in), size, format, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20); len = strlen((char *)(state->in)); # else len = snprintf((char *)(state->in), size, format, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20); # endif #endif /* check that printf() results fit in buffer */ if (len <= 0 || len >= (int)size || state->in[size - 1] != 0) return 0; /* update buffer and position, defer compression until needed */ strm->avail_in = (unsigned)len; strm->next_in = state->in; state->x.pos += len; return len; } #endif /* -- see zlib.h -- */ int ZEXPORT gzflush(file, flush) gzFile file; int flush; { gz_statep state; /* get internal structure */ if (file == NULL) return -1; state = (gz_statep)file; /* check that we're writing and that there's no error */ if (state->mode != GZ_WRITE || state->err != Z_OK) return Z_STREAM_ERROR; /* check flush parameter */ if (flush < 0 || flush > Z_FINISH) return Z_STREAM_ERROR; /* check for seek request */ if (state->seek) { state->seek = 0; if (gz_zero(state, state->skip) == -1) return -1; } /* compress remaining data with requested flush */ gz_comp(state, flush); return state->err; } /* -- see zlib.h -- */ int ZEXPORT gzsetparams(file, level, strategy) gzFile file; int level; int strategy; { gz_statep state; z_streamp strm; /* get internal structure */ if (file == NULL) return Z_STREAM_ERROR; state = (gz_statep)file; strm = &(state->strm); /* check that we're writing and that there's no error */ if (state->mode != GZ_WRITE || state->err != Z_OK) return Z_STREAM_ERROR; /* if no change is requested, then do nothing */ if (level == state->level && strategy == state->strategy) return Z_OK; /* check for seek request */ if (state->seek) { state->seek = 0; if (gz_zero(state, state->skip) == -1) return -1; } /* change compression parameters for subsequent input */ if (state->size) { /* flush previous input with previous parameters before changing */ if (strm->avail_in && gz_comp(state, Z_PARTIAL_FLUSH) == -1) return state->err; deflateParams(strm, level, strategy); } state->level = level; state->strategy = strategy; return Z_OK; } /* -- see zlib.h -- */ int ZEXPORT gzclose_w(file) gzFile file; { int ret = Z_OK; gz_statep state; /* get internal structure */ if (file == NULL) return Z_STREAM_ERROR; state = (gz_statep)file; /* check that we're writing */ if (state->mode != GZ_WRITE) return Z_STREAM_ERROR; /* check for seek request */ if (state->seek) { state->seek = 0; if (gz_zero(state, state->skip) == -1) ret = state->err; } /* flush, free memory, and close file */ if (gz_comp(state, Z_FINISH) == -1) ret = state->err; if (state->size) { if (!state->direct) { (void)deflateEnd(&(state->strm)); free(state->out); } free(state->in); } gz_error(state, Z_OK, NULL); free(state->path); if (close(state->fd) == -1) ret = Z_ERRNO; free(state); return ret; } c-blosc-1.7.0/internal-complibs/zlib-1.2.8/infback.c000066400000000000000000000542651254622165100216500ustar00rootroot00000000000000/* infback.c -- inflate using a call-back interface * Copyright (C) 1995-2011 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ /* This code is largely copied from inflate.c. Normally either infback.o or inflate.o would be linked into an application--not both. The interface with inffast.c is retained so that optimized assembler-coded versions of inflate_fast() can be used with either inflate.c or infback.c. */ #include "zutil.h" #include "inftrees.h" #include "inflate.h" #include "inffast.h" /* function prototypes */ local void fixedtables OF((struct inflate_state FAR *state)); /* strm provides memory allocation functions in zalloc and zfree, or Z_NULL to use the library memory allocation functions. windowBits is in the range 8..15, and window is a user-supplied window and output buffer that is 2**windowBits bytes. */ int ZEXPORT inflateBackInit_(strm, windowBits, window, version, stream_size) z_streamp strm; int windowBits; unsigned char FAR *window; const char *version; int stream_size; { struct inflate_state FAR *state; if (version == Z_NULL || version[0] != ZLIB_VERSION[0] || stream_size != (int)(sizeof(z_stream))) return Z_VERSION_ERROR; if (strm == Z_NULL || window == Z_NULL || windowBits < 8 || windowBits > 15) return Z_STREAM_ERROR; strm->msg = Z_NULL; /* in case we return an error */ if (strm->zalloc == (alloc_func)0) { #ifdef Z_SOLO return Z_STREAM_ERROR; #else strm->zalloc = zcalloc; strm->opaque = (voidpf)0; #endif } if (strm->zfree == (free_func)0) #ifdef Z_SOLO return Z_STREAM_ERROR; #else strm->zfree = zcfree; #endif state = (struct inflate_state FAR *)ZALLOC(strm, 1, sizeof(struct inflate_state)); if (state == Z_NULL) return Z_MEM_ERROR; Tracev((stderr, "inflate: allocated\n")); strm->state = (struct internal_state FAR *)state; state->dmax = 32768U; state->wbits = windowBits; state->wsize = 1U << windowBits; state->window = window; state->wnext = 0; state->whave = 0; return Z_OK; } /* Return state with length and distance decoding tables and index sizes set to fixed code decoding. Normally this returns fixed tables from inffixed.h. If BUILDFIXED is defined, then instead this routine builds the tables the first time it's called, and returns those tables the first time and thereafter. This reduces the size of the code by about 2K bytes, in exchange for a little execution time. However, BUILDFIXED should not be used for threaded applications, since the rewriting of the tables and virgin may not be thread-safe. */ local void fixedtables(state) struct inflate_state FAR *state; { #ifdef BUILDFIXED static int virgin = 1; static code *lenfix, *distfix; static code fixed[544]; /* build fixed huffman tables if first call (may not be thread safe) */ if (virgin) { unsigned sym, bits; static code *next; /* literal/length table */ sym = 0; while (sym < 144) state->lens[sym++] = 8; while (sym < 256) state->lens[sym++] = 9; while (sym < 280) state->lens[sym++] = 7; while (sym < 288) state->lens[sym++] = 8; next = fixed; lenfix = next; bits = 9; inflate_table(LENS, state->lens, 288, &(next), &(bits), state->work); /* distance table */ sym = 0; while (sym < 32) state->lens[sym++] = 5; distfix = next; bits = 5; inflate_table(DISTS, state->lens, 32, &(next), &(bits), state->work); /* do this just once */ virgin = 0; } #else /* !BUILDFIXED */ # include "inffixed.h" #endif /* BUILDFIXED */ state->lencode = lenfix; state->lenbits = 9; state->distcode = distfix; state->distbits = 5; } /* Macros for inflateBack(): */ /* Load returned state from inflate_fast() */ #define LOAD() \ do { \ put = strm->next_out; \ left = strm->avail_out; \ next = strm->next_in; \ have = strm->avail_in; \ hold = state->hold; \ bits = state->bits; \ } while (0) /* Set state from registers for inflate_fast() */ #define RESTORE() \ do { \ strm->next_out = put; \ strm->avail_out = left; \ strm->next_in = next; \ strm->avail_in = have; \ state->hold = hold; \ state->bits = bits; \ } while (0) /* Clear the input bit accumulator */ #define INITBITS() \ do { \ hold = 0; \ bits = 0; \ } while (0) /* Assure that some input is available. If input is requested, but denied, then return a Z_BUF_ERROR from inflateBack(). */ #define PULL() \ do { \ if (have == 0) { \ have = in(in_desc, &next); \ if (have == 0) { \ next = Z_NULL; \ ret = Z_BUF_ERROR; \ goto inf_leave; \ } \ } \ } while (0) /* Get a byte of input into the bit accumulator, or return from inflateBack() with an error if there is no input available. */ #define PULLBYTE() \ do { \ PULL(); \ have--; \ hold += (unsigned long)(*next++) << bits; \ bits += 8; \ } while (0) /* Assure that there are at least n bits in the bit accumulator. If there is not enough available input to do that, then return from inflateBack() with an error. */ #define NEEDBITS(n) \ do { \ while (bits < (unsigned)(n)) \ PULLBYTE(); \ } while (0) /* Return the low n bits of the bit accumulator (n < 16) */ #define BITS(n) \ ((unsigned)hold & ((1U << (n)) - 1)) /* Remove n bits from the bit accumulator */ #define DROPBITS(n) \ do { \ hold >>= (n); \ bits -= (unsigned)(n); \ } while (0) /* Remove zero to seven bits as needed to go to a byte boundary */ #define BYTEBITS() \ do { \ hold >>= bits & 7; \ bits -= bits & 7; \ } while (0) /* Assure that some output space is available, by writing out the window if it's full. If the write fails, return from inflateBack() with a Z_BUF_ERROR. */ #define ROOM() \ do { \ if (left == 0) { \ put = state->window; \ left = state->wsize; \ state->whave = left; \ if (out(out_desc, put, left)) { \ ret = Z_BUF_ERROR; \ goto inf_leave; \ } \ } \ } while (0) /* strm provides the memory allocation functions and window buffer on input, and provides information on the unused input on return. For Z_DATA_ERROR returns, strm will also provide an error message. in() and out() are the call-back input and output functions. When inflateBack() needs more input, it calls in(). When inflateBack() has filled the window with output, or when it completes with data in the window, it calls out() to write out the data. The application must not change the provided input until in() is called again or inflateBack() returns. The application must not change the window/output buffer until inflateBack() returns. in() and out() are called with a descriptor parameter provided in the inflateBack() call. This parameter can be a structure that provides the information required to do the read or write, as well as accumulated information on the input and output such as totals and check values. in() should return zero on failure. out() should return non-zero on failure. If either in() or out() fails, than inflateBack() returns a Z_BUF_ERROR. strm->next_in can be checked for Z_NULL to see whether it was in() or out() that caused in the error. Otherwise, inflateBack() returns Z_STREAM_END on success, Z_DATA_ERROR for an deflate format error, or Z_MEM_ERROR if it could not allocate memory for the state. inflateBack() can also return Z_STREAM_ERROR if the input parameters are not correct, i.e. strm is Z_NULL or the state was not initialized. */ int ZEXPORT inflateBack(strm, in, in_desc, out, out_desc) z_streamp strm; in_func in; void FAR *in_desc; out_func out; void FAR *out_desc; { struct inflate_state FAR *state; z_const unsigned char FAR *next; /* next input */ unsigned char FAR *put; /* next output */ unsigned have, left; /* available input and output */ unsigned long hold; /* bit buffer */ unsigned bits; /* bits in bit buffer */ unsigned copy; /* number of stored or match bytes to copy */ unsigned char FAR *from; /* where to copy match bytes from */ code here; /* current decoding table entry */ code last; /* parent table entry */ unsigned len; /* length to copy for repeats, bits to drop */ int ret; /* return code */ static const unsigned short order[19] = /* permutation of code lengths */ {16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15}; /* Check that the strm exists and that the state was initialized */ if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; /* Reset the state */ strm->msg = Z_NULL; state->mode = TYPE; state->last = 0; state->whave = 0; next = strm->next_in; have = next != Z_NULL ? strm->avail_in : 0; hold = 0; bits = 0; put = state->window; left = state->wsize; /* Inflate until end of block marked as last */ for (;;) switch (state->mode) { case TYPE: /* determine and dispatch block type */ if (state->last) { BYTEBITS(); state->mode = DONE; break; } NEEDBITS(3); state->last = BITS(1); DROPBITS(1); switch (BITS(2)) { case 0: /* stored block */ Tracev((stderr, "inflate: stored block%s\n", state->last ? " (last)" : "")); state->mode = STORED; break; case 1: /* fixed block */ fixedtables(state); Tracev((stderr, "inflate: fixed codes block%s\n", state->last ? " (last)" : "")); state->mode = LEN; /* decode codes */ break; case 2: /* dynamic block */ Tracev((stderr, "inflate: dynamic codes block%s\n", state->last ? " (last)" : "")); state->mode = TABLE; break; case 3: strm->msg = (char *)"invalid block type"; state->mode = BAD; } DROPBITS(2); break; case STORED: /* get and verify stored block length */ BYTEBITS(); /* go to byte boundary */ NEEDBITS(32); if ((hold & 0xffff) != ((hold >> 16) ^ 0xffff)) { strm->msg = (char *)"invalid stored block lengths"; state->mode = BAD; break; } state->length = (unsigned)hold & 0xffff; Tracev((stderr, "inflate: stored length %u\n", state->length)); INITBITS(); /* copy stored block from input to output */ while (state->length != 0) { copy = state->length; PULL(); ROOM(); if (copy > have) copy = have; if (copy > left) copy = left; zmemcpy(put, next, copy); have -= copy; next += copy; left -= copy; put += copy; state->length -= copy; } Tracev((stderr, "inflate: stored end\n")); state->mode = TYPE; break; case TABLE: /* get dynamic table entries descriptor */ NEEDBITS(14); state->nlen = BITS(5) + 257; DROPBITS(5); state->ndist = BITS(5) + 1; DROPBITS(5); state->ncode = BITS(4) + 4; DROPBITS(4); #ifndef PKZIP_BUG_WORKAROUND if (state->nlen > 286 || state->ndist > 30) { strm->msg = (char *)"too many length or distance symbols"; state->mode = BAD; break; } #endif Tracev((stderr, "inflate: table sizes ok\n")); /* get code length code lengths (not a typo) */ state->have = 0; while (state->have < state->ncode) { NEEDBITS(3); state->lens[order[state->have++]] = (unsigned short)BITS(3); DROPBITS(3); } while (state->have < 19) state->lens[order[state->have++]] = 0; state->next = state->codes; state->lencode = (code const FAR *)(state->next); state->lenbits = 7; ret = inflate_table(CODES, state->lens, 19, &(state->next), &(state->lenbits), state->work); if (ret) { strm->msg = (char *)"invalid code lengths set"; state->mode = BAD; break; } Tracev((stderr, "inflate: code lengths ok\n")); /* get length and distance code code lengths */ state->have = 0; while (state->have < state->nlen + state->ndist) { for (;;) { here = state->lencode[BITS(state->lenbits)]; if ((unsigned)(here.bits) <= bits) break; PULLBYTE(); } if (here.val < 16) { DROPBITS(here.bits); state->lens[state->have++] = here.val; } else { if (here.val == 16) { NEEDBITS(here.bits + 2); DROPBITS(here.bits); if (state->have == 0) { strm->msg = (char *)"invalid bit length repeat"; state->mode = BAD; break; } len = (unsigned)(state->lens[state->have - 1]); copy = 3 + BITS(2); DROPBITS(2); } else if (here.val == 17) { NEEDBITS(here.bits + 3); DROPBITS(here.bits); len = 0; copy = 3 + BITS(3); DROPBITS(3); } else { NEEDBITS(here.bits + 7); DROPBITS(here.bits); len = 0; copy = 11 + BITS(7); DROPBITS(7); } if (state->have + copy > state->nlen + state->ndist) { strm->msg = (char *)"invalid bit length repeat"; state->mode = BAD; break; } while (copy--) state->lens[state->have++] = (unsigned short)len; } } /* handle error breaks in while */ if (state->mode == BAD) break; /* check for end-of-block code (better have one) */ if (state->lens[256] == 0) { strm->msg = (char *)"invalid code -- missing end-of-block"; state->mode = BAD; break; } /* build code tables -- note: do not change the lenbits or distbits values here (9 and 6) without reading the comments in inftrees.h concerning the ENOUGH constants, which depend on those values */ state->next = state->codes; state->lencode = (code const FAR *)(state->next); state->lenbits = 9; ret = inflate_table(LENS, state->lens, state->nlen, &(state->next), &(state->lenbits), state->work); if (ret) { strm->msg = (char *)"invalid literal/lengths set"; state->mode = BAD; break; } state->distcode = (code const FAR *)(state->next); state->distbits = 6; ret = inflate_table(DISTS, state->lens + state->nlen, state->ndist, &(state->next), &(state->distbits), state->work); if (ret) { strm->msg = (char *)"invalid distances set"; state->mode = BAD; break; } Tracev((stderr, "inflate: codes ok\n")); state->mode = LEN; case LEN: /* use inflate_fast() if we have enough input and output */ if (have >= 6 && left >= 258) { RESTORE(); if (state->whave < state->wsize) state->whave = state->wsize - left; inflate_fast(strm, state->wsize); LOAD(); break; } /* get a literal, length, or end-of-block code */ for (;;) { here = state->lencode[BITS(state->lenbits)]; if ((unsigned)(here.bits) <= bits) break; PULLBYTE(); } if (here.op && (here.op & 0xf0) == 0) { last = here; for (;;) { here = state->lencode[last.val + (BITS(last.bits + last.op) >> last.bits)]; if ((unsigned)(last.bits + here.bits) <= bits) break; PULLBYTE(); } DROPBITS(last.bits); } DROPBITS(here.bits); state->length = (unsigned)here.val; /* process literal */ if (here.op == 0) { Tracevv((stderr, here.val >= 0x20 && here.val < 0x7f ? "inflate: literal '%c'\n" : "inflate: literal 0x%02x\n", here.val)); ROOM(); *put++ = (unsigned char)(state->length); left--; state->mode = LEN; break; } /* process end of block */ if (here.op & 32) { Tracevv((stderr, "inflate: end of block\n")); state->mode = TYPE; break; } /* invalid code */ if (here.op & 64) { strm->msg = (char *)"invalid literal/length code"; state->mode = BAD; break; } /* length code -- get extra bits, if any */ state->extra = (unsigned)(here.op) & 15; if (state->extra != 0) { NEEDBITS(state->extra); state->length += BITS(state->extra); DROPBITS(state->extra); } Tracevv((stderr, "inflate: length %u\n", state->length)); /* get distance code */ for (;;) { here = state->distcode[BITS(state->distbits)]; if ((unsigned)(here.bits) <= bits) break; PULLBYTE(); } if ((here.op & 0xf0) == 0) { last = here; for (;;) { here = state->distcode[last.val + (BITS(last.bits + last.op) >> last.bits)]; if ((unsigned)(last.bits + here.bits) <= bits) break; PULLBYTE(); } DROPBITS(last.bits); } DROPBITS(here.bits); if (here.op & 64) { strm->msg = (char *)"invalid distance code"; state->mode = BAD; break; } state->offset = (unsigned)here.val; /* get distance extra bits, if any */ state->extra = (unsigned)(here.op) & 15; if (state->extra != 0) { NEEDBITS(state->extra); state->offset += BITS(state->extra); DROPBITS(state->extra); } if (state->offset > state->wsize - (state->whave < state->wsize ? left : 0)) { strm->msg = (char *)"invalid distance too far back"; state->mode = BAD; break; } Tracevv((stderr, "inflate: distance %u\n", state->offset)); /* copy match from window to output */ do { ROOM(); copy = state->wsize - state->offset; if (copy < left) { from = put + copy; copy = left - copy; } else { from = put - state->offset; copy = left; } if (copy > state->length) copy = state->length; state->length -= copy; left -= copy; do { *put++ = *from++; } while (--copy); } while (state->length != 0); break; case DONE: /* inflate stream terminated properly -- write leftover output */ ret = Z_STREAM_END; if (left < state->wsize) { if (out(out_desc, state->window, state->wsize - left)) ret = Z_BUF_ERROR; } goto inf_leave; case BAD: ret = Z_DATA_ERROR; goto inf_leave; default: /* can't happen, but makes compilers happy */ ret = Z_STREAM_ERROR; goto inf_leave; } /* Return unused input */ inf_leave: strm->next_in = next; strm->avail_in = have; return ret; } int ZEXPORT inflateBackEnd(strm) z_streamp strm; { if (strm == Z_NULL || strm->state == Z_NULL || strm->zfree == (free_func)0) return Z_STREAM_ERROR; ZFREE(strm, strm->state); strm->state = Z_NULL; Tracev((stderr, "inflate: end\n")); return Z_OK; } c-blosc-1.7.0/internal-complibs/zlib-1.2.8/inffast.c000066400000000000000000000322171254622165100216760ustar00rootroot00000000000000/* inffast.c -- fast decoding * Copyright (C) 1995-2008, 2010, 2013 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ #include "zutil.h" #include "inftrees.h" #include "inflate.h" #include "inffast.h" #ifndef ASMINF /* Allow machine dependent optimization for post-increment or pre-increment. Based on testing to date, Pre-increment preferred for: - PowerPC G3 (Adler) - MIPS R5000 (Randers-Pehrson) Post-increment preferred for: - none No measurable difference: - Pentium III (Anderson) - M68060 (Nikl) */ #ifdef POSTINC # define OFF 0 # define PUP(a) *(a)++ #else # define OFF 1 # define PUP(a) *++(a) #endif /* Decode literal, length, and distance codes and write out the resulting literal and match bytes until either not enough input or output is available, an end-of-block is encountered, or a data error is encountered. When large enough input and output buffers are supplied to inflate(), for example, a 16K input buffer and a 64K output buffer, more than 95% of the inflate execution time is spent in this routine. Entry assumptions: state->mode == LEN strm->avail_in >= 6 strm->avail_out >= 258 start >= strm->avail_out state->bits < 8 On return, state->mode is one of: LEN -- ran out of enough output space or enough available input TYPE -- reached end of block code, inflate() to interpret next block BAD -- error in block data Notes: - The maximum input bits used by a length/distance pair is 15 bits for the length code, 5 bits for the length extra, 15 bits for the distance code, and 13 bits for the distance extra. This totals 48 bits, or six bytes. Therefore if strm->avail_in >= 6, then there is enough input to avoid checking for available input while decoding. - The maximum bytes that a single length/distance pair can output is 258 bytes, which is the maximum length that can be coded. inflate_fast() requires strm->avail_out >= 258 for each loop to avoid checking for output space. */ void ZLIB_INTERNAL inflate_fast(strm, start) z_streamp strm; unsigned start; /* inflate()'s starting value for strm->avail_out */ { struct inflate_state FAR *state; z_const unsigned char FAR *in; /* local strm->next_in */ z_const unsigned char FAR *last; /* have enough input while in < last */ unsigned char FAR *out; /* local strm->next_out */ unsigned char FAR *beg; /* inflate()'s initial strm->next_out */ unsigned char FAR *end; /* while out < end, enough space available */ #ifdef INFLATE_STRICT unsigned dmax; /* maximum distance from zlib header */ #endif unsigned wsize; /* window size or zero if not using window */ unsigned whave; /* valid bytes in the window */ unsigned wnext; /* window write index */ unsigned char FAR *window; /* allocated sliding window, if wsize != 0 */ unsigned long hold; /* local strm->hold */ unsigned bits; /* local strm->bits */ code const FAR *lcode; /* local strm->lencode */ code const FAR *dcode; /* local strm->distcode */ unsigned lmask; /* mask for first level of length codes */ unsigned dmask; /* mask for first level of distance codes */ code here; /* retrieved table entry */ unsigned op; /* code bits, operation, extra bits, or */ /* window position, window bytes to copy */ unsigned len; /* match length, unused bytes */ unsigned dist; /* match distance */ unsigned char FAR *from; /* where to copy match from */ /* copy state to local variables */ state = (struct inflate_state FAR *)strm->state; in = strm->next_in - OFF; last = in + (strm->avail_in - 5); out = strm->next_out - OFF; beg = out - (start - strm->avail_out); end = out + (strm->avail_out - 257); #ifdef INFLATE_STRICT dmax = state->dmax; #endif wsize = state->wsize; whave = state->whave; wnext = state->wnext; window = state->window; hold = state->hold; bits = state->bits; lcode = state->lencode; dcode = state->distcode; lmask = (1U << state->lenbits) - 1; dmask = (1U << state->distbits) - 1; /* decode literals and length/distances until end-of-block or not enough input data or output space */ do { if (bits < 15) { hold += (unsigned long)(PUP(in)) << bits; bits += 8; hold += (unsigned long)(PUP(in)) << bits; bits += 8; } here = lcode[hold & lmask]; dolen: op = (unsigned)(here.bits); hold >>= op; bits -= op; op = (unsigned)(here.op); if (op == 0) { /* literal */ Tracevv((stderr, here.val >= 0x20 && here.val < 0x7f ? "inflate: literal '%c'\n" : "inflate: literal 0x%02x\n", here.val)); PUP(out) = (unsigned char)(here.val); } else if (op & 16) { /* length base */ len = (unsigned)(here.val); op &= 15; /* number of extra bits */ if (op) { if (bits < op) { hold += (unsigned long)(PUP(in)) << bits; bits += 8; } len += (unsigned)hold & ((1U << op) - 1); hold >>= op; bits -= op; } Tracevv((stderr, "inflate: length %u\n", len)); if (bits < 15) { hold += (unsigned long)(PUP(in)) << bits; bits += 8; hold += (unsigned long)(PUP(in)) << bits; bits += 8; } here = dcode[hold & dmask]; dodist: op = (unsigned)(here.bits); hold >>= op; bits -= op; op = (unsigned)(here.op); if (op & 16) { /* distance base */ dist = (unsigned)(here.val); op &= 15; /* number of extra bits */ if (bits < op) { hold += (unsigned long)(PUP(in)) << bits; bits += 8; if (bits < op) { hold += (unsigned long)(PUP(in)) << bits; bits += 8; } } dist += (unsigned)hold & ((1U << op) - 1); #ifdef INFLATE_STRICT if (dist > dmax) { strm->msg = (char *)"invalid distance too far back"; state->mode = BAD; break; } #endif hold >>= op; bits -= op; Tracevv((stderr, "inflate: distance %u\n", dist)); op = (unsigned)(out - beg); /* max distance in output */ if (dist > op) { /* see if copy from window */ op = dist - op; /* distance back in window */ if (op > whave) { if (state->sane) { strm->msg = (char *)"invalid distance too far back"; state->mode = BAD; break; } #ifdef INFLATE_ALLOW_INVALID_DISTANCE_TOOFAR_ARRR if (len <= op - whave) { do { PUP(out) = 0; } while (--len); continue; } len -= op - whave; do { PUP(out) = 0; } while (--op > whave); if (op == 0) { from = out - dist; do { PUP(out) = PUP(from); } while (--len); continue; } #endif } from = window - OFF; if (wnext == 0) { /* very common case */ from += wsize - op; if (op < len) { /* some from window */ len -= op; do { PUP(out) = PUP(from); } while (--op); from = out - dist; /* rest from output */ } } else if (wnext < op) { /* wrap around window */ from += wsize + wnext - op; op -= wnext; if (op < len) { /* some from end of window */ len -= op; do { PUP(out) = PUP(from); } while (--op); from = window - OFF; if (wnext < len) { /* some from start of window */ op = wnext; len -= op; do { PUP(out) = PUP(from); } while (--op); from = out - dist; /* rest from output */ } } } else { /* contiguous in window */ from += wnext - op; if (op < len) { /* some from window */ len -= op; do { PUP(out) = PUP(from); } while (--op); from = out - dist; /* rest from output */ } } while (len > 2) { PUP(out) = PUP(from); PUP(out) = PUP(from); PUP(out) = PUP(from); len -= 3; } if (len) { PUP(out) = PUP(from); if (len > 1) PUP(out) = PUP(from); } } else { from = out - dist; /* copy direct from output */ do { /* minimum length is three */ PUP(out) = PUP(from); PUP(out) = PUP(from); PUP(out) = PUP(from); len -= 3; } while (len > 2); if (len) { PUP(out) = PUP(from); if (len > 1) PUP(out) = PUP(from); } } } else if ((op & 64) == 0) { /* 2nd level distance code */ here = dcode[here.val + (hold & ((1U << op) - 1))]; goto dodist; } else { strm->msg = (char *)"invalid distance code"; state->mode = BAD; break; } } else if ((op & 64) == 0) { /* 2nd level length code */ here = lcode[here.val + (hold & ((1U << op) - 1))]; goto dolen; } else if (op & 32) { /* end-of-block */ Tracevv((stderr, "inflate: end of block\n")); state->mode = TYPE; break; } else { strm->msg = (char *)"invalid literal/length code"; state->mode = BAD; break; } } while (in < last && out < end); /* return unused bytes (on entry, bits < 8, so in won't go too far back) */ len = bits >> 3; in -= len; bits -= len << 3; hold &= (1U << bits) - 1; /* update state and return */ strm->next_in = in + OFF; strm->next_out = out + OFF; strm->avail_in = (unsigned)(in < last ? 5 + (last - in) : 5 - (in - last)); strm->avail_out = (unsigned)(out < end ? 257 + (end - out) : 257 - (out - end)); state->hold = hold; state->bits = bits; return; } /* inflate_fast() speedups that turned out slower (on a PowerPC G3 750CXe): - Using bit fields for code structure - Different op definition to avoid & for extra bits (do & for table bits) - Three separate decoding do-loops for direct, window, and wnext == 0 - Special case for distance > 1 copies to do overlapped load and store copy - Explicit branch predictions (based on measured branch probabilities) - Deferring match copy and interspersed it with decoding subsequent codes - Swapping literal/length else - Swapping window/direct else - Larger unrolled copy loops (three is about right) - Moving len -= 3 statement into middle of loop */ #endif /* !ASMINF */ c-blosc-1.7.0/internal-complibs/zlib-1.2.8/inffast.h000066400000000000000000000006531254622165100217020ustar00rootroot00000000000000/* inffast.h -- header to use inffast.c * Copyright (C) 1995-2003, 2010 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ /* WARNING: this file should *not* be used by applications. It is part of the implementation of the compression library and is subject to change. Applications should only use zlib.h. */ void ZLIB_INTERNAL inflate_fast OF((z_streamp strm, unsigned start)); c-blosc-1.7.0/internal-complibs/zlib-1.2.8/inffixed.h000066400000000000000000000142741254622165100220500ustar00rootroot00000000000000 /* inffixed.h -- table for decoding fixed codes * Generated automatically by makefixed(). */ /* WARNING: this file should *not* be used by applications. It is part of the implementation of this library and is subject to change. Applications should only use zlib.h. */ static const code lenfix[512] = { {96,7,0},{0,8,80},{0,8,16},{20,8,115},{18,7,31},{0,8,112},{0,8,48}, {0,9,192},{16,7,10},{0,8,96},{0,8,32},{0,9,160},{0,8,0},{0,8,128}, {0,8,64},{0,9,224},{16,7,6},{0,8,88},{0,8,24},{0,9,144},{19,7,59}, {0,8,120},{0,8,56},{0,9,208},{17,7,17},{0,8,104},{0,8,40},{0,9,176}, {0,8,8},{0,8,136},{0,8,72},{0,9,240},{16,7,4},{0,8,84},{0,8,20}, {21,8,227},{19,7,43},{0,8,116},{0,8,52},{0,9,200},{17,7,13},{0,8,100}, {0,8,36},{0,9,168},{0,8,4},{0,8,132},{0,8,68},{0,9,232},{16,7,8}, {0,8,92},{0,8,28},{0,9,152},{20,7,83},{0,8,124},{0,8,60},{0,9,216}, {18,7,23},{0,8,108},{0,8,44},{0,9,184},{0,8,12},{0,8,140},{0,8,76}, {0,9,248},{16,7,3},{0,8,82},{0,8,18},{21,8,163},{19,7,35},{0,8,114}, {0,8,50},{0,9,196},{17,7,11},{0,8,98},{0,8,34},{0,9,164},{0,8,2}, {0,8,130},{0,8,66},{0,9,228},{16,7,7},{0,8,90},{0,8,26},{0,9,148}, {20,7,67},{0,8,122},{0,8,58},{0,9,212},{18,7,19},{0,8,106},{0,8,42}, {0,9,180},{0,8,10},{0,8,138},{0,8,74},{0,9,244},{16,7,5},{0,8,86}, {0,8,22},{64,8,0},{19,7,51},{0,8,118},{0,8,54},{0,9,204},{17,7,15}, {0,8,102},{0,8,38},{0,9,172},{0,8,6},{0,8,134},{0,8,70},{0,9,236}, {16,7,9},{0,8,94},{0,8,30},{0,9,156},{20,7,99},{0,8,126},{0,8,62}, {0,9,220},{18,7,27},{0,8,110},{0,8,46},{0,9,188},{0,8,14},{0,8,142}, {0,8,78},{0,9,252},{96,7,0},{0,8,81},{0,8,17},{21,8,131},{18,7,31}, {0,8,113},{0,8,49},{0,9,194},{16,7,10},{0,8,97},{0,8,33},{0,9,162}, {0,8,1},{0,8,129},{0,8,65},{0,9,226},{16,7,6},{0,8,89},{0,8,25}, {0,9,146},{19,7,59},{0,8,121},{0,8,57},{0,9,210},{17,7,17},{0,8,105}, {0,8,41},{0,9,178},{0,8,9},{0,8,137},{0,8,73},{0,9,242},{16,7,4}, {0,8,85},{0,8,21},{16,8,258},{19,7,43},{0,8,117},{0,8,53},{0,9,202}, {17,7,13},{0,8,101},{0,8,37},{0,9,170},{0,8,5},{0,8,133},{0,8,69}, {0,9,234},{16,7,8},{0,8,93},{0,8,29},{0,9,154},{20,7,83},{0,8,125}, {0,8,61},{0,9,218},{18,7,23},{0,8,109},{0,8,45},{0,9,186},{0,8,13}, {0,8,141},{0,8,77},{0,9,250},{16,7,3},{0,8,83},{0,8,19},{21,8,195}, {19,7,35},{0,8,115},{0,8,51},{0,9,198},{17,7,11},{0,8,99},{0,8,35}, {0,9,166},{0,8,3},{0,8,131},{0,8,67},{0,9,230},{16,7,7},{0,8,91}, {0,8,27},{0,9,150},{20,7,67},{0,8,123},{0,8,59},{0,9,214},{18,7,19}, {0,8,107},{0,8,43},{0,9,182},{0,8,11},{0,8,139},{0,8,75},{0,9,246}, {16,7,5},{0,8,87},{0,8,23},{64,8,0},{19,7,51},{0,8,119},{0,8,55}, {0,9,206},{17,7,15},{0,8,103},{0,8,39},{0,9,174},{0,8,7},{0,8,135}, {0,8,71},{0,9,238},{16,7,9},{0,8,95},{0,8,31},{0,9,158},{20,7,99}, {0,8,127},{0,8,63},{0,9,222},{18,7,27},{0,8,111},{0,8,47},{0,9,190}, {0,8,15},{0,8,143},{0,8,79},{0,9,254},{96,7,0},{0,8,80},{0,8,16}, {20,8,115},{18,7,31},{0,8,112},{0,8,48},{0,9,193},{16,7,10},{0,8,96}, {0,8,32},{0,9,161},{0,8,0},{0,8,128},{0,8,64},{0,9,225},{16,7,6}, {0,8,88},{0,8,24},{0,9,145},{19,7,59},{0,8,120},{0,8,56},{0,9,209}, {17,7,17},{0,8,104},{0,8,40},{0,9,177},{0,8,8},{0,8,136},{0,8,72}, {0,9,241},{16,7,4},{0,8,84},{0,8,20},{21,8,227},{19,7,43},{0,8,116}, {0,8,52},{0,9,201},{17,7,13},{0,8,100},{0,8,36},{0,9,169},{0,8,4}, {0,8,132},{0,8,68},{0,9,233},{16,7,8},{0,8,92},{0,8,28},{0,9,153}, {20,7,83},{0,8,124},{0,8,60},{0,9,217},{18,7,23},{0,8,108},{0,8,44}, {0,9,185},{0,8,12},{0,8,140},{0,8,76},{0,9,249},{16,7,3},{0,8,82}, {0,8,18},{21,8,163},{19,7,35},{0,8,114},{0,8,50},{0,9,197},{17,7,11}, {0,8,98},{0,8,34},{0,9,165},{0,8,2},{0,8,130},{0,8,66},{0,9,229}, {16,7,7},{0,8,90},{0,8,26},{0,9,149},{20,7,67},{0,8,122},{0,8,58}, {0,9,213},{18,7,19},{0,8,106},{0,8,42},{0,9,181},{0,8,10},{0,8,138}, {0,8,74},{0,9,245},{16,7,5},{0,8,86},{0,8,22},{64,8,0},{19,7,51}, {0,8,118},{0,8,54},{0,9,205},{17,7,15},{0,8,102},{0,8,38},{0,9,173}, {0,8,6},{0,8,134},{0,8,70},{0,9,237},{16,7,9},{0,8,94},{0,8,30}, {0,9,157},{20,7,99},{0,8,126},{0,8,62},{0,9,221},{18,7,27},{0,8,110}, {0,8,46},{0,9,189},{0,8,14},{0,8,142},{0,8,78},{0,9,253},{96,7,0}, {0,8,81},{0,8,17},{21,8,131},{18,7,31},{0,8,113},{0,8,49},{0,9,195}, {16,7,10},{0,8,97},{0,8,33},{0,9,163},{0,8,1},{0,8,129},{0,8,65}, {0,9,227},{16,7,6},{0,8,89},{0,8,25},{0,9,147},{19,7,59},{0,8,121}, {0,8,57},{0,9,211},{17,7,17},{0,8,105},{0,8,41},{0,9,179},{0,8,9}, {0,8,137},{0,8,73},{0,9,243},{16,7,4},{0,8,85},{0,8,21},{16,8,258}, {19,7,43},{0,8,117},{0,8,53},{0,9,203},{17,7,13},{0,8,101},{0,8,37}, {0,9,171},{0,8,5},{0,8,133},{0,8,69},{0,9,235},{16,7,8},{0,8,93}, {0,8,29},{0,9,155},{20,7,83},{0,8,125},{0,8,61},{0,9,219},{18,7,23}, {0,8,109},{0,8,45},{0,9,187},{0,8,13},{0,8,141},{0,8,77},{0,9,251}, {16,7,3},{0,8,83},{0,8,19},{21,8,195},{19,7,35},{0,8,115},{0,8,51}, {0,9,199},{17,7,11},{0,8,99},{0,8,35},{0,9,167},{0,8,3},{0,8,131}, {0,8,67},{0,9,231},{16,7,7},{0,8,91},{0,8,27},{0,9,151},{20,7,67}, {0,8,123},{0,8,59},{0,9,215},{18,7,19},{0,8,107},{0,8,43},{0,9,183}, {0,8,11},{0,8,139},{0,8,75},{0,9,247},{16,7,5},{0,8,87},{0,8,23}, {64,8,0},{19,7,51},{0,8,119},{0,8,55},{0,9,207},{17,7,15},{0,8,103}, {0,8,39},{0,9,175},{0,8,7},{0,8,135},{0,8,71},{0,9,239},{16,7,9}, {0,8,95},{0,8,31},{0,9,159},{20,7,99},{0,8,127},{0,8,63},{0,9,223}, {18,7,27},{0,8,111},{0,8,47},{0,9,191},{0,8,15},{0,8,143},{0,8,79}, {0,9,255} }; static const code distfix[32] = { {16,5,1},{23,5,257},{19,5,17},{27,5,4097},{17,5,5},{25,5,1025}, {21,5,65},{29,5,16385},{16,5,3},{24,5,513},{20,5,33},{28,5,8193}, {18,5,9},{26,5,2049},{22,5,129},{64,5,0},{16,5,2},{23,5,385}, {19,5,25},{27,5,6145},{17,5,7},{25,5,1537},{21,5,97},{29,5,24577}, {16,5,4},{24,5,769},{20,5,49},{28,5,12289},{18,5,13},{26,5,3073}, {22,5,193},{64,5,0} }; c-blosc-1.7.0/internal-complibs/zlib-1.2.8/inflate.c000066400000000000000000001504101254622165100216620ustar00rootroot00000000000000/* inflate.c -- zlib decompression * Copyright (C) 1995-2012 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ /* * Change history: * * 1.2.beta0 24 Nov 2002 * - First version -- complete rewrite of inflate to simplify code, avoid * creation of window when not needed, minimize use of window when it is * needed, make inffast.c even faster, implement gzip decoding, and to * improve code readability and style over the previous zlib inflate code * * 1.2.beta1 25 Nov 2002 * - Use pointers for available input and output checking in inffast.c * - Remove input and output counters in inffast.c * - Change inffast.c entry and loop from avail_in >= 7 to >= 6 * - Remove unnecessary second byte pull from length extra in inffast.c * - Unroll direct copy to three copies per loop in inffast.c * * 1.2.beta2 4 Dec 2002 * - Change external routine names to reduce potential conflicts * - Correct filename to inffixed.h for fixed tables in inflate.c * - Make hbuf[] unsigned char to match parameter type in inflate.c * - Change strm->next_out[-state->offset] to *(strm->next_out - state->offset) * to avoid negation problem on Alphas (64 bit) in inflate.c * * 1.2.beta3 22 Dec 2002 * - Add comments on state->bits assertion in inffast.c * - Add comments on op field in inftrees.h * - Fix bug in reuse of allocated window after inflateReset() * - Remove bit fields--back to byte structure for speed * - Remove distance extra == 0 check in inflate_fast()--only helps for lengths * - Change post-increments to pre-increments in inflate_fast(), PPC biased? * - Add compile time option, POSTINC, to use post-increments instead (Intel?) * - Make MATCH copy in inflate() much faster for when inflate_fast() not used * - Use local copies of stream next and avail values, as well as local bit * buffer and bit count in inflate()--for speed when inflate_fast() not used * * 1.2.beta4 1 Jan 2003 * - Split ptr - 257 statements in inflate_table() to avoid compiler warnings * - Move a comment on output buffer sizes from inffast.c to inflate.c * - Add comments in inffast.c to introduce the inflate_fast() routine * - Rearrange window copies in inflate_fast() for speed and simplification * - Unroll last copy for window match in inflate_fast() * - Use local copies of window variables in inflate_fast() for speed * - Pull out common wnext == 0 case for speed in inflate_fast() * - Make op and len in inflate_fast() unsigned for consistency * - Add FAR to lcode and dcode declarations in inflate_fast() * - Simplified bad distance check in inflate_fast() * - Added inflateBackInit(), inflateBack(), and inflateBackEnd() in new * source file infback.c to provide a call-back interface to inflate for * programs like gzip and unzip -- uses window as output buffer to avoid * window copying * * 1.2.beta5 1 Jan 2003 * - Improved inflateBack() interface to allow the caller to provide initial * input in strm. * - Fixed stored blocks bug in inflateBack() * * 1.2.beta6 4 Jan 2003 * - Added comments in inffast.c on effectiveness of POSTINC * - Typecasting all around to reduce compiler warnings * - Changed loops from while (1) or do {} while (1) to for (;;), again to * make compilers happy * - Changed type of window in inflateBackInit() to unsigned char * * * 1.2.beta7 27 Jan 2003 * - Changed many types to unsigned or unsigned short to avoid warnings * - Added inflateCopy() function * * 1.2.0 9 Mar 2003 * - Changed inflateBack() interface to provide separate opaque descriptors * for the in() and out() functions * - Changed inflateBack() argument and in_func typedef to swap the length * and buffer address return values for the input function * - Check next_in and next_out for Z_NULL on entry to inflate() * * The history for versions after 1.2.0 are in ChangeLog in zlib distribution. */ #include "zutil.h" #include "inftrees.h" #include "inflate.h" #include "inffast.h" #ifdef MAKEFIXED # ifndef BUILDFIXED # define BUILDFIXED # endif #endif /* function prototypes */ local void fixedtables OF((struct inflate_state FAR *state)); local int updatewindow OF((z_streamp strm, const unsigned char FAR *end, unsigned copy)); #ifdef BUILDFIXED void makefixed OF((void)); #endif local unsigned syncsearch OF((unsigned FAR *have, const unsigned char FAR *buf, unsigned len)); int ZEXPORT inflateResetKeep(strm) z_streamp strm; { struct inflate_state FAR *state; if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; strm->total_in = strm->total_out = state->total = 0; strm->msg = Z_NULL; if (state->wrap) /* to support ill-conceived Java test suite */ strm->adler = state->wrap & 1; state->mode = HEAD; state->last = 0; state->havedict = 0; state->dmax = 32768U; state->head = Z_NULL; state->hold = 0; state->bits = 0; state->lencode = state->distcode = state->next = state->codes; state->sane = 1; state->back = -1; Tracev((stderr, "inflate: reset\n")); return Z_OK; } int ZEXPORT inflateReset(strm) z_streamp strm; { struct inflate_state FAR *state; if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; state->wsize = 0; state->whave = 0; state->wnext = 0; return inflateResetKeep(strm); } int ZEXPORT inflateReset2(strm, windowBits) z_streamp strm; int windowBits; { int wrap; struct inflate_state FAR *state; /* get the state */ if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; /* extract wrap request from windowBits parameter */ if (windowBits < 0) { wrap = 0; windowBits = -windowBits; } else { wrap = (windowBits >> 4) + 1; #ifdef GUNZIP if (windowBits < 48) windowBits &= 15; #endif } /* set number of window bits, free window if different */ if (windowBits && (windowBits < 8 || windowBits > 15)) return Z_STREAM_ERROR; if (state->window != Z_NULL && state->wbits != (unsigned)windowBits) { ZFREE(strm, state->window); state->window = Z_NULL; } /* update state and reset the rest of it */ state->wrap = wrap; state->wbits = (unsigned)windowBits; return inflateReset(strm); } int ZEXPORT inflateInit2_(strm, windowBits, version, stream_size) z_streamp strm; int windowBits; const char *version; int stream_size; { int ret; struct inflate_state FAR *state; if (version == Z_NULL || version[0] != ZLIB_VERSION[0] || stream_size != (int)(sizeof(z_stream))) return Z_VERSION_ERROR; if (strm == Z_NULL) return Z_STREAM_ERROR; strm->msg = Z_NULL; /* in case we return an error */ if (strm->zalloc == (alloc_func)0) { #ifdef Z_SOLO return Z_STREAM_ERROR; #else strm->zalloc = zcalloc; strm->opaque = (voidpf)0; #endif } if (strm->zfree == (free_func)0) #ifdef Z_SOLO return Z_STREAM_ERROR; #else strm->zfree = zcfree; #endif state = (struct inflate_state FAR *) ZALLOC(strm, 1, sizeof(struct inflate_state)); if (state == Z_NULL) return Z_MEM_ERROR; Tracev((stderr, "inflate: allocated\n")); strm->state = (struct internal_state FAR *)state; state->window = Z_NULL; ret = inflateReset2(strm, windowBits); if (ret != Z_OK) { ZFREE(strm, state); strm->state = Z_NULL; } return ret; } int ZEXPORT inflateInit_(strm, version, stream_size) z_streamp strm; const char *version; int stream_size; { return inflateInit2_(strm, DEF_WBITS, version, stream_size); } int ZEXPORT inflatePrime(strm, bits, value) z_streamp strm; int bits; int value; { struct inflate_state FAR *state; if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; if (bits < 0) { state->hold = 0; state->bits = 0; return Z_OK; } if (bits > 16 || state->bits + bits > 32) return Z_STREAM_ERROR; value &= (1L << bits) - 1; state->hold += value << state->bits; state->bits += bits; return Z_OK; } /* Return state with length and distance decoding tables and index sizes set to fixed code decoding. Normally this returns fixed tables from inffixed.h. If BUILDFIXED is defined, then instead this routine builds the tables the first time it's called, and returns those tables the first time and thereafter. This reduces the size of the code by about 2K bytes, in exchange for a little execution time. However, BUILDFIXED should not be used for threaded applications, since the rewriting of the tables and virgin may not be thread-safe. */ local void fixedtables(state) struct inflate_state FAR *state; { #ifdef BUILDFIXED static int virgin = 1; static code *lenfix, *distfix; static code fixed[544]; /* build fixed huffman tables if first call (may not be thread safe) */ if (virgin) { unsigned sym, bits; static code *next; /* literal/length table */ sym = 0; while (sym < 144) state->lens[sym++] = 8; while (sym < 256) state->lens[sym++] = 9; while (sym < 280) state->lens[sym++] = 7; while (sym < 288) state->lens[sym++] = 8; next = fixed; lenfix = next; bits = 9; inflate_table(LENS, state->lens, 288, &(next), &(bits), state->work); /* distance table */ sym = 0; while (sym < 32) state->lens[sym++] = 5; distfix = next; bits = 5; inflate_table(DISTS, state->lens, 32, &(next), &(bits), state->work); /* do this just once */ virgin = 0; } #else /* !BUILDFIXED */ # include "inffixed.h" #endif /* BUILDFIXED */ state->lencode = lenfix; state->lenbits = 9; state->distcode = distfix; state->distbits = 5; } #ifdef MAKEFIXED #include /* Write out the inffixed.h that is #include'd above. Defining MAKEFIXED also defines BUILDFIXED, so the tables are built on the fly. makefixed() writes those tables to stdout, which would be piped to inffixed.h. A small program can simply call makefixed to do this: void makefixed(void); int main(void) { makefixed(); return 0; } Then that can be linked with zlib built with MAKEFIXED defined and run: a.out > inffixed.h */ void makefixed() { unsigned low, size; struct inflate_state state; fixedtables(&state); puts(" /* inffixed.h -- table for decoding fixed codes"); puts(" * Generated automatically by makefixed()."); puts(" */"); puts(""); puts(" /* WARNING: this file should *not* be used by applications."); puts(" It is part of the implementation of this library and is"); puts(" subject to change. Applications should only use zlib.h."); puts(" */"); puts(""); size = 1U << 9; printf(" static const code lenfix[%u] = {", size); low = 0; for (;;) { if ((low % 7) == 0) printf("\n "); printf("{%u,%u,%d}", (low & 127) == 99 ? 64 : state.lencode[low].op, state.lencode[low].bits, state.lencode[low].val); if (++low == size) break; putchar(','); } puts("\n };"); size = 1U << 5; printf("\n static const code distfix[%u] = {", size); low = 0; for (;;) { if ((low % 6) == 0) printf("\n "); printf("{%u,%u,%d}", state.distcode[low].op, state.distcode[low].bits, state.distcode[low].val); if (++low == size) break; putchar(','); } puts("\n };"); } #endif /* MAKEFIXED */ /* Update the window with the last wsize (normally 32K) bytes written before returning. If window does not exist yet, create it. This is only called when a window is already in use, or when output has been written during this inflate call, but the end of the deflate stream has not been reached yet. It is also called to create a window for dictionary data when a dictionary is loaded. Providing output buffers larger than 32K to inflate() should provide a speed advantage, since only the last 32K of output is copied to the sliding window upon return from inflate(), and since all distances after the first 32K of output will fall in the output data, making match copies simpler and faster. The advantage may be dependent on the size of the processor's data caches. */ local int updatewindow(strm, end, copy) z_streamp strm; const Bytef *end; unsigned copy; { struct inflate_state FAR *state; unsigned dist; state = (struct inflate_state FAR *)strm->state; /* if it hasn't been done already, allocate space for the window */ if (state->window == Z_NULL) { state->window = (unsigned char FAR *) ZALLOC(strm, 1U << state->wbits, sizeof(unsigned char)); if (state->window == Z_NULL) return 1; } /* if window not in use yet, initialize */ if (state->wsize == 0) { state->wsize = 1U << state->wbits; state->wnext = 0; state->whave = 0; } /* copy state->wsize or less output bytes into the circular window */ if (copy >= state->wsize) { zmemcpy(state->window, end - state->wsize, state->wsize); state->wnext = 0; state->whave = state->wsize; } else { dist = state->wsize - state->wnext; if (dist > copy) dist = copy; zmemcpy(state->window + state->wnext, end - copy, dist); copy -= dist; if (copy) { zmemcpy(state->window, end - copy, copy); state->wnext = copy; state->whave = state->wsize; } else { state->wnext += dist; if (state->wnext == state->wsize) state->wnext = 0; if (state->whave < state->wsize) state->whave += dist; } } return 0; } /* Macros for inflate(): */ /* check function to use adler32() for zlib or crc32() for gzip */ #ifdef GUNZIP # define UPDATE(check, buf, len) \ (state->flags ? crc32(check, buf, len) : adler32(check, buf, len)) #else # define UPDATE(check, buf, len) adler32(check, buf, len) #endif /* check macros for header crc */ #ifdef GUNZIP # define CRC2(check, word) \ do { \ hbuf[0] = (unsigned char)(word); \ hbuf[1] = (unsigned char)((word) >> 8); \ check = crc32(check, hbuf, 2); \ } while (0) # define CRC4(check, word) \ do { \ hbuf[0] = (unsigned char)(word); \ hbuf[1] = (unsigned char)((word) >> 8); \ hbuf[2] = (unsigned char)((word) >> 16); \ hbuf[3] = (unsigned char)((word) >> 24); \ check = crc32(check, hbuf, 4); \ } while (0) #endif /* Load registers with state in inflate() for speed */ #define LOAD() \ do { \ put = strm->next_out; \ left = strm->avail_out; \ next = strm->next_in; \ have = strm->avail_in; \ hold = state->hold; \ bits = state->bits; \ } while (0) /* Restore state from registers in inflate() */ #define RESTORE() \ do { \ strm->next_out = put; \ strm->avail_out = left; \ strm->next_in = next; \ strm->avail_in = have; \ state->hold = hold; \ state->bits = bits; \ } while (0) /* Clear the input bit accumulator */ #define INITBITS() \ do { \ hold = 0; \ bits = 0; \ } while (0) /* Get a byte of input into the bit accumulator, or return from inflate() if there is no input available. */ #define PULLBYTE() \ do { \ if (have == 0) goto inf_leave; \ have--; \ hold += (unsigned long)(*next++) << bits; \ bits += 8; \ } while (0) /* Assure that there are at least n bits in the bit accumulator. If there is not enough available input to do that, then return from inflate(). */ #define NEEDBITS(n) \ do { \ while (bits < (unsigned)(n)) \ PULLBYTE(); \ } while (0) /* Return the low n bits of the bit accumulator (n < 16) */ #define BITS(n) \ ((unsigned)hold & ((1U << (n)) - 1)) /* Remove n bits from the bit accumulator */ #define DROPBITS(n) \ do { \ hold >>= (n); \ bits -= (unsigned)(n); \ } while (0) /* Remove zero to seven bits as needed to go to a byte boundary */ #define BYTEBITS() \ do { \ hold >>= bits & 7; \ bits -= bits & 7; \ } while (0) /* inflate() uses a state machine to process as much input data and generate as much output data as possible before returning. The state machine is structured roughly as follows: for (;;) switch (state) { ... case STATEn: if (not enough input data or output space to make progress) return; ... make progress ... state = STATEm; break; ... } so when inflate() is called again, the same case is attempted again, and if the appropriate resources are provided, the machine proceeds to the next state. The NEEDBITS() macro is usually the way the state evaluates whether it can proceed or should return. NEEDBITS() does the return if the requested bits are not available. The typical use of the BITS macros is: NEEDBITS(n); ... do something with BITS(n) ... DROPBITS(n); where NEEDBITS(n) either returns from inflate() if there isn't enough input left to load n bits into the accumulator, or it continues. BITS(n) gives the low n bits in the accumulator. When done, DROPBITS(n) drops the low n bits off the accumulator. INITBITS() clears the accumulator and sets the number of available bits to zero. BYTEBITS() discards just enough bits to put the accumulator on a byte boundary. After BYTEBITS() and a NEEDBITS(8), then BITS(8) would return the next byte in the stream. NEEDBITS(n) uses PULLBYTE() to get an available byte of input, or to return if there is no input available. The decoding of variable length codes uses PULLBYTE() directly in order to pull just enough bytes to decode the next code, and no more. Some states loop until they get enough input, making sure that enough state information is maintained to continue the loop where it left off if NEEDBITS() returns in the loop. For example, want, need, and keep would all have to actually be part of the saved state in case NEEDBITS() returns: case STATEw: while (want < need) { NEEDBITS(n); keep[want++] = BITS(n); DROPBITS(n); } state = STATEx; case STATEx: As shown above, if the next state is also the next case, then the break is omitted. A state may also return if there is not enough output space available to complete that state. Those states are copying stored data, writing a literal byte, and copying a matching string. When returning, a "goto inf_leave" is used to update the total counters, update the check value, and determine whether any progress has been made during that inflate() call in order to return the proper return code. Progress is defined as a change in either strm->avail_in or strm->avail_out. When there is a window, goto inf_leave will update the window with the last output written. If a goto inf_leave occurs in the middle of decompression and there is no window currently, goto inf_leave will create one and copy output to the window for the next call of inflate(). In this implementation, the flush parameter of inflate() only affects the return code (per zlib.h). inflate() always writes as much as possible to strm->next_out, given the space available and the provided input--the effect documented in zlib.h of Z_SYNC_FLUSH. Furthermore, inflate() always defers the allocation of and copying into a sliding window until necessary, which provides the effect documented in zlib.h for Z_FINISH when the entire input stream available. So the only thing the flush parameter actually does is: when flush is set to Z_FINISH, inflate() cannot return Z_OK. Instead it will return Z_BUF_ERROR if it has not reached the end of the stream. */ int ZEXPORT inflate(strm, flush) z_streamp strm; int flush; { struct inflate_state FAR *state; z_const unsigned char FAR *next; /* next input */ unsigned char FAR *put; /* next output */ unsigned have, left; /* available input and output */ unsigned long hold; /* bit buffer */ unsigned bits; /* bits in bit buffer */ unsigned in, out; /* save starting available input and output */ unsigned copy; /* number of stored or match bytes to copy */ unsigned char FAR *from; /* where to copy match bytes from */ code here; /* current decoding table entry */ code last; /* parent table entry */ unsigned len; /* length to copy for repeats, bits to drop */ int ret; /* return code */ #ifdef GUNZIP unsigned char hbuf[4]; /* buffer for gzip header crc calculation */ #endif static const unsigned short order[19] = /* permutation of code lengths */ {16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15}; if (strm == Z_NULL || strm->state == Z_NULL || strm->next_out == Z_NULL || (strm->next_in == Z_NULL && strm->avail_in != 0)) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; if (state->mode == TYPE) state->mode = TYPEDO; /* skip check */ LOAD(); in = have; out = left; ret = Z_OK; for (;;) switch (state->mode) { case HEAD: if (state->wrap == 0) { state->mode = TYPEDO; break; } NEEDBITS(16); #ifdef GUNZIP if ((state->wrap & 2) && hold == 0x8b1f) { /* gzip header */ state->check = crc32(0L, Z_NULL, 0); CRC2(state->check, hold); INITBITS(); state->mode = FLAGS; break; } state->flags = 0; /* expect zlib header */ if (state->head != Z_NULL) state->head->done = -1; if (!(state->wrap & 1) || /* check if zlib header allowed */ #else if ( #endif ((BITS(8) << 8) + (hold >> 8)) % 31) { strm->msg = (char *)"incorrect header check"; state->mode = BAD; break; } if (BITS(4) != Z_DEFLATED) { strm->msg = (char *)"unknown compression method"; state->mode = BAD; break; } DROPBITS(4); len = BITS(4) + 8; if (state->wbits == 0) state->wbits = len; else if (len > state->wbits) { strm->msg = (char *)"invalid window size"; state->mode = BAD; break; } state->dmax = 1U << len; Tracev((stderr, "inflate: zlib header ok\n")); strm->adler = state->check = adler32(0L, Z_NULL, 0); state->mode = hold & 0x200 ? DICTID : TYPE; INITBITS(); break; #ifdef GUNZIP case FLAGS: NEEDBITS(16); state->flags = (int)(hold); if ((state->flags & 0xff) != Z_DEFLATED) { strm->msg = (char *)"unknown compression method"; state->mode = BAD; break; } if (state->flags & 0xe000) { strm->msg = (char *)"unknown header flags set"; state->mode = BAD; break; } if (state->head != Z_NULL) state->head->text = (int)((hold >> 8) & 1); if (state->flags & 0x0200) CRC2(state->check, hold); INITBITS(); state->mode = TIME; case TIME: NEEDBITS(32); if (state->head != Z_NULL) state->head->time = hold; if (state->flags & 0x0200) CRC4(state->check, hold); INITBITS(); state->mode = OS; case OS: NEEDBITS(16); if (state->head != Z_NULL) { state->head->xflags = (int)(hold & 0xff); state->head->os = (int)(hold >> 8); } if (state->flags & 0x0200) CRC2(state->check, hold); INITBITS(); state->mode = EXLEN; case EXLEN: if (state->flags & 0x0400) { NEEDBITS(16); state->length = (unsigned)(hold); if (state->head != Z_NULL) state->head->extra_len = (unsigned)hold; if (state->flags & 0x0200) CRC2(state->check, hold); INITBITS(); } else if (state->head != Z_NULL) state->head->extra = Z_NULL; state->mode = EXTRA; case EXTRA: if (state->flags & 0x0400) { copy = state->length; if (copy > have) copy = have; if (copy) { if (state->head != Z_NULL && state->head->extra != Z_NULL) { len = state->head->extra_len - state->length; zmemcpy(state->head->extra + len, next, len + copy > state->head->extra_max ? state->head->extra_max - len : copy); } if (state->flags & 0x0200) state->check = crc32(state->check, next, copy); have -= copy; next += copy; state->length -= copy; } if (state->length) goto inf_leave; } state->length = 0; state->mode = NAME; case NAME: if (state->flags & 0x0800) { if (have == 0) goto inf_leave; copy = 0; do { len = (unsigned)(next[copy++]); if (state->head != Z_NULL && state->head->name != Z_NULL && state->length < state->head->name_max) state->head->name[state->length++] = len; } while (len && copy < have); if (state->flags & 0x0200) state->check = crc32(state->check, next, copy); have -= copy; next += copy; if (len) goto inf_leave; } else if (state->head != Z_NULL) state->head->name = Z_NULL; state->length = 0; state->mode = COMMENT; case COMMENT: if (state->flags & 0x1000) { if (have == 0) goto inf_leave; copy = 0; do { len = (unsigned)(next[copy++]); if (state->head != Z_NULL && state->head->comment != Z_NULL && state->length < state->head->comm_max) state->head->comment[state->length++] = len; } while (len && copy < have); if (state->flags & 0x0200) state->check = crc32(state->check, next, copy); have -= copy; next += copy; if (len) goto inf_leave; } else if (state->head != Z_NULL) state->head->comment = Z_NULL; state->mode = HCRC; case HCRC: if (state->flags & 0x0200) { NEEDBITS(16); if (hold != (state->check & 0xffff)) { strm->msg = (char *)"header crc mismatch"; state->mode = BAD; break; } INITBITS(); } if (state->head != Z_NULL) { state->head->hcrc = (int)((state->flags >> 9) & 1); state->head->done = 1; } strm->adler = state->check = crc32(0L, Z_NULL, 0); state->mode = TYPE; break; #endif case DICTID: NEEDBITS(32); strm->adler = state->check = ZSWAP32(hold); INITBITS(); state->mode = DICT; case DICT: if (state->havedict == 0) { RESTORE(); return Z_NEED_DICT; } strm->adler = state->check = adler32(0L, Z_NULL, 0); state->mode = TYPE; case TYPE: if (flush == Z_BLOCK || flush == Z_TREES) goto inf_leave; case TYPEDO: if (state->last) { BYTEBITS(); state->mode = CHECK; break; } NEEDBITS(3); state->last = BITS(1); DROPBITS(1); switch (BITS(2)) { case 0: /* stored block */ Tracev((stderr, "inflate: stored block%s\n", state->last ? " (last)" : "")); state->mode = STORED; break; case 1: /* fixed block */ fixedtables(state); Tracev((stderr, "inflate: fixed codes block%s\n", state->last ? " (last)" : "")); state->mode = LEN_; /* decode codes */ if (flush == Z_TREES) { DROPBITS(2); goto inf_leave; } break; case 2: /* dynamic block */ Tracev((stderr, "inflate: dynamic codes block%s\n", state->last ? " (last)" : "")); state->mode = TABLE; break; case 3: strm->msg = (char *)"invalid block type"; state->mode = BAD; } DROPBITS(2); break; case STORED: BYTEBITS(); /* go to byte boundary */ NEEDBITS(32); if ((hold & 0xffff) != ((hold >> 16) ^ 0xffff)) { strm->msg = (char *)"invalid stored block lengths"; state->mode = BAD; break; } state->length = (unsigned)hold & 0xffff; Tracev((stderr, "inflate: stored length %u\n", state->length)); INITBITS(); state->mode = COPY_; if (flush == Z_TREES) goto inf_leave; case COPY_: state->mode = COPY; case COPY: copy = state->length; if (copy) { if (copy > have) copy = have; if (copy > left) copy = left; if (copy == 0) goto inf_leave; zmemcpy(put, next, copy); have -= copy; next += copy; left -= copy; put += copy; state->length -= copy; break; } Tracev((stderr, "inflate: stored end\n")); state->mode = TYPE; break; case TABLE: NEEDBITS(14); state->nlen = BITS(5) + 257; DROPBITS(5); state->ndist = BITS(5) + 1; DROPBITS(5); state->ncode = BITS(4) + 4; DROPBITS(4); #ifndef PKZIP_BUG_WORKAROUND if (state->nlen > 286 || state->ndist > 30) { strm->msg = (char *)"too many length or distance symbols"; state->mode = BAD; break; } #endif Tracev((stderr, "inflate: table sizes ok\n")); state->have = 0; state->mode = LENLENS; case LENLENS: while (state->have < state->ncode) { NEEDBITS(3); state->lens[order[state->have++]] = (unsigned short)BITS(3); DROPBITS(3); } while (state->have < 19) state->lens[order[state->have++]] = 0; state->next = state->codes; state->lencode = (const code FAR *)(state->next); state->lenbits = 7; ret = inflate_table(CODES, state->lens, 19, &(state->next), &(state->lenbits), state->work); if (ret) { strm->msg = (char *)"invalid code lengths set"; state->mode = BAD; break; } Tracev((stderr, "inflate: code lengths ok\n")); state->have = 0; state->mode = CODELENS; case CODELENS: while (state->have < state->nlen + state->ndist) { for (;;) { here = state->lencode[BITS(state->lenbits)]; if ((unsigned)(here.bits) <= bits) break; PULLBYTE(); } if (here.val < 16) { DROPBITS(here.bits); state->lens[state->have++] = here.val; } else { if (here.val == 16) { NEEDBITS(here.bits + 2); DROPBITS(here.bits); if (state->have == 0) { strm->msg = (char *)"invalid bit length repeat"; state->mode = BAD; break; } len = state->lens[state->have - 1]; copy = 3 + BITS(2); DROPBITS(2); } else if (here.val == 17) { NEEDBITS(here.bits + 3); DROPBITS(here.bits); len = 0; copy = 3 + BITS(3); DROPBITS(3); } else { NEEDBITS(here.bits + 7); DROPBITS(here.bits); len = 0; copy = 11 + BITS(7); DROPBITS(7); } if (state->have + copy > state->nlen + state->ndist) { strm->msg = (char *)"invalid bit length repeat"; state->mode = BAD; break; } while (copy--) state->lens[state->have++] = (unsigned short)len; } } /* handle error breaks in while */ if (state->mode == BAD) break; /* check for end-of-block code (better have one) */ if (state->lens[256] == 0) { strm->msg = (char *)"invalid code -- missing end-of-block"; state->mode = BAD; break; } /* build code tables -- note: do not change the lenbits or distbits values here (9 and 6) without reading the comments in inftrees.h concerning the ENOUGH constants, which depend on those values */ state->next = state->codes; state->lencode = (const code FAR *)(state->next); state->lenbits = 9; ret = inflate_table(LENS, state->lens, state->nlen, &(state->next), &(state->lenbits), state->work); if (ret) { strm->msg = (char *)"invalid literal/lengths set"; state->mode = BAD; break; } state->distcode = (const code FAR *)(state->next); state->distbits = 6; ret = inflate_table(DISTS, state->lens + state->nlen, state->ndist, &(state->next), &(state->distbits), state->work); if (ret) { strm->msg = (char *)"invalid distances set"; state->mode = BAD; break; } Tracev((stderr, "inflate: codes ok\n")); state->mode = LEN_; if (flush == Z_TREES) goto inf_leave; case LEN_: state->mode = LEN; case LEN: if (have >= 6 && left >= 258) { RESTORE(); inflate_fast(strm, out); LOAD(); if (state->mode == TYPE) state->back = -1; break; } state->back = 0; for (;;) { here = state->lencode[BITS(state->lenbits)]; if ((unsigned)(here.bits) <= bits) break; PULLBYTE(); } if (here.op && (here.op & 0xf0) == 0) { last = here; for (;;) { here = state->lencode[last.val + (BITS(last.bits + last.op) >> last.bits)]; if ((unsigned)(last.bits + here.bits) <= bits) break; PULLBYTE(); } DROPBITS(last.bits); state->back += last.bits; } DROPBITS(here.bits); state->back += here.bits; state->length = (unsigned)here.val; if ((int)(here.op) == 0) { Tracevv((stderr, here.val >= 0x20 && here.val < 0x7f ? "inflate: literal '%c'\n" : "inflate: literal 0x%02x\n", here.val)); state->mode = LIT; break; } if (here.op & 32) { Tracevv((stderr, "inflate: end of block\n")); state->back = -1; state->mode = TYPE; break; } if (here.op & 64) { strm->msg = (char *)"invalid literal/length code"; state->mode = BAD; break; } state->extra = (unsigned)(here.op) & 15; state->mode = LENEXT; case LENEXT: if (state->extra) { NEEDBITS(state->extra); state->length += BITS(state->extra); DROPBITS(state->extra); state->back += state->extra; } Tracevv((stderr, "inflate: length %u\n", state->length)); state->was = state->length; state->mode = DIST; case DIST: for (;;) { here = state->distcode[BITS(state->distbits)]; if ((unsigned)(here.bits) <= bits) break; PULLBYTE(); } if ((here.op & 0xf0) == 0) { last = here; for (;;) { here = state->distcode[last.val + (BITS(last.bits + last.op) >> last.bits)]; if ((unsigned)(last.bits + here.bits) <= bits) break; PULLBYTE(); } DROPBITS(last.bits); state->back += last.bits; } DROPBITS(here.bits); state->back += here.bits; if (here.op & 64) { strm->msg = (char *)"invalid distance code"; state->mode = BAD; break; } state->offset = (unsigned)here.val; state->extra = (unsigned)(here.op) & 15; state->mode = DISTEXT; case DISTEXT: if (state->extra) { NEEDBITS(state->extra); state->offset += BITS(state->extra); DROPBITS(state->extra); state->back += state->extra; } #ifdef INFLATE_STRICT if (state->offset > state->dmax) { strm->msg = (char *)"invalid distance too far back"; state->mode = BAD; break; } #endif Tracevv((stderr, "inflate: distance %u\n", state->offset)); state->mode = MATCH; case MATCH: if (left == 0) goto inf_leave; copy = out - left; if (state->offset > copy) { /* copy from window */ copy = state->offset - copy; if (copy > state->whave) { if (state->sane) { strm->msg = (char *)"invalid distance too far back"; state->mode = BAD; break; } #ifdef INFLATE_ALLOW_INVALID_DISTANCE_TOOFAR_ARRR Trace((stderr, "inflate.c too far\n")); copy -= state->whave; if (copy > state->length) copy = state->length; if (copy > left) copy = left; left -= copy; state->length -= copy; do { *put++ = 0; } while (--copy); if (state->length == 0) state->mode = LEN; break; #endif } if (copy > state->wnext) { copy -= state->wnext; from = state->window + (state->wsize - copy); } else from = state->window + (state->wnext - copy); if (copy > state->length) copy = state->length; } else { /* copy from output */ from = put - state->offset; copy = state->length; } if (copy > left) copy = left; left -= copy; state->length -= copy; do { *put++ = *from++; } while (--copy); if (state->length == 0) state->mode = LEN; break; case LIT: if (left == 0) goto inf_leave; *put++ = (unsigned char)(state->length); left--; state->mode = LEN; break; case CHECK: if (state->wrap) { NEEDBITS(32); out -= left; strm->total_out += out; state->total += out; if (out) strm->adler = state->check = UPDATE(state->check, put - out, out); out = left; if (( #ifdef GUNZIP state->flags ? hold : #endif ZSWAP32(hold)) != state->check) { strm->msg = (char *)"incorrect data check"; state->mode = BAD; break; } INITBITS(); Tracev((stderr, "inflate: check matches trailer\n")); } #ifdef GUNZIP state->mode = LENGTH; case LENGTH: if (state->wrap && state->flags) { NEEDBITS(32); if (hold != (state->total & 0xffffffffUL)) { strm->msg = (char *)"incorrect length check"; state->mode = BAD; break; } INITBITS(); Tracev((stderr, "inflate: length matches trailer\n")); } #endif state->mode = DONE; case DONE: ret = Z_STREAM_END; goto inf_leave; case BAD: ret = Z_DATA_ERROR; goto inf_leave; case MEM: return Z_MEM_ERROR; case SYNC: default: return Z_STREAM_ERROR; } /* Return from inflate(), updating the total counts and the check value. If there was no progress during the inflate() call, return a buffer error. Call updatewindow() to create and/or update the window state. Note: a memory error from inflate() is non-recoverable. */ inf_leave: RESTORE(); if (state->wsize || (out != strm->avail_out && state->mode < BAD && (state->mode < CHECK || flush != Z_FINISH))) if (updatewindow(strm, strm->next_out, out - strm->avail_out)) { state->mode = MEM; return Z_MEM_ERROR; } in -= strm->avail_in; out -= strm->avail_out; strm->total_in += in; strm->total_out += out; state->total += out; if (state->wrap && out) strm->adler = state->check = UPDATE(state->check, strm->next_out - out, out); strm->data_type = state->bits + (state->last ? 64 : 0) + (state->mode == TYPE ? 128 : 0) + (state->mode == LEN_ || state->mode == COPY_ ? 256 : 0); if (((in == 0 && out == 0) || flush == Z_FINISH) && ret == Z_OK) ret = Z_BUF_ERROR; return ret; } int ZEXPORT inflateEnd(strm) z_streamp strm; { struct inflate_state FAR *state; if (strm == Z_NULL || strm->state == Z_NULL || strm->zfree == (free_func)0) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; if (state->window != Z_NULL) ZFREE(strm, state->window); ZFREE(strm, strm->state); strm->state = Z_NULL; Tracev((stderr, "inflate: end\n")); return Z_OK; } int ZEXPORT inflateGetDictionary(strm, dictionary, dictLength) z_streamp strm; Bytef *dictionary; uInt *dictLength; { struct inflate_state FAR *state; /* check state */ if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; /* copy dictionary */ if (state->whave && dictionary != Z_NULL) { zmemcpy(dictionary, state->window + state->wnext, state->whave - state->wnext); zmemcpy(dictionary + state->whave - state->wnext, state->window, state->wnext); } if (dictLength != Z_NULL) *dictLength = state->whave; return Z_OK; } int ZEXPORT inflateSetDictionary(strm, dictionary, dictLength) z_streamp strm; const Bytef *dictionary; uInt dictLength; { struct inflate_state FAR *state; unsigned long dictid; int ret; /* check state */ if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; if (state->wrap != 0 && state->mode != DICT) return Z_STREAM_ERROR; /* check for correct dictionary identifier */ if (state->mode == DICT) { dictid = adler32(0L, Z_NULL, 0); dictid = adler32(dictid, dictionary, dictLength); if (dictid != state->check) return Z_DATA_ERROR; } /* copy dictionary to window using updatewindow(), which will amend the existing dictionary if appropriate */ ret = updatewindow(strm, dictionary + dictLength, dictLength); if (ret) { state->mode = MEM; return Z_MEM_ERROR; } state->havedict = 1; Tracev((stderr, "inflate: dictionary set\n")); return Z_OK; } int ZEXPORT inflateGetHeader(strm, head) z_streamp strm; gz_headerp head; { struct inflate_state FAR *state; /* check state */ if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; if ((state->wrap & 2) == 0) return Z_STREAM_ERROR; /* save header structure */ state->head = head; head->done = 0; return Z_OK; } /* Search buf[0..len-1] for the pattern: 0, 0, 0xff, 0xff. Return when found or when out of input. When called, *have is the number of pattern bytes found in order so far, in 0..3. On return *have is updated to the new state. If on return *have equals four, then the pattern was found and the return value is how many bytes were read including the last byte of the pattern. If *have is less than four, then the pattern has not been found yet and the return value is len. In the latter case, syncsearch() can be called again with more data and the *have state. *have is initialized to zero for the first call. */ local unsigned syncsearch(have, buf, len) unsigned FAR *have; const unsigned char FAR *buf; unsigned len; { unsigned got; unsigned next; got = *have; next = 0; while (next < len && got < 4) { if ((int)(buf[next]) == (got < 2 ? 0 : 0xff)) got++; else if (buf[next]) got = 0; else got = 4 - got; next++; } *have = got; return next; } int ZEXPORT inflateSync(strm) z_streamp strm; { unsigned len; /* number of bytes to look at or looked at */ unsigned long in, out; /* temporary to save total_in and total_out */ unsigned char buf[4]; /* to restore bit buffer to byte string */ struct inflate_state FAR *state; /* check parameters */ if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; if (strm->avail_in == 0 && state->bits < 8) return Z_BUF_ERROR; /* if first time, start search in bit buffer */ if (state->mode != SYNC) { state->mode = SYNC; state->hold <<= state->bits & 7; state->bits -= state->bits & 7; len = 0; while (state->bits >= 8) { buf[len++] = (unsigned char)(state->hold); state->hold >>= 8; state->bits -= 8; } state->have = 0; syncsearch(&(state->have), buf, len); } /* search available input */ len = syncsearch(&(state->have), strm->next_in, strm->avail_in); strm->avail_in -= len; strm->next_in += len; strm->total_in += len; /* return no joy or set up to restart inflate() on a new block */ if (state->have != 4) return Z_DATA_ERROR; in = strm->total_in; out = strm->total_out; inflateReset(strm); strm->total_in = in; strm->total_out = out; state->mode = TYPE; return Z_OK; } /* Returns true if inflate is currently at the end of a block generated by Z_SYNC_FLUSH or Z_FULL_FLUSH. This function is used by one PPP implementation to provide an additional safety check. PPP uses Z_SYNC_FLUSH but removes the length bytes of the resulting empty stored block. When decompressing, PPP checks that at the end of input packet, inflate is waiting for these length bytes. */ int ZEXPORT inflateSyncPoint(strm) z_streamp strm; { struct inflate_state FAR *state; if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; return state->mode == STORED && state->bits == 0; } int ZEXPORT inflateCopy(dest, source) z_streamp dest; z_streamp source; { struct inflate_state FAR *state; struct inflate_state FAR *copy; unsigned char FAR *window; unsigned wsize; /* check input */ if (dest == Z_NULL || source == Z_NULL || source->state == Z_NULL || source->zalloc == (alloc_func)0 || source->zfree == (free_func)0) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)source->state; /* allocate space */ copy = (struct inflate_state FAR *) ZALLOC(source, 1, sizeof(struct inflate_state)); if (copy == Z_NULL) return Z_MEM_ERROR; window = Z_NULL; if (state->window != Z_NULL) { window = (unsigned char FAR *) ZALLOC(source, 1U << state->wbits, sizeof(unsigned char)); if (window == Z_NULL) { ZFREE(source, copy); return Z_MEM_ERROR; } } /* copy state */ zmemcpy((voidpf)dest, (voidpf)source, sizeof(z_stream)); zmemcpy((voidpf)copy, (voidpf)state, sizeof(struct inflate_state)); if (state->lencode >= state->codes && state->lencode <= state->codes + ENOUGH - 1) { copy->lencode = copy->codes + (state->lencode - state->codes); copy->distcode = copy->codes + (state->distcode - state->codes); } copy->next = copy->codes + (state->next - state->codes); if (window != Z_NULL) { wsize = 1U << state->wbits; zmemcpy(window, state->window, wsize); } copy->window = window; dest->state = (struct internal_state FAR *)copy; return Z_OK; } int ZEXPORT inflateUndermine(strm, subvert) z_streamp strm; int subvert; { struct inflate_state FAR *state; if (strm == Z_NULL || strm->state == Z_NULL) return Z_STREAM_ERROR; state = (struct inflate_state FAR *)strm->state; state->sane = !subvert; #ifdef INFLATE_ALLOW_INVALID_DISTANCE_TOOFAR_ARRR return Z_OK; #else state->sane = 1; return Z_DATA_ERROR; #endif } long ZEXPORT inflateMark(strm) z_streamp strm; { struct inflate_state FAR *state; if (strm == Z_NULL || strm->state == Z_NULL) return -1L << 16; state = (struct inflate_state FAR *)strm->state; return ((long)(state->back) << 16) + (state->mode == COPY ? state->length : (state->mode == MATCH ? state->was - state->length : 0)); } c-blosc-1.7.0/internal-complibs/zlib-1.2.8/inflate.h000066400000000000000000000143771254622165100217020ustar00rootroot00000000000000/* inflate.h -- internal inflate state definition * Copyright (C) 1995-2009 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ /* WARNING: this file should *not* be used by applications. It is part of the implementation of the compression library and is subject to change. Applications should only use zlib.h. */ /* define NO_GZIP when compiling if you want to disable gzip header and trailer decoding by inflate(). NO_GZIP would be used to avoid linking in the crc code when it is not needed. For shared libraries, gzip decoding should be left enabled. */ #ifndef NO_GZIP # define GUNZIP #endif /* Possible inflate modes between inflate() calls */ typedef enum { HEAD, /* i: waiting for magic header */ FLAGS, /* i: waiting for method and flags (gzip) */ TIME, /* i: waiting for modification time (gzip) */ OS, /* i: waiting for extra flags and operating system (gzip) */ EXLEN, /* i: waiting for extra length (gzip) */ EXTRA, /* i: waiting for extra bytes (gzip) */ NAME, /* i: waiting for end of file name (gzip) */ COMMENT, /* i: waiting for end of comment (gzip) */ HCRC, /* i: waiting for header crc (gzip) */ DICTID, /* i: waiting for dictionary check value */ DICT, /* waiting for inflateSetDictionary() call */ TYPE, /* i: waiting for type bits, including last-flag bit */ TYPEDO, /* i: same, but skip check to exit inflate on new block */ STORED, /* i: waiting for stored size (length and complement) */ COPY_, /* i/o: same as COPY below, but only first time in */ COPY, /* i/o: waiting for input or output to copy stored block */ TABLE, /* i: waiting for dynamic block table lengths */ LENLENS, /* i: waiting for code length code lengths */ CODELENS, /* i: waiting for length/lit and distance code lengths */ LEN_, /* i: same as LEN below, but only first time in */ LEN, /* i: waiting for length/lit/eob code */ LENEXT, /* i: waiting for length extra bits */ DIST, /* i: waiting for distance code */ DISTEXT, /* i: waiting for distance extra bits */ MATCH, /* o: waiting for output space to copy string */ LIT, /* o: waiting for output space to write literal */ CHECK, /* i: waiting for 32-bit check value */ LENGTH, /* i: waiting for 32-bit length (gzip) */ DONE, /* finished check, done -- remain here until reset */ BAD, /* got a data error -- remain here until reset */ MEM, /* got an inflate() memory error -- remain here until reset */ SYNC /* looking for synchronization bytes to restart inflate() */ } inflate_mode; /* State transitions between above modes - (most modes can go to BAD or MEM on error -- not shown for clarity) Process header: HEAD -> (gzip) or (zlib) or (raw) (gzip) -> FLAGS -> TIME -> OS -> EXLEN -> EXTRA -> NAME -> COMMENT -> HCRC -> TYPE (zlib) -> DICTID or TYPE DICTID -> DICT -> TYPE (raw) -> TYPEDO Read deflate blocks: TYPE -> TYPEDO -> STORED or TABLE or LEN_ or CHECK STORED -> COPY_ -> COPY -> TYPE TABLE -> LENLENS -> CODELENS -> LEN_ LEN_ -> LEN Read deflate codes in fixed or dynamic block: LEN -> LENEXT or LIT or TYPE LENEXT -> DIST -> DISTEXT -> MATCH -> LEN LIT -> LEN Process trailer: CHECK -> LENGTH -> DONE */ /* state maintained between inflate() calls. Approximately 10K bytes. */ struct inflate_state { inflate_mode mode; /* current inflate mode */ int last; /* true if processing last block */ int wrap; /* bit 0 true for zlib, bit 1 true for gzip */ int havedict; /* true if dictionary provided */ int flags; /* gzip header method and flags (0 if zlib) */ unsigned dmax; /* zlib header max distance (INFLATE_STRICT) */ unsigned long check; /* protected copy of check value */ unsigned long total; /* protected copy of output count */ gz_headerp head; /* where to save gzip header information */ /* sliding window */ unsigned wbits; /* log base 2 of requested window size */ unsigned wsize; /* window size or zero if not using window */ unsigned whave; /* valid bytes in the window */ unsigned wnext; /* window write index */ unsigned char FAR *window; /* allocated sliding window, if needed */ /* bit accumulator */ unsigned long hold; /* input bit accumulator */ unsigned bits; /* number of bits in "in" */ /* for string and stored block copying */ unsigned length; /* literal or length of data to copy */ unsigned offset; /* distance back to copy string from */ /* for table and code decoding */ unsigned extra; /* extra bits needed */ /* fixed and dynamic code tables */ code const FAR *lencode; /* starting table for length/literal codes */ code const FAR *distcode; /* starting table for distance codes */ unsigned lenbits; /* index bits for lencode */ unsigned distbits; /* index bits for distcode */ /* dynamic table building */ unsigned ncode; /* number of code length code lengths */ unsigned nlen; /* number of length code lengths */ unsigned ndist; /* number of distance code lengths */ unsigned have; /* number of code lengths in lens[] */ code FAR *next; /* next available space in codes[] */ unsigned short lens[320]; /* temporary storage for code lengths */ unsigned short work[288]; /* work area for code table building */ code codes[ENOUGH]; /* space for code tables */ int sane; /* if false, allow invalid distance too far */ int back; /* bits back of last unprocessed length/lit */ unsigned was; /* initial length of match */ }; c-blosc-1.7.0/internal-complibs/zlib-1.2.8/inftrees.c000066400000000000000000000313441254622165100220630ustar00rootroot00000000000000/* inftrees.c -- generate Huffman trees for efficient decoding * Copyright (C) 1995-2013 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ #include "zutil.h" #include "inftrees.h" #define MAXBITS 15 const char inflate_copyright[] = " inflate 1.2.8 Copyright 1995-2013 Mark Adler "; /* If you use the zlib library in a product, an acknowledgment is welcome in the documentation of your product. If for some reason you cannot include such an acknowledgment, I would appreciate that you keep this copyright string in the executable of your product. */ /* Build a set of tables to decode the provided canonical Huffman code. The code lengths are lens[0..codes-1]. The result starts at *table, whose indices are 0..2^bits-1. work is a writable array of at least lens shorts, which is used as a work area. type is the type of code to be generated, CODES, LENS, or DISTS. On return, zero is success, -1 is an invalid code, and +1 means that ENOUGH isn't enough. table on return points to the next available entry's address. bits is the requested root table index bits, and on return it is the actual root table index bits. It will differ if the request is greater than the longest code or if it is less than the shortest code. */ int ZLIB_INTERNAL inflate_table(type, lens, codes, table, bits, work) codetype type; unsigned short FAR *lens; unsigned codes; code FAR * FAR *table; unsigned FAR *bits; unsigned short FAR *work; { unsigned len; /* a code's length in bits */ unsigned sym; /* index of code symbols */ unsigned min, max; /* minimum and maximum code lengths */ unsigned root; /* number of index bits for root table */ unsigned curr; /* number of index bits for current table */ unsigned drop; /* code bits to drop for sub-table */ int left; /* number of prefix codes available */ unsigned used; /* code entries in table used */ unsigned huff; /* Huffman code */ unsigned incr; /* for incrementing code, index */ unsigned fill; /* index for replicating entries */ unsigned low; /* low bits for current root entry */ unsigned mask; /* mask for low root bits */ code here; /* table entry for duplication */ code FAR *next; /* next available space in table */ const unsigned short FAR *base; /* base value table to use */ const unsigned short FAR *extra; /* extra bits table to use */ int end; /* use base and extra for symbol > end */ unsigned short count[MAXBITS+1]; /* number of codes of each length */ unsigned short offs[MAXBITS+1]; /* offsets in table for each length */ static const unsigned short lbase[31] = { /* Length codes 257..285 base */ 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 15, 17, 19, 23, 27, 31, 35, 43, 51, 59, 67, 83, 99, 115, 131, 163, 195, 227, 258, 0, 0}; static const unsigned short lext[31] = { /* Length codes 257..285 extra */ 16, 16, 16, 16, 16, 16, 16, 16, 17, 17, 17, 17, 18, 18, 18, 18, 19, 19, 19, 19, 20, 20, 20, 20, 21, 21, 21, 21, 16, 72, 78}; static const unsigned short dbase[32] = { /* Distance codes 0..29 base */ 1, 2, 3, 4, 5, 7, 9, 13, 17, 25, 33, 49, 65, 97, 129, 193, 257, 385, 513, 769, 1025, 1537, 2049, 3073, 4097, 6145, 8193, 12289, 16385, 24577, 0, 0}; static const unsigned short dext[32] = { /* Distance codes 0..29 extra */ 16, 16, 16, 16, 17, 17, 18, 18, 19, 19, 20, 20, 21, 21, 22, 22, 23, 23, 24, 24, 25, 25, 26, 26, 27, 27, 28, 28, 29, 29, 64, 64}; /* Process a set of code lengths to create a canonical Huffman code. The code lengths are lens[0..codes-1]. Each length corresponds to the symbols 0..codes-1. The Huffman code is generated by first sorting the symbols by length from short to long, and retaining the symbol order for codes with equal lengths. Then the code starts with all zero bits for the first code of the shortest length, and the codes are integer increments for the same length, and zeros are appended as the length increases. For the deflate format, these bits are stored backwards from their more natural integer increment ordering, and so when the decoding tables are built in the large loop below, the integer codes are incremented backwards. This routine assumes, but does not check, that all of the entries in lens[] are in the range 0..MAXBITS. The caller must assure this. 1..MAXBITS is interpreted as that code length. zero means that that symbol does not occur in this code. The codes are sorted by computing a count of codes for each length, creating from that a table of starting indices for each length in the sorted table, and then entering the symbols in order in the sorted table. The sorted table is work[], with that space being provided by the caller. The length counts are used for other purposes as well, i.e. finding the minimum and maximum length codes, determining if there are any codes at all, checking for a valid set of lengths, and looking ahead at length counts to determine sub-table sizes when building the decoding tables. */ /* accumulate lengths for codes (assumes lens[] all in 0..MAXBITS) */ for (len = 0; len <= MAXBITS; len++) count[len] = 0; for (sym = 0; sym < codes; sym++) count[lens[sym]]++; /* bound code lengths, force root to be within code lengths */ root = *bits; for (max = MAXBITS; max >= 1; max--) if (count[max] != 0) break; if (root > max) root = max; if (max == 0) { /* no symbols to code at all */ here.op = (unsigned char)64; /* invalid code marker */ here.bits = (unsigned char)1; here.val = (unsigned short)0; *(*table)++ = here; /* make a table to force an error */ *(*table)++ = here; *bits = 1; return 0; /* no symbols, but wait for decoding to report error */ } for (min = 1; min < max; min++) if (count[min] != 0) break; if (root < min) root = min; /* check for an over-subscribed or incomplete set of lengths */ left = 1; for (len = 1; len <= MAXBITS; len++) { left <<= 1; left -= count[len]; if (left < 0) return -1; /* over-subscribed */ } if (left > 0 && (type == CODES || max != 1)) return -1; /* incomplete set */ /* generate offsets into symbol table for each length for sorting */ offs[1] = 0; for (len = 1; len < MAXBITS; len++) offs[len + 1] = offs[len] + count[len]; /* sort symbols by length, by symbol order within each length */ for (sym = 0; sym < codes; sym++) if (lens[sym] != 0) work[offs[lens[sym]]++] = (unsigned short)sym; /* Create and fill in decoding tables. In this loop, the table being filled is at next and has curr index bits. The code being used is huff with length len. That code is converted to an index by dropping drop bits off of the bottom. For codes where len is less than drop + curr, those top drop + curr - len bits are incremented through all values to fill the table with replicated entries. root is the number of index bits for the root table. When len exceeds root, sub-tables are created pointed to by the root entry with an index of the low root bits of huff. This is saved in low to check for when a new sub-table should be started. drop is zero when the root table is being filled, and drop is root when sub-tables are being filled. When a new sub-table is needed, it is necessary to look ahead in the code lengths to determine what size sub-table is needed. The length counts are used for this, and so count[] is decremented as codes are entered in the tables. used keeps track of how many table entries have been allocated from the provided *table space. It is checked for LENS and DIST tables against the constants ENOUGH_LENS and ENOUGH_DISTS to guard against changes in the initial root table size constants. See the comments in inftrees.h for more information. sym increments through all symbols, and the loop terminates when all codes of length max, i.e. all codes, have been processed. This routine permits incomplete codes, so another loop after this one fills in the rest of the decoding tables with invalid code markers. */ /* set up for code type */ switch (type) { case CODES: base = extra = work; /* dummy value--not used */ end = 19; break; case LENS: base = lbase; base -= 257; extra = lext; extra -= 257; end = 256; break; default: /* DISTS */ base = dbase; extra = dext; end = -1; } /* initialize state for loop */ huff = 0; /* starting code */ sym = 0; /* starting code symbol */ len = min; /* starting code length */ next = *table; /* current table to fill in */ curr = root; /* current table index bits */ drop = 0; /* current bits to drop from code for index */ low = (unsigned)(-1); /* trigger new sub-table when len > root */ used = 1U << root; /* use root table entries */ mask = used - 1; /* mask for comparing low */ /* check available table space */ if ((type == LENS && used > ENOUGH_LENS) || (type == DISTS && used > ENOUGH_DISTS)) return 1; /* process all codes and make table entries */ for (;;) { /* create table entry */ here.bits = (unsigned char)(len - drop); if ((int)(work[sym]) < end) { here.op = (unsigned char)0; here.val = work[sym]; } else if ((int)(work[sym]) > end) { here.op = (unsigned char)(extra[work[sym]]); here.val = base[work[sym]]; } else { here.op = (unsigned char)(32 + 64); /* end of block */ here.val = 0; } /* replicate for those indices with low len bits equal to huff */ incr = 1U << (len - drop); fill = 1U << curr; min = fill; /* save offset to next table */ do { fill -= incr; next[(huff >> drop) + fill] = here; } while (fill != 0); /* backwards increment the len-bit code huff */ incr = 1U << (len - 1); while (huff & incr) incr >>= 1; if (incr != 0) { huff &= incr - 1; huff += incr; } else huff = 0; /* go to next symbol, update count, len */ sym++; if (--(count[len]) == 0) { if (len == max) break; len = lens[work[sym]]; } /* create new sub-table if needed */ if (len > root && (huff & mask) != low) { /* if first time, transition to sub-tables */ if (drop == 0) drop = root; /* increment past last table */ next += min; /* here min is 1 << curr */ /* determine length of next table */ curr = len - drop; left = (int)(1 << curr); while (curr + drop < max) { left -= count[curr + drop]; if (left <= 0) break; curr++; left <<= 1; } /* check for enough space */ used += 1U << curr; if ((type == LENS && used > ENOUGH_LENS) || (type == DISTS && used > ENOUGH_DISTS)) return 1; /* point entry in root table to sub-table */ low = huff & mask; (*table)[low].op = (unsigned char)curr; (*table)[low].bits = (unsigned char)root; (*table)[low].val = (unsigned short)(next - *table); } } /* fill in remaining table entry if code is incomplete (guaranteed to have at most one remaining entry, since if the code is incomplete, the maximum code length that was allowed to get this far is one bit) */ if (huff != 0) { here.op = (unsigned char)64; /* invalid code marker */ here.bits = (unsigned char)(len - drop); here.val = (unsigned short)0; next[huff] = here; } /* set return parameters */ *table += used; *bits = root; return 0; } c-blosc-1.7.0/internal-complibs/zlib-1.2.8/inftrees.h000066400000000000000000000055601254622165100220710ustar00rootroot00000000000000/* inftrees.h -- header to use inftrees.c * Copyright (C) 1995-2005, 2010 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ /* WARNING: this file should *not* be used by applications. It is part of the implementation of the compression library and is subject to change. Applications should only use zlib.h. */ /* Structure for decoding tables. Each entry provides either the information needed to do the operation requested by the code that indexed that table entry, or it provides a pointer to another table that indexes more bits of the code. op indicates whether the entry is a pointer to another table, a literal, a length or distance, an end-of-block, or an invalid code. For a table pointer, the low four bits of op is the number of index bits of that table. For a length or distance, the low four bits of op is the number of extra bits to get after the code. bits is the number of bits in this code or part of the code to drop off of the bit buffer. val is the actual byte to output in the case of a literal, the base length or distance, or the offset from the current table to the next table. Each entry is four bytes. */ typedef struct { unsigned char op; /* operation, extra bits, table bits */ unsigned char bits; /* bits in this part of the code */ unsigned short val; /* offset in table or code value */ } code; /* op values as set by inflate_table(): 00000000 - literal 0000tttt - table link, tttt != 0 is the number of table index bits 0001eeee - length or distance, eeee is the number of extra bits 01100000 - end of block 01000000 - invalid code */ /* Maximum size of the dynamic table. The maximum number of code structures is 1444, which is the sum of 852 for literal/length codes and 592 for distance codes. These values were found by exhaustive searches using the program examples/enough.c found in the zlib distribtution. The arguments to that program are the number of symbols, the initial root table size, and the maximum bit length of a code. "enough 286 9 15" for literal/length codes returns returns 852, and "enough 30 6 15" for distance codes returns 592. The initial root table size (9 or 6) is found in the fifth argument of the inflate_table() calls in inflate.c and infback.c. If the root table size is changed, then these maximum sizes would be need to be recalculated and updated. */ #define ENOUGH_LENS 852 #define ENOUGH_DISTS 592 #define ENOUGH (ENOUGH_LENS+ENOUGH_DISTS) /* Type of code to build for inflate_table() */ typedef enum { CODES, LENS, DISTS } codetype; int ZLIB_INTERNAL inflate_table OF((codetype type, unsigned short FAR *lens, unsigned codes, code FAR * FAR *table, unsigned FAR *bits, unsigned short FAR *work)); c-blosc-1.7.0/internal-complibs/zlib-1.2.8/trees.c000066400000000000000000001263371254622165100213750ustar00rootroot00000000000000/* trees.c -- output deflated data using Huffman coding * Copyright (C) 1995-2012 Jean-loup Gailly * detect_data_type() function provided freely by Cosmin Truta, 2006 * For conditions of distribution and use, see copyright notice in zlib.h */ /* * ALGORITHM * * The "deflation" process uses several Huffman trees. The more * common source values are represented by shorter bit sequences. * * Each code tree is stored in a compressed form which is itself * a Huffman encoding of the lengths of all the code strings (in * ascending order by source values). The actual code strings are * reconstructed from the lengths in the inflate process, as described * in the deflate specification. * * REFERENCES * * Deutsch, L.P.,"'Deflate' Compressed Data Format Specification". * Available in ftp.uu.net:/pub/archiving/zip/doc/deflate-1.1.doc * * Storer, James A. * Data Compression: Methods and Theory, pp. 49-50. * Computer Science Press, 1988. ISBN 0-7167-8156-5. * * Sedgewick, R. * Algorithms, p290. * Addison-Wesley, 1983. ISBN 0-201-06672-6. */ /* @(#) $Id$ */ /* #define GEN_TREES_H */ #include "deflate.h" #ifdef DEBUG # include #endif /* =========================================================================== * Constants */ #define MAX_BL_BITS 7 /* Bit length codes must not exceed MAX_BL_BITS bits */ #define END_BLOCK 256 /* end of block literal code */ #define REP_3_6 16 /* repeat previous bit length 3-6 times (2 bits of repeat count) */ #define REPZ_3_10 17 /* repeat a zero length 3-10 times (3 bits of repeat count) */ #define REPZ_11_138 18 /* repeat a zero length 11-138 times (7 bits of repeat count) */ local const int extra_lbits[LENGTH_CODES] /* extra bits for each length code */ = {0,0,0,0,0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3,4,4,4,4,5,5,5,5,0}; local const int extra_dbits[D_CODES] /* extra bits for each distance code */ = {0,0,0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,12,12,13,13}; local const int extra_blbits[BL_CODES]/* extra bits for each bit length code */ = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,3,7}; local const uch bl_order[BL_CODES] = {16,17,18,0,8,7,9,6,10,5,11,4,12,3,13,2,14,1,15}; /* The lengths of the bit length codes are sent in order of decreasing * probability, to avoid transmitting the lengths for unused bit length codes. */ /* =========================================================================== * Local data. These are initialized only once. */ #define DIST_CODE_LEN 512 /* see definition of array dist_code below */ #if defined(GEN_TREES_H) || !defined(STDC) /* non ANSI compilers may not accept trees.h */ local ct_data static_ltree[L_CODES+2]; /* The static literal tree. Since the bit lengths are imposed, there is no * need for the L_CODES extra codes used during heap construction. However * The codes 286 and 287 are needed to build a canonical tree (see _tr_init * below). */ local ct_data static_dtree[D_CODES]; /* The static distance tree. (Actually a trivial tree since all codes use * 5 bits.) */ uch _dist_code[DIST_CODE_LEN]; /* Distance codes. The first 256 values correspond to the distances * 3 .. 258, the last 256 values correspond to the top 8 bits of * the 15 bit distances. */ uch _length_code[MAX_MATCH-MIN_MATCH+1]; /* length code for each normalized match length (0 == MIN_MATCH) */ local int base_length[LENGTH_CODES]; /* First normalized length for each code (0 = MIN_MATCH) */ local int base_dist[D_CODES]; /* First normalized distance for each code (0 = distance of 1) */ #else # include "trees.h" #endif /* GEN_TREES_H */ struct static_tree_desc_s { const ct_data *static_tree; /* static tree or NULL */ const intf *extra_bits; /* extra bits for each code or NULL */ int extra_base; /* base index for extra_bits */ int elems; /* max number of elements in the tree */ int max_length; /* max bit length for the codes */ }; local static_tree_desc static_l_desc = {static_ltree, extra_lbits, LITERALS+1, L_CODES, MAX_BITS}; local static_tree_desc static_d_desc = {static_dtree, extra_dbits, 0, D_CODES, MAX_BITS}; local static_tree_desc static_bl_desc = {(const ct_data *)0, extra_blbits, 0, BL_CODES, MAX_BL_BITS}; /* =========================================================================== * Local (static) routines in this file. */ local void tr_static_init OF((void)); local void init_block OF((deflate_state *s)); local void pqdownheap OF((deflate_state *s, ct_data *tree, int k)); local void gen_bitlen OF((deflate_state *s, tree_desc *desc)); local void gen_codes OF((ct_data *tree, int max_code, ushf *bl_count)); local void build_tree OF((deflate_state *s, tree_desc *desc)); local void scan_tree OF((deflate_state *s, ct_data *tree, int max_code)); local void send_tree OF((deflate_state *s, ct_data *tree, int max_code)); local int build_bl_tree OF((deflate_state *s)); local void send_all_trees OF((deflate_state *s, int lcodes, int dcodes, int blcodes)); local void compress_block OF((deflate_state *s, const ct_data *ltree, const ct_data *dtree)); local int detect_data_type OF((deflate_state *s)); local unsigned bi_reverse OF((unsigned value, int length)); local void bi_windup OF((deflate_state *s)); local void bi_flush OF((deflate_state *s)); local void copy_block OF((deflate_state *s, charf *buf, unsigned len, int header)); #ifdef GEN_TREES_H local void gen_trees_header OF((void)); #endif #ifndef DEBUG # define send_code(s, c, tree) send_bits(s, tree[c].Code, tree[c].Len) /* Send a code of the given tree. c and tree must not have side effects */ #else /* DEBUG */ # define send_code(s, c, tree) \ { if (z_verbose>2) fprintf(stderr,"\ncd %3d ",(c)); \ send_bits(s, tree[c].Code, tree[c].Len); } #endif /* =========================================================================== * Output a short LSB first on the stream. * IN assertion: there is enough room in pendingBuf. */ #define put_short(s, w) { \ put_byte(s, (uch)((w) & 0xff)); \ put_byte(s, (uch)((ush)(w) >> 8)); \ } /* =========================================================================== * Send a value on a given number of bits. * IN assertion: length <= 16 and value fits in length bits. */ #ifdef DEBUG local void send_bits OF((deflate_state *s, int value, int length)); local void send_bits(s, value, length) deflate_state *s; int value; /* value to send */ int length; /* number of bits */ { Tracevv((stderr," l %2d v %4x ", length, value)); Assert(length > 0 && length <= 15, "invalid length"); s->bits_sent += (ulg)length; /* If not enough room in bi_buf, use (valid) bits from bi_buf and * (16 - bi_valid) bits from value, leaving (width - (16-bi_valid)) * unused bits in value. */ if (s->bi_valid > (int)Buf_size - length) { s->bi_buf |= (ush)value << s->bi_valid; put_short(s, s->bi_buf); s->bi_buf = (ush)value >> (Buf_size - s->bi_valid); s->bi_valid += length - Buf_size; } else { s->bi_buf |= (ush)value << s->bi_valid; s->bi_valid += length; } } #else /* !DEBUG */ #define send_bits(s, value, length) \ { int len = length;\ if (s->bi_valid > (int)Buf_size - len) {\ int val = value;\ s->bi_buf |= (ush)val << s->bi_valid;\ put_short(s, s->bi_buf);\ s->bi_buf = (ush)val >> (Buf_size - s->bi_valid);\ s->bi_valid += len - Buf_size;\ } else {\ s->bi_buf |= (ush)(value) << s->bi_valid;\ s->bi_valid += len;\ }\ } #endif /* DEBUG */ /* the arguments must not have side effects */ /* =========================================================================== * Initialize the various 'constant' tables. */ local void tr_static_init() { #if defined(GEN_TREES_H) || !defined(STDC) static int static_init_done = 0; int n; /* iterates over tree elements */ int bits; /* bit counter */ int length; /* length value */ int code; /* code value */ int dist; /* distance index */ ush bl_count[MAX_BITS+1]; /* number of codes at each bit length for an optimal tree */ if (static_init_done) return; /* For some embedded targets, global variables are not initialized: */ #ifdef NO_INIT_GLOBAL_POINTERS static_l_desc.static_tree = static_ltree; static_l_desc.extra_bits = extra_lbits; static_d_desc.static_tree = static_dtree; static_d_desc.extra_bits = extra_dbits; static_bl_desc.extra_bits = extra_blbits; #endif /* Initialize the mapping length (0..255) -> length code (0..28) */ length = 0; for (code = 0; code < LENGTH_CODES-1; code++) { base_length[code] = length; for (n = 0; n < (1< dist code (0..29) */ dist = 0; for (code = 0 ; code < 16; code++) { base_dist[code] = dist; for (n = 0; n < (1<>= 7; /* from now on, all distances are divided by 128 */ for ( ; code < D_CODES; code++) { base_dist[code] = dist << 7; for (n = 0; n < (1<<(extra_dbits[code]-7)); n++) { _dist_code[256 + dist++] = (uch)code; } } Assert (dist == 256, "tr_static_init: 256+dist != 512"); /* Construct the codes of the static literal tree */ for (bits = 0; bits <= MAX_BITS; bits++) bl_count[bits] = 0; n = 0; while (n <= 143) static_ltree[n++].Len = 8, bl_count[8]++; while (n <= 255) static_ltree[n++].Len = 9, bl_count[9]++; while (n <= 279) static_ltree[n++].Len = 7, bl_count[7]++; while (n <= 287) static_ltree[n++].Len = 8, bl_count[8]++; /* Codes 286 and 287 do not exist, but we must include them in the * tree construction to get a canonical Huffman tree (longest code * all ones) */ gen_codes((ct_data *)static_ltree, L_CODES+1, bl_count); /* The static distance tree is trivial: */ for (n = 0; n < D_CODES; n++) { static_dtree[n].Len = 5; static_dtree[n].Code = bi_reverse((unsigned)n, 5); } static_init_done = 1; # ifdef GEN_TREES_H gen_trees_header(); # endif #endif /* defined(GEN_TREES_H) || !defined(STDC) */ } /* =========================================================================== * Genererate the file trees.h describing the static trees. */ #ifdef GEN_TREES_H # ifndef DEBUG # include # endif # define SEPARATOR(i, last, width) \ ((i) == (last)? "\n};\n\n" : \ ((i) % (width) == (width)-1 ? ",\n" : ", ")) void gen_trees_header() { FILE *header = fopen("trees.h", "w"); int i; Assert (header != NULL, "Can't open trees.h"); fprintf(header, "/* header created automatically with -DGEN_TREES_H */\n\n"); fprintf(header, "local const ct_data static_ltree[L_CODES+2] = {\n"); for (i = 0; i < L_CODES+2; i++) { fprintf(header, "{{%3u},{%3u}}%s", static_ltree[i].Code, static_ltree[i].Len, SEPARATOR(i, L_CODES+1, 5)); } fprintf(header, "local const ct_data static_dtree[D_CODES] = {\n"); for (i = 0; i < D_CODES; i++) { fprintf(header, "{{%2u},{%2u}}%s", static_dtree[i].Code, static_dtree[i].Len, SEPARATOR(i, D_CODES-1, 5)); } fprintf(header, "const uch ZLIB_INTERNAL _dist_code[DIST_CODE_LEN] = {\n"); for (i = 0; i < DIST_CODE_LEN; i++) { fprintf(header, "%2u%s", _dist_code[i], SEPARATOR(i, DIST_CODE_LEN-1, 20)); } fprintf(header, "const uch ZLIB_INTERNAL _length_code[MAX_MATCH-MIN_MATCH+1]= {\n"); for (i = 0; i < MAX_MATCH-MIN_MATCH+1; i++) { fprintf(header, "%2u%s", _length_code[i], SEPARATOR(i, MAX_MATCH-MIN_MATCH, 20)); } fprintf(header, "local const int base_length[LENGTH_CODES] = {\n"); for (i = 0; i < LENGTH_CODES; i++) { fprintf(header, "%1u%s", base_length[i], SEPARATOR(i, LENGTH_CODES-1, 20)); } fprintf(header, "local const int base_dist[D_CODES] = {\n"); for (i = 0; i < D_CODES; i++) { fprintf(header, "%5u%s", base_dist[i], SEPARATOR(i, D_CODES-1, 10)); } fclose(header); } #endif /* GEN_TREES_H */ /* =========================================================================== * Initialize the tree data structures for a new zlib stream. */ void ZLIB_INTERNAL _tr_init(s) deflate_state *s; { tr_static_init(); s->l_desc.dyn_tree = s->dyn_ltree; s->l_desc.stat_desc = &static_l_desc; s->d_desc.dyn_tree = s->dyn_dtree; s->d_desc.stat_desc = &static_d_desc; s->bl_desc.dyn_tree = s->bl_tree; s->bl_desc.stat_desc = &static_bl_desc; s->bi_buf = 0; s->bi_valid = 0; #ifdef DEBUG s->compressed_len = 0L; s->bits_sent = 0L; #endif /* Initialize the first block of the first file: */ init_block(s); } /* =========================================================================== * Initialize a new block. */ local void init_block(s) deflate_state *s; { int n; /* iterates over tree elements */ /* Initialize the trees. */ for (n = 0; n < L_CODES; n++) s->dyn_ltree[n].Freq = 0; for (n = 0; n < D_CODES; n++) s->dyn_dtree[n].Freq = 0; for (n = 0; n < BL_CODES; n++) s->bl_tree[n].Freq = 0; s->dyn_ltree[END_BLOCK].Freq = 1; s->opt_len = s->static_len = 0L; s->last_lit = s->matches = 0; } #define SMALLEST 1 /* Index within the heap array of least frequent node in the Huffman tree */ /* =========================================================================== * Remove the smallest element from the heap and recreate the heap with * one less element. Updates heap and heap_len. */ #define pqremove(s, tree, top) \ {\ top = s->heap[SMALLEST]; \ s->heap[SMALLEST] = s->heap[s->heap_len--]; \ pqdownheap(s, tree, SMALLEST); \ } /* =========================================================================== * Compares to subtrees, using the tree depth as tie breaker when * the subtrees have equal frequency. This minimizes the worst case length. */ #define smaller(tree, n, m, depth) \ (tree[n].Freq < tree[m].Freq || \ (tree[n].Freq == tree[m].Freq && depth[n] <= depth[m])) /* =========================================================================== * Restore the heap property by moving down the tree starting at node k, * exchanging a node with the smallest of its two sons if necessary, stopping * when the heap property is re-established (each father smaller than its * two sons). */ local void pqdownheap(s, tree, k) deflate_state *s; ct_data *tree; /* the tree to restore */ int k; /* node to move down */ { int v = s->heap[k]; int j = k << 1; /* left son of k */ while (j <= s->heap_len) { /* Set j to the smallest of the two sons: */ if (j < s->heap_len && smaller(tree, s->heap[j+1], s->heap[j], s->depth)) { j++; } /* Exit if v is smaller than both sons */ if (smaller(tree, v, s->heap[j], s->depth)) break; /* Exchange v with the smallest son */ s->heap[k] = s->heap[j]; k = j; /* And continue down the tree, setting j to the left son of k */ j <<= 1; } s->heap[k] = v; } /* =========================================================================== * Compute the optimal bit lengths for a tree and update the total bit length * for the current block. * IN assertion: the fields freq and dad are set, heap[heap_max] and * above are the tree nodes sorted by increasing frequency. * OUT assertions: the field len is set to the optimal bit length, the * array bl_count contains the frequencies for each bit length. * The length opt_len is updated; static_len is also updated if stree is * not null. */ local void gen_bitlen(s, desc) deflate_state *s; tree_desc *desc; /* the tree descriptor */ { ct_data *tree = desc->dyn_tree; int max_code = desc->max_code; const ct_data *stree = desc->stat_desc->static_tree; const intf *extra = desc->stat_desc->extra_bits; int base = desc->stat_desc->extra_base; int max_length = desc->stat_desc->max_length; int h; /* heap index */ int n, m; /* iterate over the tree elements */ int bits; /* bit length */ int xbits; /* extra bits */ ush f; /* frequency */ int overflow = 0; /* number of elements with bit length too large */ for (bits = 0; bits <= MAX_BITS; bits++) s->bl_count[bits] = 0; /* In a first pass, compute the optimal bit lengths (which may * overflow in the case of the bit length tree). */ tree[s->heap[s->heap_max]].Len = 0; /* root of the heap */ for (h = s->heap_max+1; h < HEAP_SIZE; h++) { n = s->heap[h]; bits = tree[tree[n].Dad].Len + 1; if (bits > max_length) bits = max_length, overflow++; tree[n].Len = (ush)bits; /* We overwrite tree[n].Dad which is no longer needed */ if (n > max_code) continue; /* not a leaf node */ s->bl_count[bits]++; xbits = 0; if (n >= base) xbits = extra[n-base]; f = tree[n].Freq; s->opt_len += (ulg)f * (bits + xbits); if (stree) s->static_len += (ulg)f * (stree[n].Len + xbits); } if (overflow == 0) return; Trace((stderr,"\nbit length overflow\n")); /* This happens for example on obj2 and pic of the Calgary corpus */ /* Find the first bit length which could increase: */ do { bits = max_length-1; while (s->bl_count[bits] == 0) bits--; s->bl_count[bits]--; /* move one leaf down the tree */ s->bl_count[bits+1] += 2; /* move one overflow item as its brother */ s->bl_count[max_length]--; /* The brother of the overflow item also moves one step up, * but this does not affect bl_count[max_length] */ overflow -= 2; } while (overflow > 0); /* Now recompute all bit lengths, scanning in increasing frequency. * h is still equal to HEAP_SIZE. (It is simpler to reconstruct all * lengths instead of fixing only the wrong ones. This idea is taken * from 'ar' written by Haruhiko Okumura.) */ for (bits = max_length; bits != 0; bits--) { n = s->bl_count[bits]; while (n != 0) { m = s->heap[--h]; if (m > max_code) continue; if ((unsigned) tree[m].Len != (unsigned) bits) { Trace((stderr,"code %d bits %d->%d\n", m, tree[m].Len, bits)); s->opt_len += ((long)bits - (long)tree[m].Len) *(long)tree[m].Freq; tree[m].Len = (ush)bits; } n--; } } } /* =========================================================================== * Generate the codes for a given tree and bit counts (which need not be * optimal). * IN assertion: the array bl_count contains the bit length statistics for * the given tree and the field len is set for all tree elements. * OUT assertion: the field code is set for all tree elements of non * zero code length. */ local void gen_codes (tree, max_code, bl_count) ct_data *tree; /* the tree to decorate */ int max_code; /* largest code with non zero frequency */ ushf *bl_count; /* number of codes at each bit length */ { ush next_code[MAX_BITS+1]; /* next code value for each bit length */ ush code = 0; /* running code value */ int bits; /* bit index */ int n; /* code index */ /* The distribution counts are first used to generate the code values * without bit reversal. */ for (bits = 1; bits <= MAX_BITS; bits++) { next_code[bits] = code = (code + bl_count[bits-1]) << 1; } /* Check that the bit counts in bl_count are consistent. The last code * must be all ones. */ Assert (code + bl_count[MAX_BITS]-1 == (1<dyn_tree; const ct_data *stree = desc->stat_desc->static_tree; int elems = desc->stat_desc->elems; int n, m; /* iterate over heap elements */ int max_code = -1; /* largest code with non zero frequency */ int node; /* new node being created */ /* Construct the initial heap, with least frequent element in * heap[SMALLEST]. The sons of heap[n] are heap[2*n] and heap[2*n+1]. * heap[0] is not used. */ s->heap_len = 0, s->heap_max = HEAP_SIZE; for (n = 0; n < elems; n++) { if (tree[n].Freq != 0) { s->heap[++(s->heap_len)] = max_code = n; s->depth[n] = 0; } else { tree[n].Len = 0; } } /* The pkzip format requires that at least one distance code exists, * and that at least one bit should be sent even if there is only one * possible code. So to avoid special checks later on we force at least * two codes of non zero frequency. */ while (s->heap_len < 2) { node = s->heap[++(s->heap_len)] = (max_code < 2 ? ++max_code : 0); tree[node].Freq = 1; s->depth[node] = 0; s->opt_len--; if (stree) s->static_len -= stree[node].Len; /* node is 0 or 1 so it does not have extra bits */ } desc->max_code = max_code; /* The elements heap[heap_len/2+1 .. heap_len] are leaves of the tree, * establish sub-heaps of increasing lengths: */ for (n = s->heap_len/2; n >= 1; n--) pqdownheap(s, tree, n); /* Construct the Huffman tree by repeatedly combining the least two * frequent nodes. */ node = elems; /* next internal node of the tree */ do { pqremove(s, tree, n); /* n = node of least frequency */ m = s->heap[SMALLEST]; /* m = node of next least frequency */ s->heap[--(s->heap_max)] = n; /* keep the nodes sorted by frequency */ s->heap[--(s->heap_max)] = m; /* Create a new node father of n and m */ tree[node].Freq = tree[n].Freq + tree[m].Freq; s->depth[node] = (uch)((s->depth[n] >= s->depth[m] ? s->depth[n] : s->depth[m]) + 1); tree[n].Dad = tree[m].Dad = (ush)node; #ifdef DUMP_BL_TREE if (tree == s->bl_tree) { fprintf(stderr,"\nnode %d(%d), sons %d(%d) %d(%d)", node, tree[node].Freq, n, tree[n].Freq, m, tree[m].Freq); } #endif /* and insert the new node in the heap */ s->heap[SMALLEST] = node++; pqdownheap(s, tree, SMALLEST); } while (s->heap_len >= 2); s->heap[--(s->heap_max)] = s->heap[SMALLEST]; /* At this point, the fields freq and dad are set. We can now * generate the bit lengths. */ gen_bitlen(s, (tree_desc *)desc); /* The field len is now set, we can generate the bit codes */ gen_codes ((ct_data *)tree, max_code, s->bl_count); } /* =========================================================================== * Scan a literal or distance tree to determine the frequencies of the codes * in the bit length tree. */ local void scan_tree (s, tree, max_code) deflate_state *s; ct_data *tree; /* the tree to be scanned */ int max_code; /* and its largest code of non zero frequency */ { int n; /* iterates over all tree elements */ int prevlen = -1; /* last emitted length */ int curlen; /* length of current code */ int nextlen = tree[0].Len; /* length of next code */ int count = 0; /* repeat count of the current code */ int max_count = 7; /* max repeat count */ int min_count = 4; /* min repeat count */ if (nextlen == 0) max_count = 138, min_count = 3; tree[max_code+1].Len = (ush)0xffff; /* guard */ for (n = 0; n <= max_code; n++) { curlen = nextlen; nextlen = tree[n+1].Len; if (++count < max_count && curlen == nextlen) { continue; } else if (count < min_count) { s->bl_tree[curlen].Freq += count; } else if (curlen != 0) { if (curlen != prevlen) s->bl_tree[curlen].Freq++; s->bl_tree[REP_3_6].Freq++; } else if (count <= 10) { s->bl_tree[REPZ_3_10].Freq++; } else { s->bl_tree[REPZ_11_138].Freq++; } count = 0; prevlen = curlen; if (nextlen == 0) { max_count = 138, min_count = 3; } else if (curlen == nextlen) { max_count = 6, min_count = 3; } else { max_count = 7, min_count = 4; } } } /* =========================================================================== * Send a literal or distance tree in compressed form, using the codes in * bl_tree. */ local void send_tree (s, tree, max_code) deflate_state *s; ct_data *tree; /* the tree to be scanned */ int max_code; /* and its largest code of non zero frequency */ { int n; /* iterates over all tree elements */ int prevlen = -1; /* last emitted length */ int curlen; /* length of current code */ int nextlen = tree[0].Len; /* length of next code */ int count = 0; /* repeat count of the current code */ int max_count = 7; /* max repeat count */ int min_count = 4; /* min repeat count */ /* tree[max_code+1].Len = -1; */ /* guard already set */ if (nextlen == 0) max_count = 138, min_count = 3; for (n = 0; n <= max_code; n++) { curlen = nextlen; nextlen = tree[n+1].Len; if (++count < max_count && curlen == nextlen) { continue; } else if (count < min_count) { do { send_code(s, curlen, s->bl_tree); } while (--count != 0); } else if (curlen != 0) { if (curlen != prevlen) { send_code(s, curlen, s->bl_tree); count--; } Assert(count >= 3 && count <= 6, " 3_6?"); send_code(s, REP_3_6, s->bl_tree); send_bits(s, count-3, 2); } else if (count <= 10) { send_code(s, REPZ_3_10, s->bl_tree); send_bits(s, count-3, 3); } else { send_code(s, REPZ_11_138, s->bl_tree); send_bits(s, count-11, 7); } count = 0; prevlen = curlen; if (nextlen == 0) { max_count = 138, min_count = 3; } else if (curlen == nextlen) { max_count = 6, min_count = 3; } else { max_count = 7, min_count = 4; } } } /* =========================================================================== * Construct the Huffman tree for the bit lengths and return the index in * bl_order of the last bit length code to send. */ local int build_bl_tree(s) deflate_state *s; { int max_blindex; /* index of last bit length code of non zero freq */ /* Determine the bit length frequencies for literal and distance trees */ scan_tree(s, (ct_data *)s->dyn_ltree, s->l_desc.max_code); scan_tree(s, (ct_data *)s->dyn_dtree, s->d_desc.max_code); /* Build the bit length tree: */ build_tree(s, (tree_desc *)(&(s->bl_desc))); /* opt_len now includes the length of the tree representations, except * the lengths of the bit lengths codes and the 5+5+4 bits for the counts. */ /* Determine the number of bit length codes to send. The pkzip format * requires that at least 4 bit length codes be sent. (appnote.txt says * 3 but the actual value used is 4.) */ for (max_blindex = BL_CODES-1; max_blindex >= 3; max_blindex--) { if (s->bl_tree[bl_order[max_blindex]].Len != 0) break; } /* Update opt_len to include the bit length tree and counts */ s->opt_len += 3*(max_blindex+1) + 5+5+4; Tracev((stderr, "\ndyn trees: dyn %ld, stat %ld", s->opt_len, s->static_len)); return max_blindex; } /* =========================================================================== * Send the header for a block using dynamic Huffman trees: the counts, the * lengths of the bit length codes, the literal tree and the distance tree. * IN assertion: lcodes >= 257, dcodes >= 1, blcodes >= 4. */ local void send_all_trees(s, lcodes, dcodes, blcodes) deflate_state *s; int lcodes, dcodes, blcodes; /* number of codes for each tree */ { int rank; /* index in bl_order */ Assert (lcodes >= 257 && dcodes >= 1 && blcodes >= 4, "not enough codes"); Assert (lcodes <= L_CODES && dcodes <= D_CODES && blcodes <= BL_CODES, "too many codes"); Tracev((stderr, "\nbl counts: ")); send_bits(s, lcodes-257, 5); /* not +255 as stated in appnote.txt */ send_bits(s, dcodes-1, 5); send_bits(s, blcodes-4, 4); /* not -3 as stated in appnote.txt */ for (rank = 0; rank < blcodes; rank++) { Tracev((stderr, "\nbl code %2d ", bl_order[rank])); send_bits(s, s->bl_tree[bl_order[rank]].Len, 3); } Tracev((stderr, "\nbl tree: sent %ld", s->bits_sent)); send_tree(s, (ct_data *)s->dyn_ltree, lcodes-1); /* literal tree */ Tracev((stderr, "\nlit tree: sent %ld", s->bits_sent)); send_tree(s, (ct_data *)s->dyn_dtree, dcodes-1); /* distance tree */ Tracev((stderr, "\ndist tree: sent %ld", s->bits_sent)); } /* =========================================================================== * Send a stored block */ void ZLIB_INTERNAL _tr_stored_block(s, buf, stored_len, last) deflate_state *s; charf *buf; /* input block */ ulg stored_len; /* length of input block */ int last; /* one if this is the last block for a file */ { send_bits(s, (STORED_BLOCK<<1)+last, 3); /* send block type */ #ifdef DEBUG s->compressed_len = (s->compressed_len + 3 + 7) & (ulg)~7L; s->compressed_len += (stored_len + 4) << 3; #endif copy_block(s, buf, (unsigned)stored_len, 1); /* with header */ } /* =========================================================================== * Flush the bits in the bit buffer to pending output (leaves at most 7 bits) */ void ZLIB_INTERNAL _tr_flush_bits(s) deflate_state *s; { bi_flush(s); } /* =========================================================================== * Send one empty static block to give enough lookahead for inflate. * This takes 10 bits, of which 7 may remain in the bit buffer. */ void ZLIB_INTERNAL _tr_align(s) deflate_state *s; { send_bits(s, STATIC_TREES<<1, 3); send_code(s, END_BLOCK, static_ltree); #ifdef DEBUG s->compressed_len += 10L; /* 3 for block type, 7 for EOB */ #endif bi_flush(s); } /* =========================================================================== * Determine the best encoding for the current block: dynamic trees, static * trees or store, and output the encoded block to the zip file. */ void ZLIB_INTERNAL _tr_flush_block(s, buf, stored_len, last) deflate_state *s; charf *buf; /* input block, or NULL if too old */ ulg stored_len; /* length of input block */ int last; /* one if this is the last block for a file */ { ulg opt_lenb, static_lenb; /* opt_len and static_len in bytes */ int max_blindex = 0; /* index of last bit length code of non zero freq */ /* Build the Huffman trees unless a stored block is forced */ if (s->level > 0) { /* Check if the file is binary or text */ if (s->strm->data_type == Z_UNKNOWN) s->strm->data_type = detect_data_type(s); /* Construct the literal and distance trees */ build_tree(s, (tree_desc *)(&(s->l_desc))); Tracev((stderr, "\nlit data: dyn %ld, stat %ld", s->opt_len, s->static_len)); build_tree(s, (tree_desc *)(&(s->d_desc))); Tracev((stderr, "\ndist data: dyn %ld, stat %ld", s->opt_len, s->static_len)); /* At this point, opt_len and static_len are the total bit lengths of * the compressed block data, excluding the tree representations. */ /* Build the bit length tree for the above two trees, and get the index * in bl_order of the last bit length code to send. */ max_blindex = build_bl_tree(s); /* Determine the best encoding. Compute the block lengths in bytes. */ opt_lenb = (s->opt_len+3+7)>>3; static_lenb = (s->static_len+3+7)>>3; Tracev((stderr, "\nopt %lu(%lu) stat %lu(%lu) stored %lu lit %u ", opt_lenb, s->opt_len, static_lenb, s->static_len, stored_len, s->last_lit)); if (static_lenb <= opt_lenb) opt_lenb = static_lenb; } else { Assert(buf != (char*)0, "lost buf"); opt_lenb = static_lenb = stored_len + 5; /* force a stored block */ } #ifdef FORCE_STORED if (buf != (char*)0) { /* force stored block */ #else if (stored_len+4 <= opt_lenb && buf != (char*)0) { /* 4: two words for the lengths */ #endif /* The test buf != NULL is only necessary if LIT_BUFSIZE > WSIZE. * Otherwise we can't have processed more than WSIZE input bytes since * the last block flush, because compression would have been * successful. If LIT_BUFSIZE <= WSIZE, it is never too late to * transform a block into a stored block. */ _tr_stored_block(s, buf, stored_len, last); #ifdef FORCE_STATIC } else if (static_lenb >= 0) { /* force static trees */ #else } else if (s->strategy == Z_FIXED || static_lenb == opt_lenb) { #endif send_bits(s, (STATIC_TREES<<1)+last, 3); compress_block(s, (const ct_data *)static_ltree, (const ct_data *)static_dtree); #ifdef DEBUG s->compressed_len += 3 + s->static_len; #endif } else { send_bits(s, (DYN_TREES<<1)+last, 3); send_all_trees(s, s->l_desc.max_code+1, s->d_desc.max_code+1, max_blindex+1); compress_block(s, (const ct_data *)s->dyn_ltree, (const ct_data *)s->dyn_dtree); #ifdef DEBUG s->compressed_len += 3 + s->opt_len; #endif } Assert (s->compressed_len == s->bits_sent, "bad compressed size"); /* The above check is made mod 2^32, for files larger than 512 MB * and uLong implemented on 32 bits. */ init_block(s); if (last) { bi_windup(s); #ifdef DEBUG s->compressed_len += 7; /* align on byte boundary */ #endif } Tracev((stderr,"\ncomprlen %lu(%lu) ", s->compressed_len>>3, s->compressed_len-7*last)); } /* =========================================================================== * Save the match info and tally the frequency counts. Return true if * the current block must be flushed. */ int ZLIB_INTERNAL _tr_tally (s, dist, lc) deflate_state *s; unsigned dist; /* distance of matched string */ unsigned lc; /* match length-MIN_MATCH or unmatched char (if dist==0) */ { s->d_buf[s->last_lit] = (ush)dist; s->l_buf[s->last_lit++] = (uch)lc; if (dist == 0) { /* lc is the unmatched char */ s->dyn_ltree[lc].Freq++; } else { s->matches++; /* Here, lc is the match length - MIN_MATCH */ dist--; /* dist = match distance - 1 */ Assert((ush)dist < (ush)MAX_DIST(s) && (ush)lc <= (ush)(MAX_MATCH-MIN_MATCH) && (ush)d_code(dist) < (ush)D_CODES, "_tr_tally: bad match"); s->dyn_ltree[_length_code[lc]+LITERALS+1].Freq++; s->dyn_dtree[d_code(dist)].Freq++; } #ifdef TRUNCATE_BLOCK /* Try to guess if it is profitable to stop the current block here */ if ((s->last_lit & 0x1fff) == 0 && s->level > 2) { /* Compute an upper bound for the compressed length */ ulg out_length = (ulg)s->last_lit*8L; ulg in_length = (ulg)((long)s->strstart - s->block_start); int dcode; for (dcode = 0; dcode < D_CODES; dcode++) { out_length += (ulg)s->dyn_dtree[dcode].Freq * (5L+extra_dbits[dcode]); } out_length >>= 3; Tracev((stderr,"\nlast_lit %u, in %ld, out ~%ld(%ld%%) ", s->last_lit, in_length, out_length, 100L - out_length*100L/in_length)); if (s->matches < s->last_lit/2 && out_length < in_length/2) return 1; } #endif return (s->last_lit == s->lit_bufsize-1); /* We avoid equality with lit_bufsize because of wraparound at 64K * on 16 bit machines and because stored blocks are restricted to * 64K-1 bytes. */ } /* =========================================================================== * Send the block data compressed using the given Huffman trees */ local void compress_block(s, ltree, dtree) deflate_state *s; const ct_data *ltree; /* literal tree */ const ct_data *dtree; /* distance tree */ { unsigned dist; /* distance of matched string */ int lc; /* match length or unmatched char (if dist == 0) */ unsigned lx = 0; /* running index in l_buf */ unsigned code; /* the code to send */ int extra; /* number of extra bits to send */ if (s->last_lit != 0) do { dist = s->d_buf[lx]; lc = s->l_buf[lx++]; if (dist == 0) { send_code(s, lc, ltree); /* send a literal byte */ Tracecv(isgraph(lc), (stderr," '%c' ", lc)); } else { /* Here, lc is the match length - MIN_MATCH */ code = _length_code[lc]; send_code(s, code+LITERALS+1, ltree); /* send the length code */ extra = extra_lbits[code]; if (extra != 0) { lc -= base_length[code]; send_bits(s, lc, extra); /* send the extra length bits */ } dist--; /* dist is now the match distance - 1 */ code = d_code(dist); Assert (code < D_CODES, "bad d_code"); send_code(s, code, dtree); /* send the distance code */ extra = extra_dbits[code]; if (extra != 0) { dist -= base_dist[code]; send_bits(s, dist, extra); /* send the extra distance bits */ } } /* literal or match pair ? */ /* Check that the overlay between pending_buf and d_buf+l_buf is ok: */ Assert((uInt)(s->pending) < s->lit_bufsize + 2*lx, "pendingBuf overflow"); } while (lx < s->last_lit); send_code(s, END_BLOCK, ltree); } /* =========================================================================== * Check if the data type is TEXT or BINARY, using the following algorithm: * - TEXT if the two conditions below are satisfied: * a) There are no non-portable control characters belonging to the * "black list" (0..6, 14..25, 28..31). * b) There is at least one printable character belonging to the * "white list" (9 {TAB}, 10 {LF}, 13 {CR}, 32..255). * - BINARY otherwise. * - The following partially-portable control characters form a * "gray list" that is ignored in this detection algorithm: * (7 {BEL}, 8 {BS}, 11 {VT}, 12 {FF}, 26 {SUB}, 27 {ESC}). * IN assertion: the fields Freq of dyn_ltree are set. */ local int detect_data_type(s) deflate_state *s; { /* black_mask is the bit mask of black-listed bytes * set bits 0..6, 14..25, and 28..31 * 0xf3ffc07f = binary 11110011111111111100000001111111 */ unsigned long black_mask = 0xf3ffc07fUL; int n; /* Check for non-textual ("black-listed") bytes. */ for (n = 0; n <= 31; n++, black_mask >>= 1) if ((black_mask & 1) && (s->dyn_ltree[n].Freq != 0)) return Z_BINARY; /* Check for textual ("white-listed") bytes. */ if (s->dyn_ltree[9].Freq != 0 || s->dyn_ltree[10].Freq != 0 || s->dyn_ltree[13].Freq != 0) return Z_TEXT; for (n = 32; n < LITERALS; n++) if (s->dyn_ltree[n].Freq != 0) return Z_TEXT; /* There are no "black-listed" or "white-listed" bytes: * this stream either is empty or has tolerated ("gray-listed") bytes only. */ return Z_BINARY; } /* =========================================================================== * Reverse the first len bits of a code, using straightforward code (a faster * method would use a table) * IN assertion: 1 <= len <= 15 */ local unsigned bi_reverse(code, len) unsigned code; /* the value to invert */ int len; /* its bit length */ { register unsigned res = 0; do { res |= code & 1; code >>= 1, res <<= 1; } while (--len > 0); return res >> 1; } /* =========================================================================== * Flush the bit buffer, keeping at most 7 bits in it. */ local void bi_flush(s) deflate_state *s; { if (s->bi_valid == 16) { put_short(s, s->bi_buf); s->bi_buf = 0; s->bi_valid = 0; } else if (s->bi_valid >= 8) { put_byte(s, (Byte)s->bi_buf); s->bi_buf >>= 8; s->bi_valid -= 8; } } /* =========================================================================== * Flush the bit buffer and align the output on a byte boundary */ local void bi_windup(s) deflate_state *s; { if (s->bi_valid > 8) { put_short(s, s->bi_buf); } else if (s->bi_valid > 0) { put_byte(s, (Byte)s->bi_buf); } s->bi_buf = 0; s->bi_valid = 0; #ifdef DEBUG s->bits_sent = (s->bits_sent+7) & ~7; #endif } /* =========================================================================== * Copy a stored block, storing first the length and its * one's complement if requested. */ local void copy_block(s, buf, len, header) deflate_state *s; charf *buf; /* the input data */ unsigned len; /* its length */ int header; /* true if block header must be written */ { bi_windup(s); /* align on byte boundary */ if (header) { put_short(s, (ush)len); put_short(s, (ush)~len); #ifdef DEBUG s->bits_sent += 2*16; #endif } #ifdef DEBUG s->bits_sent += (ulg)len<<3; #endif while (len--) { put_byte(s, *buf++); } } c-blosc-1.7.0/internal-complibs/zlib-1.2.8/trees.h000066400000000000000000000204301254622165100213650ustar00rootroot00000000000000/* header created automatically with -DGEN_TREES_H */ local const ct_data static_ltree[L_CODES+2] = { {{ 12},{ 8}}, {{140},{ 8}}, {{ 76},{ 8}}, {{204},{ 8}}, {{ 44},{ 8}}, {{172},{ 8}}, {{108},{ 8}}, {{236},{ 8}}, {{ 28},{ 8}}, {{156},{ 8}}, {{ 92},{ 8}}, {{220},{ 8}}, {{ 60},{ 8}}, {{188},{ 8}}, {{124},{ 8}}, {{252},{ 8}}, {{ 2},{ 8}}, {{130},{ 8}}, {{ 66},{ 8}}, {{194},{ 8}}, {{ 34},{ 8}}, {{162},{ 8}}, {{ 98},{ 8}}, {{226},{ 8}}, {{ 18},{ 8}}, {{146},{ 8}}, {{ 82},{ 8}}, {{210},{ 8}}, {{ 50},{ 8}}, {{178},{ 8}}, {{114},{ 8}}, {{242},{ 8}}, {{ 10},{ 8}}, {{138},{ 8}}, {{ 74},{ 8}}, {{202},{ 8}}, {{ 42},{ 8}}, {{170},{ 8}}, {{106},{ 8}}, {{234},{ 8}}, {{ 26},{ 8}}, {{154},{ 8}}, {{ 90},{ 8}}, {{218},{ 8}}, {{ 58},{ 8}}, {{186},{ 8}}, {{122},{ 8}}, {{250},{ 8}}, {{ 6},{ 8}}, {{134},{ 8}}, {{ 70},{ 8}}, {{198},{ 8}}, {{ 38},{ 8}}, {{166},{ 8}}, {{102},{ 8}}, {{230},{ 8}}, {{ 22},{ 8}}, {{150},{ 8}}, {{ 86},{ 8}}, {{214},{ 8}}, {{ 54},{ 8}}, {{182},{ 8}}, {{118},{ 8}}, {{246},{ 8}}, {{ 14},{ 8}}, {{142},{ 8}}, {{ 78},{ 8}}, {{206},{ 8}}, {{ 46},{ 8}}, {{174},{ 8}}, {{110},{ 8}}, {{238},{ 8}}, {{ 30},{ 8}}, {{158},{ 8}}, {{ 94},{ 8}}, {{222},{ 8}}, {{ 62},{ 8}}, {{190},{ 8}}, {{126},{ 8}}, {{254},{ 8}}, {{ 1},{ 8}}, {{129},{ 8}}, {{ 65},{ 8}}, {{193},{ 8}}, {{ 33},{ 8}}, {{161},{ 8}}, {{ 97},{ 8}}, {{225},{ 8}}, {{ 17},{ 8}}, {{145},{ 8}}, {{ 81},{ 8}}, {{209},{ 8}}, {{ 49},{ 8}}, {{177},{ 8}}, {{113},{ 8}}, {{241},{ 8}}, {{ 9},{ 8}}, {{137},{ 8}}, {{ 73},{ 8}}, {{201},{ 8}}, {{ 41},{ 8}}, {{169},{ 8}}, {{105},{ 8}}, {{233},{ 8}}, {{ 25},{ 8}}, {{153},{ 8}}, {{ 89},{ 8}}, {{217},{ 8}}, {{ 57},{ 8}}, {{185},{ 8}}, {{121},{ 8}}, {{249},{ 8}}, {{ 5},{ 8}}, {{133},{ 8}}, {{ 69},{ 8}}, {{197},{ 8}}, {{ 37},{ 8}}, {{165},{ 8}}, {{101},{ 8}}, {{229},{ 8}}, {{ 21},{ 8}}, {{149},{ 8}}, {{ 85},{ 8}}, {{213},{ 8}}, {{ 53},{ 8}}, {{181},{ 8}}, {{117},{ 8}}, {{245},{ 8}}, {{ 13},{ 8}}, {{141},{ 8}}, {{ 77},{ 8}}, {{205},{ 8}}, {{ 45},{ 8}}, {{173},{ 8}}, {{109},{ 8}}, {{237},{ 8}}, {{ 29},{ 8}}, {{157},{ 8}}, {{ 93},{ 8}}, {{221},{ 8}}, {{ 61},{ 8}}, {{189},{ 8}}, {{125},{ 8}}, {{253},{ 8}}, {{ 19},{ 9}}, {{275},{ 9}}, {{147},{ 9}}, {{403},{ 9}}, {{ 83},{ 9}}, {{339},{ 9}}, {{211},{ 9}}, {{467},{ 9}}, {{ 51},{ 9}}, {{307},{ 9}}, {{179},{ 9}}, {{435},{ 9}}, {{115},{ 9}}, {{371},{ 9}}, {{243},{ 9}}, {{499},{ 9}}, {{ 11},{ 9}}, {{267},{ 9}}, {{139},{ 9}}, {{395},{ 9}}, {{ 75},{ 9}}, {{331},{ 9}}, {{203},{ 9}}, {{459},{ 9}}, {{ 43},{ 9}}, {{299},{ 9}}, {{171},{ 9}}, {{427},{ 9}}, {{107},{ 9}}, {{363},{ 9}}, {{235},{ 9}}, {{491},{ 9}}, {{ 27},{ 9}}, {{283},{ 9}}, {{155},{ 9}}, {{411},{ 9}}, {{ 91},{ 9}}, {{347},{ 9}}, {{219},{ 9}}, {{475},{ 9}}, {{ 59},{ 9}}, {{315},{ 9}}, {{187},{ 9}}, {{443},{ 9}}, {{123},{ 9}}, {{379},{ 9}}, {{251},{ 9}}, {{507},{ 9}}, {{ 7},{ 9}}, {{263},{ 9}}, {{135},{ 9}}, {{391},{ 9}}, {{ 71},{ 9}}, {{327},{ 9}}, {{199},{ 9}}, {{455},{ 9}}, {{ 39},{ 9}}, {{295},{ 9}}, {{167},{ 9}}, {{423},{ 9}}, {{103},{ 9}}, {{359},{ 9}}, {{231},{ 9}}, {{487},{ 9}}, {{ 23},{ 9}}, {{279},{ 9}}, {{151},{ 9}}, {{407},{ 9}}, {{ 87},{ 9}}, {{343},{ 9}}, {{215},{ 9}}, {{471},{ 9}}, {{ 55},{ 9}}, {{311},{ 9}}, {{183},{ 9}}, {{439},{ 9}}, {{119},{ 9}}, {{375},{ 9}}, {{247},{ 9}}, {{503},{ 9}}, {{ 15},{ 9}}, {{271},{ 9}}, {{143},{ 9}}, {{399},{ 9}}, {{ 79},{ 9}}, {{335},{ 9}}, {{207},{ 9}}, {{463},{ 9}}, {{ 47},{ 9}}, {{303},{ 9}}, {{175},{ 9}}, {{431},{ 9}}, {{111},{ 9}}, {{367},{ 9}}, {{239},{ 9}}, {{495},{ 9}}, {{ 31},{ 9}}, {{287},{ 9}}, {{159},{ 9}}, {{415},{ 9}}, {{ 95},{ 9}}, {{351},{ 9}}, {{223},{ 9}}, {{479},{ 9}}, {{ 63},{ 9}}, {{319},{ 9}}, {{191},{ 9}}, {{447},{ 9}}, {{127},{ 9}}, {{383},{ 9}}, {{255},{ 9}}, {{511},{ 9}}, {{ 0},{ 7}}, {{ 64},{ 7}}, {{ 32},{ 7}}, {{ 96},{ 7}}, {{ 16},{ 7}}, {{ 80},{ 7}}, {{ 48},{ 7}}, {{112},{ 7}}, {{ 8},{ 7}}, {{ 72},{ 7}}, {{ 40},{ 7}}, {{104},{ 7}}, {{ 24},{ 7}}, {{ 88},{ 7}}, {{ 56},{ 7}}, {{120},{ 7}}, {{ 4},{ 7}}, {{ 68},{ 7}}, {{ 36},{ 7}}, {{100},{ 7}}, {{ 20},{ 7}}, {{ 84},{ 7}}, {{ 52},{ 7}}, {{116},{ 7}}, {{ 3},{ 8}}, {{131},{ 8}}, {{ 67},{ 8}}, {{195},{ 8}}, {{ 35},{ 8}}, {{163},{ 8}}, {{ 99},{ 8}}, {{227},{ 8}} }; local const ct_data static_dtree[D_CODES] = { {{ 0},{ 5}}, {{16},{ 5}}, {{ 8},{ 5}}, {{24},{ 5}}, {{ 4},{ 5}}, {{20},{ 5}}, {{12},{ 5}}, {{28},{ 5}}, {{ 2},{ 5}}, {{18},{ 5}}, {{10},{ 5}}, {{26},{ 5}}, {{ 6},{ 5}}, {{22},{ 5}}, {{14},{ 5}}, {{30},{ 5}}, {{ 1},{ 5}}, {{17},{ 5}}, {{ 9},{ 5}}, {{25},{ 5}}, {{ 5},{ 5}}, {{21},{ 5}}, {{13},{ 5}}, {{29},{ 5}}, {{ 3},{ 5}}, {{19},{ 5}}, {{11},{ 5}}, {{27},{ 5}}, {{ 7},{ 5}}, {{23},{ 5}} }; const uch ZLIB_INTERNAL _dist_code[DIST_CODE_LEN] = { 0, 1, 2, 3, 4, 4, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 0, 0, 16, 17, 18, 18, 19, 19, 20, 20, 20, 20, 21, 21, 21, 21, 22, 22, 22, 22, 22, 22, 22, 22, 23, 23, 23, 23, 23, 23, 23, 23, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29 }; const uch ZLIB_INTERNAL _length_code[MAX_MATCH-MIN_MATCH+1]= { 0, 1, 2, 3, 4, 5, 6, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 12, 12, 13, 13, 13, 13, 14, 14, 14, 14, 15, 15, 15, 15, 16, 16, 16, 16, 16, 16, 16, 16, 17, 17, 17, 17, 17, 17, 17, 17, 18, 18, 18, 18, 18, 18, 18, 18, 19, 19, 19, 19, 19, 19, 19, 19, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 28 }; local const int base_length[LENGTH_CODES] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 16, 20, 24, 28, 32, 40, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224, 0 }; local const int base_dist[D_CODES] = { 0, 1, 2, 3, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 192, 256, 384, 512, 768, 1024, 1536, 2048, 3072, 4096, 6144, 8192, 12288, 16384, 24576 }; c-blosc-1.7.0/internal-complibs/zlib-1.2.8/uncompr.c000066400000000000000000000037231254622165100217270ustar00rootroot00000000000000/* uncompr.c -- decompress a memory buffer * Copyright (C) 1995-2003, 2010 Jean-loup Gailly. * For conditions of distribution and use, see copyright notice in zlib.h */ /* @(#) $Id$ */ #define ZLIB_INTERNAL #include "zlib.h" /* =========================================================================== Decompresses the source buffer into the destination buffer. sourceLen is the byte length of the source buffer. Upon entry, destLen is the total size of the destination buffer, which must be large enough to hold the entire uncompressed data. (The size of the uncompressed data must have been saved previously by the compressor and transmitted to the decompressor by some mechanism outside the scope of this compression library.) Upon exit, destLen is the actual size of the compressed buffer. uncompress returns Z_OK if success, Z_MEM_ERROR if there was not enough memory, Z_BUF_ERROR if there was not enough room in the output buffer, or Z_DATA_ERROR if the input data was corrupted. */ int ZEXPORT uncompress (dest, destLen, source, sourceLen) Bytef *dest; uLongf *destLen; const Bytef *source; uLong sourceLen; { z_stream stream; int err; stream.next_in = (z_const Bytef *)source; stream.avail_in = (uInt)sourceLen; /* Check for source > 64K on 16-bit machine: */ if ((uLong)stream.avail_in != sourceLen) return Z_BUF_ERROR; stream.next_out = dest; stream.avail_out = (uInt)*destLen; if ((uLong)stream.avail_out != *destLen) return Z_BUF_ERROR; stream.zalloc = (alloc_func)0; stream.zfree = (free_func)0; err = inflateInit(&stream); if (err != Z_OK) return err; err = inflate(&stream, Z_FINISH); if (err != Z_STREAM_END) { inflateEnd(&stream); if (err == Z_NEED_DICT || (err == Z_BUF_ERROR && stream.avail_in == 0)) return Z_DATA_ERROR; return err; } *destLen = stream.total_out; err = inflateEnd(&stream); return err; } c-blosc-1.7.0/internal-complibs/zlib-1.2.8/zconf.h000066400000000000000000000362241254622165100213720ustar00rootroot00000000000000/* zconf.h -- configuration of the zlib compression library * Copyright (C) 1995-2013 Jean-loup Gailly. * For conditions of distribution and use, see copyright notice in zlib.h */ /* @(#) $Id$ */ #ifndef ZCONF_H #define ZCONF_H /* * If you *really* need a unique prefix for all types and library functions, * compile with -DZ_PREFIX. The "standard" zlib should be compiled without it. * Even better than compiling with -DZ_PREFIX would be to use configure to set * this permanently in zconf.h using "./configure --zprefix". */ #ifdef Z_PREFIX /* may be set to #if 1 by ./configure */ # define Z_PREFIX_SET /* all linked symbols */ # define _dist_code z__dist_code # define _length_code z__length_code # define _tr_align z__tr_align # define _tr_flush_bits z__tr_flush_bits # define _tr_flush_block z__tr_flush_block # define _tr_init z__tr_init # define _tr_stored_block z__tr_stored_block # define _tr_tally z__tr_tally # define adler32 z_adler32 # define adler32_combine z_adler32_combine # define adler32_combine64 z_adler32_combine64 # ifndef Z_SOLO # define compress z_compress # define compress2 z_compress2 # define compressBound z_compressBound # endif # define crc32 z_crc32 # define crc32_combine z_crc32_combine # define crc32_combine64 z_crc32_combine64 # define deflate z_deflate # define deflateBound z_deflateBound # define deflateCopy z_deflateCopy # define deflateEnd z_deflateEnd # define deflateInit2_ z_deflateInit2_ # define deflateInit_ z_deflateInit_ # define deflateParams z_deflateParams # define deflatePending z_deflatePending # define deflatePrime z_deflatePrime # define deflateReset z_deflateReset # define deflateResetKeep z_deflateResetKeep # define deflateSetDictionary z_deflateSetDictionary # define deflateSetHeader z_deflateSetHeader # define deflateTune z_deflateTune # define deflate_copyright z_deflate_copyright # define get_crc_table z_get_crc_table # ifndef Z_SOLO # define gz_error z_gz_error # define gz_intmax z_gz_intmax # define gz_strwinerror z_gz_strwinerror # define gzbuffer z_gzbuffer # define gzclearerr z_gzclearerr # define gzclose z_gzclose # define gzclose_r z_gzclose_r # define gzclose_w z_gzclose_w # define gzdirect z_gzdirect # define gzdopen z_gzdopen # define gzeof z_gzeof # define gzerror z_gzerror # define gzflush z_gzflush # define gzgetc z_gzgetc # define gzgetc_ z_gzgetc_ # define gzgets z_gzgets # define gzoffset z_gzoffset # define gzoffset64 z_gzoffset64 # define gzopen z_gzopen # define gzopen64 z_gzopen64 # ifdef _WIN32 # define gzopen_w z_gzopen_w # endif # define gzprintf z_gzprintf # define gzvprintf z_gzvprintf # define gzputc z_gzputc # define gzputs z_gzputs # define gzread z_gzread # define gzrewind z_gzrewind # define gzseek z_gzseek # define gzseek64 z_gzseek64 # define gzsetparams z_gzsetparams # define gztell z_gztell # define gztell64 z_gztell64 # define gzungetc z_gzungetc # define gzwrite z_gzwrite # endif # define inflate z_inflate # define inflateBack z_inflateBack # define inflateBackEnd z_inflateBackEnd # define inflateBackInit_ z_inflateBackInit_ # define inflateCopy z_inflateCopy # define inflateEnd z_inflateEnd # define inflateGetHeader z_inflateGetHeader # define inflateInit2_ z_inflateInit2_ # define inflateInit_ z_inflateInit_ # define inflateMark z_inflateMark # define inflatePrime z_inflatePrime # define inflateReset z_inflateReset # define inflateReset2 z_inflateReset2 # define inflateSetDictionary z_inflateSetDictionary # define inflateGetDictionary z_inflateGetDictionary # define inflateSync z_inflateSync # define inflateSyncPoint z_inflateSyncPoint # define inflateUndermine z_inflateUndermine # define inflateResetKeep z_inflateResetKeep # define inflate_copyright z_inflate_copyright # define inflate_fast z_inflate_fast # define inflate_table z_inflate_table # ifndef Z_SOLO # define uncompress z_uncompress # endif # define zError z_zError # ifndef Z_SOLO # define zcalloc z_zcalloc # define zcfree z_zcfree # endif # define zlibCompileFlags z_zlibCompileFlags # define zlibVersion z_zlibVersion /* all zlib typedefs in zlib.h and zconf.h */ # define Byte z_Byte # define Bytef z_Bytef # define alloc_func z_alloc_func # define charf z_charf # define free_func z_free_func # ifndef Z_SOLO # define gzFile z_gzFile # endif # define gz_header z_gz_header # define gz_headerp z_gz_headerp # define in_func z_in_func # define intf z_intf # define out_func z_out_func # define uInt z_uInt # define uIntf z_uIntf # define uLong z_uLong # define uLongf z_uLongf # define voidp z_voidp # define voidpc z_voidpc # define voidpf z_voidpf /* all zlib structs in zlib.h and zconf.h */ # define gz_header_s z_gz_header_s # define internal_state z_internal_state #endif #if defined(__MSDOS__) && !defined(MSDOS) # define MSDOS #endif #if (defined(OS_2) || defined(__OS2__)) && !defined(OS2) # define OS2 #endif #if defined(_WINDOWS) && !defined(WINDOWS) # define WINDOWS #endif #if defined(_WIN32) || defined(_WIN32_WCE) || defined(__WIN32__) # ifndef WIN32 # define WIN32 # endif #endif #if (defined(MSDOS) || defined(OS2) || defined(WINDOWS)) && !defined(WIN32) # if !defined(__GNUC__) && !defined(__FLAT__) && !defined(__386__) # ifndef SYS16BIT # define SYS16BIT # endif # endif #endif /* * Compile with -DMAXSEG_64K if the alloc function cannot allocate more * than 64k bytes at a time (needed on systems with 16-bit int). */ #ifdef SYS16BIT # define MAXSEG_64K #endif #ifdef MSDOS # define UNALIGNED_OK #endif #ifdef __STDC_VERSION__ # ifndef STDC # define STDC # endif # if __STDC_VERSION__ >= 199901L # ifndef STDC99 # define STDC99 # endif # endif #endif #if !defined(STDC) && (defined(__STDC__) || defined(__cplusplus)) # define STDC #endif #if !defined(STDC) && (defined(__GNUC__) || defined(__BORLANDC__)) # define STDC #endif #if !defined(STDC) && (defined(MSDOS) || defined(WINDOWS) || defined(WIN32)) # define STDC #endif #if !defined(STDC) && (defined(OS2) || defined(__HOS_AIX__)) # define STDC #endif #if defined(__OS400__) && !defined(STDC) /* iSeries (formerly AS/400). */ # define STDC #endif #ifndef STDC # ifndef const /* cannot use !defined(STDC) && !defined(const) on Mac */ # define const /* note: need a more gentle solution here */ # endif #endif #if defined(ZLIB_CONST) && !defined(z_const) # define z_const const #else # define z_const #endif /* Some Mac compilers merge all .h files incorrectly: */ #if defined(__MWERKS__)||defined(applec)||defined(THINK_C)||defined(__SC__) # define NO_DUMMY_DECL #endif /* Maximum value for memLevel in deflateInit2 */ #ifndef MAX_MEM_LEVEL # ifdef MAXSEG_64K # define MAX_MEM_LEVEL 8 # else # define MAX_MEM_LEVEL 9 # endif #endif /* Maximum value for windowBits in deflateInit2 and inflateInit2. * WARNING: reducing MAX_WBITS makes minigzip unable to extract .gz files * created by gzip. (Files created by minigzip can still be extracted by * gzip.) */ #ifndef MAX_WBITS # define MAX_WBITS 15 /* 32K LZ77 window */ #endif /* The memory requirements for deflate are (in bytes): (1 << (windowBits+2)) + (1 << (memLevel+9)) that is: 128K for windowBits=15 + 128K for memLevel = 8 (default values) plus a few kilobytes for small objects. For example, if you want to reduce the default memory requirements from 256K to 128K, compile with make CFLAGS="-O -DMAX_WBITS=14 -DMAX_MEM_LEVEL=7" Of course this will generally degrade compression (there's no free lunch). The memory requirements for inflate are (in bytes) 1 << windowBits that is, 32K for windowBits=15 (default value) plus a few kilobytes for small objects. */ /* Type declarations */ #ifndef OF /* function prototypes */ # ifdef STDC # define OF(args) args # else # define OF(args) () # endif #endif #ifndef Z_ARG /* function prototypes for stdarg */ # if defined(STDC) || defined(Z_HAVE_STDARG_H) # define Z_ARG(args) args # else # define Z_ARG(args) () # endif #endif /* The following definitions for FAR are needed only for MSDOS mixed * model programming (small or medium model with some far allocations). * This was tested only with MSC; for other MSDOS compilers you may have * to define NO_MEMCPY in zutil.h. If you don't need the mixed model, * just define FAR to be empty. */ #ifdef SYS16BIT # if defined(M_I86SM) || defined(M_I86MM) /* MSC small or medium model */ # define SMALL_MEDIUM # ifdef _MSC_VER # define FAR _far # else # define FAR far # endif # endif # if (defined(__SMALL__) || defined(__MEDIUM__)) /* Turbo C small or medium model */ # define SMALL_MEDIUM # ifdef __BORLANDC__ # define FAR _far # else # define FAR far # endif # endif #endif #if defined(WINDOWS) || defined(WIN32) /* If building or using zlib as a DLL, define ZLIB_DLL. * This is not mandatory, but it offers a little performance increase. */ # ifdef ZLIB_DLL # if defined(WIN32) && (!defined(__BORLANDC__) || (__BORLANDC__ >= 0x500)) # ifdef ZLIB_INTERNAL # define ZEXTERN extern __declspec(dllexport) # else # define ZEXTERN extern __declspec(dllimport) # endif # endif # endif /* ZLIB_DLL */ /* If building or using zlib with the WINAPI/WINAPIV calling convention, * define ZLIB_WINAPI. * Caution: the standard ZLIB1.DLL is NOT compiled using ZLIB_WINAPI. */ # ifdef ZLIB_WINAPI # ifdef FAR # undef FAR # endif # include /* No need for _export, use ZLIB.DEF instead. */ /* For complete Windows compatibility, use WINAPI, not __stdcall. */ # define ZEXPORT WINAPI # ifdef WIN32 # define ZEXPORTVA WINAPIV # else # define ZEXPORTVA FAR CDECL # endif # endif #endif #if defined (__BEOS__) # ifdef ZLIB_DLL # ifdef ZLIB_INTERNAL # define ZEXPORT __declspec(dllexport) # define ZEXPORTVA __declspec(dllexport) # else # define ZEXPORT __declspec(dllimport) # define ZEXPORTVA __declspec(dllimport) # endif # endif #endif #ifndef ZEXTERN # define ZEXTERN extern #endif #ifndef ZEXPORT # define ZEXPORT #endif #ifndef ZEXPORTVA # define ZEXPORTVA #endif #ifndef FAR # define FAR #endif #if !defined(__MACTYPES__) typedef unsigned char Byte; /* 8 bits */ #endif typedef unsigned int uInt; /* 16 bits or more */ typedef unsigned long uLong; /* 32 bits or more */ #ifdef SMALL_MEDIUM /* Borland C/C++ and some old MSC versions ignore FAR inside typedef */ # define Bytef Byte FAR #else typedef Byte FAR Bytef; #endif typedef char FAR charf; typedef int FAR intf; typedef uInt FAR uIntf; typedef uLong FAR uLongf; #ifdef STDC typedef void const *voidpc; typedef void FAR *voidpf; typedef void *voidp; #else typedef Byte const *voidpc; typedef Byte FAR *voidpf; typedef Byte *voidp; #endif #if !defined(Z_U4) && !defined(Z_SOLO) && defined(STDC) # include # if (UINT_MAX == 0xffffffffUL) # define Z_U4 unsigned # elif (ULONG_MAX == 0xffffffffUL) # define Z_U4 unsigned long # elif (USHRT_MAX == 0xffffffffUL) # define Z_U4 unsigned short # endif #endif #ifdef Z_U4 typedef Z_U4 z_crc_t; #else typedef unsigned long z_crc_t; #endif #ifdef HAVE_UNISTD_H /* may be set to #if 1 by ./configure */ # define Z_HAVE_UNISTD_H #endif #ifdef HAVE_STDARG_H /* may be set to #if 1 by ./configure */ # define Z_HAVE_STDARG_H #endif #ifdef STDC # ifndef Z_SOLO # include /* for off_t */ # endif #endif #if defined(STDC) || defined(Z_HAVE_STDARG_H) # ifndef Z_SOLO # include /* for va_list */ # endif #endif #ifdef _WIN32 # ifndef Z_SOLO # include /* for wchar_t */ # endif #endif /* a little trick to accommodate both "#define _LARGEFILE64_SOURCE" and * "#define _LARGEFILE64_SOURCE 1" as requesting 64-bit operations, (even * though the former does not conform to the LFS document), but considering * both "#undef _LARGEFILE64_SOURCE" and "#define _LARGEFILE64_SOURCE 0" as * equivalently requesting no 64-bit operations */ #if defined(_LARGEFILE64_SOURCE) && -_LARGEFILE64_SOURCE - -1 == 1 # undef _LARGEFILE64_SOURCE #endif #if defined(__WATCOMC__) && !defined(Z_HAVE_UNISTD_H) # define Z_HAVE_UNISTD_H #endif #ifndef Z_SOLO # if defined(Z_HAVE_UNISTD_H) || defined(_LARGEFILE64_SOURCE) # include /* for SEEK_*, off_t, and _LFS64_LARGEFILE */ # ifdef VMS # include /* for off_t */ # endif # ifndef z_off_t # define z_off_t off_t # endif # endif #endif #if defined(_LFS64_LARGEFILE) && _LFS64_LARGEFILE-0 # define Z_LFS64 #endif #if defined(_LARGEFILE64_SOURCE) && defined(Z_LFS64) # define Z_LARGE64 #endif #if defined(_FILE_OFFSET_BITS) && _FILE_OFFSET_BITS-0 == 64 && defined(Z_LFS64) # define Z_WANT64 #endif #if !defined(SEEK_SET) && !defined(Z_SOLO) # define SEEK_SET 0 /* Seek from beginning of file. */ # define SEEK_CUR 1 /* Seek from current position. */ # define SEEK_END 2 /* Set file pointer to EOF plus "offset" */ #endif #ifndef z_off_t # define z_off_t long #endif #if !defined(_WIN32) && defined(Z_LARGE64) # define z_off64_t off64_t #else # if defined(_WIN32) && !defined(__GNUC__) && !defined(Z_SOLO) # define z_off64_t __int64 # else # define z_off64_t z_off_t # endif #endif /* MVS linker does not support external names larger than 8 bytes */ #if defined(__MVS__) #pragma map(deflateInit_,"DEIN") #pragma map(deflateInit2_,"DEIN2") #pragma map(deflateEnd,"DEEND") #pragma map(deflateBound,"DEBND") #pragma map(inflateInit_,"ININ") #pragma map(inflateInit2_,"ININ2") #pragma map(inflateEnd,"INEND") #pragma map(inflateSync,"INSY") #pragma map(inflateSetDictionary,"INSEDI") #pragma map(compressBound,"CMBND") #pragma map(inflate_table,"INTABL") #pragma map(inflate_fast,"INFA") #pragma map(inflate_copyright,"INCOPY") #endif #endif /* ZCONF_H */ c-blosc-1.7.0/internal-complibs/zlib-1.2.8/zlib.h000066400000000000000000002535131254622165100212150ustar00rootroot00000000000000/* zlib.h -- interface of the 'zlib' general purpose compression library version 1.2.8, April 28th, 2013 Copyright (C) 1995-2013 Jean-loup Gailly and Mark Adler This software is provided 'as-is', without any express or implied warranty. In no event will the authors be held liable for any damages arising from the use of this software. Permission is granted to anyone to use this software for any purpose, including commercial applications, and to alter it and redistribute it freely, subject to the following restrictions: 1. The origin of this software must not be misrepresented; you must not claim that you wrote the original software. If you use this software in a product, an acknowledgment in the product documentation would be appreciated but is not required. 2. Altered source versions must be plainly marked as such, and must not be misrepresented as being the original software. 3. This notice may not be removed or altered from any source distribution. Jean-loup Gailly Mark Adler jloup@gzip.org madler@alumni.caltech.edu The data format used by the zlib library is described by RFCs (Request for Comments) 1950 to 1952 in the files http://tools.ietf.org/html/rfc1950 (zlib format), rfc1951 (deflate format) and rfc1952 (gzip format). */ #ifndef ZLIB_H #define ZLIB_H #include "zconf.h" #ifdef __cplusplus extern "C" { #endif #define ZLIB_VERSION "1.2.8" #define ZLIB_VERNUM 0x1280 #define ZLIB_VER_MAJOR 1 #define ZLIB_VER_MINOR 2 #define ZLIB_VER_REVISION 8 #define ZLIB_VER_SUBREVISION 0 /* The 'zlib' compression library provides in-memory compression and decompression functions, including integrity checks of the uncompressed data. This version of the library supports only one compression method (deflation) but other algorithms will be added later and will have the same stream interface. Compression can be done in a single step if the buffers are large enough, or can be done by repeated calls of the compression function. In the latter case, the application must provide more input and/or consume the output (providing more output space) before each call. The compressed data format used by default by the in-memory functions is the zlib format, which is a zlib wrapper documented in RFC 1950, wrapped around a deflate stream, which is itself documented in RFC 1951. The library also supports reading and writing files in gzip (.gz) format with an interface similar to that of stdio using the functions that start with "gz". The gzip format is different from the zlib format. gzip is a gzip wrapper, documented in RFC 1952, wrapped around a deflate stream. This library can optionally read and write gzip streams in memory as well. The zlib format was designed to be compact and fast for use in memory and on communications channels. The gzip format was designed for single- file compression on file systems, has a larger header than zlib to maintain directory information, and uses a different, slower check method than zlib. The library does not install any signal handler. The decoder checks the consistency of the compressed data, so the library should never crash even in case of corrupted input. */ typedef voidpf (*alloc_func) OF((voidpf opaque, uInt items, uInt size)); typedef void (*free_func) OF((voidpf opaque, voidpf address)); struct internal_state; typedef struct z_stream_s { z_const Bytef *next_in; /* next input byte */ uInt avail_in; /* number of bytes available at next_in */ uLong total_in; /* total number of input bytes read so far */ Bytef *next_out; /* next output byte should be put there */ uInt avail_out; /* remaining free space at next_out */ uLong total_out; /* total number of bytes output so far */ z_const char *msg; /* last error message, NULL if no error */ struct internal_state FAR *state; /* not visible by applications */ alloc_func zalloc; /* used to allocate the internal state */ free_func zfree; /* used to free the internal state */ voidpf opaque; /* private data object passed to zalloc and zfree */ int data_type; /* best guess about the data type: binary or text */ uLong adler; /* adler32 value of the uncompressed data */ uLong reserved; /* reserved for future use */ } z_stream; typedef z_stream FAR *z_streamp; /* gzip header information passed to and from zlib routines. See RFC 1952 for more details on the meanings of these fields. */ typedef struct gz_header_s { int text; /* true if compressed data believed to be text */ uLong time; /* modification time */ int xflags; /* extra flags (not used when writing a gzip file) */ int os; /* operating system */ Bytef *extra; /* pointer to extra field or Z_NULL if none */ uInt extra_len; /* extra field length (valid if extra != Z_NULL) */ uInt extra_max; /* space at extra (only when reading header) */ Bytef *name; /* pointer to zero-terminated file name or Z_NULL */ uInt name_max; /* space at name (only when reading header) */ Bytef *comment; /* pointer to zero-terminated comment or Z_NULL */ uInt comm_max; /* space at comment (only when reading header) */ int hcrc; /* true if there was or will be a header crc */ int done; /* true when done reading gzip header (not used when writing a gzip file) */ } gz_header; typedef gz_header FAR *gz_headerp; /* The application must update next_in and avail_in when avail_in has dropped to zero. It must update next_out and avail_out when avail_out has dropped to zero. The application must initialize zalloc, zfree and opaque before calling the init function. All other fields are set by the compression library and must not be updated by the application. The opaque value provided by the application will be passed as the first parameter for calls of zalloc and zfree. This can be useful for custom memory management. The compression library attaches no meaning to the opaque value. zalloc must return Z_NULL if there is not enough memory for the object. If zlib is used in a multi-threaded application, zalloc and zfree must be thread safe. On 16-bit systems, the functions zalloc and zfree must be able to allocate exactly 65536 bytes, but will not be required to allocate more than this if the symbol MAXSEG_64K is defined (see zconf.h). WARNING: On MSDOS, pointers returned by zalloc for objects of exactly 65536 bytes *must* have their offset normalized to zero. The default allocation function provided by this library ensures this (see zutil.c). To reduce memory requirements and avoid any allocation of 64K objects, at the expense of compression ratio, compile the library with -DMAX_WBITS=14 (see zconf.h). The fields total_in and total_out can be used for statistics or progress reports. After compression, total_in holds the total size of the uncompressed data and may be saved for use in the decompressor (particularly if the decompressor wants to decompress everything in a single step). */ /* constants */ #define Z_NO_FLUSH 0 #define Z_PARTIAL_FLUSH 1 #define Z_SYNC_FLUSH 2 #define Z_FULL_FLUSH 3 #define Z_FINISH 4 #define Z_BLOCK 5 #define Z_TREES 6 /* Allowed flush values; see deflate() and inflate() below for details */ #define Z_OK 0 #define Z_STREAM_END 1 #define Z_NEED_DICT 2 #define Z_ERRNO (-1) #define Z_STREAM_ERROR (-2) #define Z_DATA_ERROR (-3) #define Z_MEM_ERROR (-4) #define Z_BUF_ERROR (-5) #define Z_VERSION_ERROR (-6) /* Return codes for the compression/decompression functions. Negative values * are errors, positive values are used for special but normal events. */ #define Z_NO_COMPRESSION 0 #define Z_BEST_SPEED 1 #define Z_BEST_COMPRESSION 9 #define Z_DEFAULT_COMPRESSION (-1) /* compression levels */ #define Z_FILTERED 1 #define Z_HUFFMAN_ONLY 2 #define Z_RLE 3 #define Z_FIXED 4 #define Z_DEFAULT_STRATEGY 0 /* compression strategy; see deflateInit2() below for details */ #define Z_BINARY 0 #define Z_TEXT 1 #define Z_ASCII Z_TEXT /* for compatibility with 1.2.2 and earlier */ #define Z_UNKNOWN 2 /* Possible values of the data_type field (though see inflate()) */ #define Z_DEFLATED 8 /* The deflate compression method (the only one supported in this version) */ #define Z_NULL 0 /* for initializing zalloc, zfree, opaque */ #define zlib_version zlibVersion() /* for compatibility with versions < 1.0.2 */ /* basic functions */ ZEXTERN const char * ZEXPORT zlibVersion OF((void)); /* The application can compare zlibVersion and ZLIB_VERSION for consistency. If the first character differs, the library code actually used is not compatible with the zlib.h header file used by the application. This check is automatically made by deflateInit and inflateInit. */ /* ZEXTERN int ZEXPORT deflateInit OF((z_streamp strm, int level)); Initializes the internal stream state for compression. The fields zalloc, zfree and opaque must be initialized before by the caller. If zalloc and zfree are set to Z_NULL, deflateInit updates them to use default allocation functions. The compression level must be Z_DEFAULT_COMPRESSION, or between 0 and 9: 1 gives best speed, 9 gives best compression, 0 gives no compression at all (the input data is simply copied a block at a time). Z_DEFAULT_COMPRESSION requests a default compromise between speed and compression (currently equivalent to level 6). deflateInit returns Z_OK if success, Z_MEM_ERROR if there was not enough memory, Z_STREAM_ERROR if level is not a valid compression level, or Z_VERSION_ERROR if the zlib library version (zlib_version) is incompatible with the version assumed by the caller (ZLIB_VERSION). msg is set to null if there is no error message. deflateInit does not perform any compression: this will be done by deflate(). */ ZEXTERN int ZEXPORT deflate OF((z_streamp strm, int flush)); /* deflate compresses as much data as possible, and stops when the input buffer becomes empty or the output buffer becomes full. It may introduce some output latency (reading input without producing any output) except when forced to flush. The detailed semantics are as follows. deflate performs one or both of the following actions: - Compress more input starting at next_in and update next_in and avail_in accordingly. If not all input can be processed (because there is not enough room in the output buffer), next_in and avail_in are updated and processing will resume at this point for the next call of deflate(). - Provide more output starting at next_out and update next_out and avail_out accordingly. This action is forced if the parameter flush is non zero. Forcing flush frequently degrades the compression ratio, so this parameter should be set only when necessary (in interactive applications). Some output may be provided even if flush is not set. Before the call of deflate(), the application should ensure that at least one of the actions is possible, by providing more input and/or consuming more output, and updating avail_in or avail_out accordingly; avail_out should never be zero before the call. The application can consume the compressed output when it wants, for example when the output buffer is full (avail_out == 0), or after each call of deflate(). If deflate returns Z_OK and with zero avail_out, it must be called again after making room in the output buffer because there might be more output pending. Normally the parameter flush is set to Z_NO_FLUSH, which allows deflate to decide how much data to accumulate before producing output, in order to maximize compression. If the parameter flush is set to Z_SYNC_FLUSH, all pending output is flushed to the output buffer and the output is aligned on a byte boundary, so that the decompressor can get all input data available so far. (In particular avail_in is zero after the call if enough output space has been provided before the call.) Flushing may degrade compression for some compression algorithms and so it should be used only when necessary. This completes the current deflate block and follows it with an empty stored block that is three bits plus filler bits to the next byte, followed by four bytes (00 00 ff ff). If flush is set to Z_PARTIAL_FLUSH, all pending output is flushed to the output buffer, but the output is not aligned to a byte boundary. All of the input data so far will be available to the decompressor, as for Z_SYNC_FLUSH. This completes the current deflate block and follows it with an empty fixed codes block that is 10 bits long. This assures that enough bytes are output in order for the decompressor to finish the block before the empty fixed code block. If flush is set to Z_BLOCK, a deflate block is completed and emitted, as for Z_SYNC_FLUSH, but the output is not aligned on a byte boundary, and up to seven bits of the current block are held to be written as the next byte after the next deflate block is completed. In this case, the decompressor may not be provided enough bits at this point in order to complete decompression of the data provided so far to the compressor. It may need to wait for the next block to be emitted. This is for advanced applications that need to control the emission of deflate blocks. If flush is set to Z_FULL_FLUSH, all output is flushed as with Z_SYNC_FLUSH, and the compression state is reset so that decompression can restart from this point if previous compressed data has been damaged or if random access is desired. Using Z_FULL_FLUSH too often can seriously degrade compression. If deflate returns with avail_out == 0, this function must be called again with the same value of the flush parameter and more output space (updated avail_out), until the flush is complete (deflate returns with non-zero avail_out). In the case of a Z_FULL_FLUSH or Z_SYNC_FLUSH, make sure that avail_out is greater than six to avoid repeated flush markers due to avail_out == 0 on return. If the parameter flush is set to Z_FINISH, pending input is processed, pending output is flushed and deflate returns with Z_STREAM_END if there was enough output space; if deflate returns with Z_OK, this function must be called again with Z_FINISH and more output space (updated avail_out) but no more input data, until it returns with Z_STREAM_END or an error. After deflate has returned Z_STREAM_END, the only possible operations on the stream are deflateReset or deflateEnd. Z_FINISH can be used immediately after deflateInit if all the compression is to be done in a single step. In this case, avail_out must be at least the value returned by deflateBound (see below). Then deflate is guaranteed to return Z_STREAM_END. If not enough output space is provided, deflate will not return Z_STREAM_END, and it must be called again as described above. deflate() sets strm->adler to the adler32 checksum of all input read so far (that is, total_in bytes). deflate() may update strm->data_type if it can make a good guess about the input data type (Z_BINARY or Z_TEXT). In doubt, the data is considered binary. This field is only for information purposes and does not affect the compression algorithm in any manner. deflate() returns Z_OK if some progress has been made (more input processed or more output produced), Z_STREAM_END if all input has been consumed and all output has been produced (only when flush is set to Z_FINISH), Z_STREAM_ERROR if the stream state was inconsistent (for example if next_in or next_out was Z_NULL), Z_BUF_ERROR if no progress is possible (for example avail_in or avail_out was zero). Note that Z_BUF_ERROR is not fatal, and deflate() can be called again with more input and more output space to continue compressing. */ ZEXTERN int ZEXPORT deflateEnd OF((z_streamp strm)); /* All dynamically allocated data structures for this stream are freed. This function discards any unprocessed input and does not flush any pending output. deflateEnd returns Z_OK if success, Z_STREAM_ERROR if the stream state was inconsistent, Z_DATA_ERROR if the stream was freed prematurely (some input or output was discarded). In the error case, msg may be set but then points to a static string (which must not be deallocated). */ /* ZEXTERN int ZEXPORT inflateInit OF((z_streamp strm)); Initializes the internal stream state for decompression. The fields next_in, avail_in, zalloc, zfree and opaque must be initialized before by the caller. If next_in is not Z_NULL and avail_in is large enough (the exact value depends on the compression method), inflateInit determines the compression method from the zlib header and allocates all data structures accordingly; otherwise the allocation will be deferred to the first call of inflate. If zalloc and zfree are set to Z_NULL, inflateInit updates them to use default allocation functions. inflateInit returns Z_OK if success, Z_MEM_ERROR if there was not enough memory, Z_VERSION_ERROR if the zlib library version is incompatible with the version assumed by the caller, or Z_STREAM_ERROR if the parameters are invalid, such as a null pointer to the structure. msg is set to null if there is no error message. inflateInit does not perform any decompression apart from possibly reading the zlib header if present: actual decompression will be done by inflate(). (So next_in and avail_in may be modified, but next_out and avail_out are unused and unchanged.) The current implementation of inflateInit() does not process any header information -- that is deferred until inflate() is called. */ ZEXTERN int ZEXPORT inflate OF((z_streamp strm, int flush)); /* inflate decompresses as much data as possible, and stops when the input buffer becomes empty or the output buffer becomes full. It may introduce some output latency (reading input without producing any output) except when forced to flush. The detailed semantics are as follows. inflate performs one or both of the following actions: - Decompress more input starting at next_in and update next_in and avail_in accordingly. If not all input can be processed (because there is not enough room in the output buffer), next_in is updated and processing will resume at this point for the next call of inflate(). - Provide more output starting at next_out and update next_out and avail_out accordingly. inflate() provides as much output as possible, until there is no more input data or no more space in the output buffer (see below about the flush parameter). Before the call of inflate(), the application should ensure that at least one of the actions is possible, by providing more input and/or consuming more output, and updating the next_* and avail_* values accordingly. The application can consume the uncompressed output when it wants, for example when the output buffer is full (avail_out == 0), or after each call of inflate(). If inflate returns Z_OK and with zero avail_out, it must be called again after making room in the output buffer because there might be more output pending. The flush parameter of inflate() can be Z_NO_FLUSH, Z_SYNC_FLUSH, Z_FINISH, Z_BLOCK, or Z_TREES. Z_SYNC_FLUSH requests that inflate() flush as much output as possible to the output buffer. Z_BLOCK requests that inflate() stop if and when it gets to the next deflate block boundary. When decoding the zlib or gzip format, this will cause inflate() to return immediately after the header and before the first block. When doing a raw inflate, inflate() will go ahead and process the first block, and will return when it gets to the end of that block, or when it runs out of data. The Z_BLOCK option assists in appending to or combining deflate streams. Also to assist in this, on return inflate() will set strm->data_type to the number of unused bits in the last byte taken from strm->next_in, plus 64 if inflate() is currently decoding the last block in the deflate stream, plus 128 if inflate() returned immediately after decoding an end-of-block code or decoding the complete header up to just before the first byte of the deflate stream. The end-of-block will not be indicated until all of the uncompressed data from that block has been written to strm->next_out. The number of unused bits may in general be greater than seven, except when bit 7 of data_type is set, in which case the number of unused bits will be less than eight. data_type is set as noted here every time inflate() returns for all flush options, and so can be used to determine the amount of currently consumed input in bits. The Z_TREES option behaves as Z_BLOCK does, but it also returns when the end of each deflate block header is reached, before any actual data in that block is decoded. This allows the caller to determine the length of the deflate block header for later use in random access within a deflate block. 256 is added to the value of strm->data_type when inflate() returns immediately after reaching the end of the deflate block header. inflate() should normally be called until it returns Z_STREAM_END or an error. However if all decompression is to be performed in a single step (a single call of inflate), the parameter flush should be set to Z_FINISH. In this case all pending input is processed and all pending output is flushed; avail_out must be large enough to hold all of the uncompressed data for the operation to complete. (The size of the uncompressed data may have been saved by the compressor for this purpose.) The use of Z_FINISH is not required to perform an inflation in one step. However it may be used to inform inflate that a faster approach can be used for the single inflate() call. Z_FINISH also informs inflate to not maintain a sliding window if the stream completes, which reduces inflate's memory footprint. If the stream does not complete, either because not all of the stream is provided or not enough output space is provided, then a sliding window will be allocated and inflate() can be called again to continue the operation as if Z_NO_FLUSH had been used. In this implementation, inflate() always flushes as much output as possible to the output buffer, and always uses the faster approach on the first call. So the effects of the flush parameter in this implementation are on the return value of inflate() as noted below, when inflate() returns early when Z_BLOCK or Z_TREES is used, and when inflate() avoids the allocation of memory for a sliding window when Z_FINISH is used. If a preset dictionary is needed after this call (see inflateSetDictionary below), inflate sets strm->adler to the Adler-32 checksum of the dictionary chosen by the compressor and returns Z_NEED_DICT; otherwise it sets strm->adler to the Adler-32 checksum of all output produced so far (that is, total_out bytes) and returns Z_OK, Z_STREAM_END or an error code as described below. At the end of the stream, inflate() checks that its computed adler32 checksum is equal to that saved by the compressor and returns Z_STREAM_END only if the checksum is correct. inflate() can decompress and check either zlib-wrapped or gzip-wrapped deflate data. The header type is detected automatically, if requested when initializing with inflateInit2(). Any information contained in the gzip header is not retained, so applications that need that information should instead use raw inflate, see inflateInit2() below, or inflateBack() and perform their own processing of the gzip header and trailer. When processing gzip-wrapped deflate data, strm->adler32 is set to the CRC-32 of the output producted so far. The CRC-32 is checked against the gzip trailer. inflate() returns Z_OK if some progress has been made (more input processed or more output produced), Z_STREAM_END if the end of the compressed data has been reached and all uncompressed output has been produced, Z_NEED_DICT if a preset dictionary is needed at this point, Z_DATA_ERROR if the input data was corrupted (input stream not conforming to the zlib format or incorrect check value), Z_STREAM_ERROR if the stream structure was inconsistent (for example next_in or next_out was Z_NULL), Z_MEM_ERROR if there was not enough memory, Z_BUF_ERROR if no progress is possible or if there was not enough room in the output buffer when Z_FINISH is used. Note that Z_BUF_ERROR is not fatal, and inflate() can be called again with more input and more output space to continue decompressing. If Z_DATA_ERROR is returned, the application may then call inflateSync() to look for a good compression block if a partial recovery of the data is desired. */ ZEXTERN int ZEXPORT inflateEnd OF((z_streamp strm)); /* All dynamically allocated data structures for this stream are freed. This function discards any unprocessed input and does not flush any pending output. inflateEnd returns Z_OK if success, Z_STREAM_ERROR if the stream state was inconsistent. In the error case, msg may be set but then points to a static string (which must not be deallocated). */ /* Advanced functions */ /* The following functions are needed only in some special applications. */ /* ZEXTERN int ZEXPORT deflateInit2 OF((z_streamp strm, int level, int method, int windowBits, int memLevel, int strategy)); This is another version of deflateInit with more compression options. The fields next_in, zalloc, zfree and opaque must be initialized before by the caller. The method parameter is the compression method. It must be Z_DEFLATED in this version of the library. The windowBits parameter is the base two logarithm of the window size (the size of the history buffer). It should be in the range 8..15 for this version of the library. Larger values of this parameter result in better compression at the expense of memory usage. The default value is 15 if deflateInit is used instead. windowBits can also be -8..-15 for raw deflate. In this case, -windowBits determines the window size. deflate() will then generate raw deflate data with no zlib header or trailer, and will not compute an adler32 check value. windowBits can also be greater than 15 for optional gzip encoding. Add 16 to windowBits to write a simple gzip header and trailer around the compressed data instead of a zlib wrapper. The gzip header will have no file name, no extra data, no comment, no modification time (set to zero), no header crc, and the operating system will be set to 255 (unknown). If a gzip stream is being written, strm->adler is a crc32 instead of an adler32. The memLevel parameter specifies how much memory should be allocated for the internal compression state. memLevel=1 uses minimum memory but is slow and reduces compression ratio; memLevel=9 uses maximum memory for optimal speed. The default value is 8. See zconf.h for total memory usage as a function of windowBits and memLevel. The strategy parameter is used to tune the compression algorithm. Use the value Z_DEFAULT_STRATEGY for normal data, Z_FILTERED for data produced by a filter (or predictor), Z_HUFFMAN_ONLY to force Huffman encoding only (no string match), or Z_RLE to limit match distances to one (run-length encoding). Filtered data consists mostly of small values with a somewhat random distribution. In this case, the compression algorithm is tuned to compress them better. The effect of Z_FILTERED is to force more Huffman coding and less string matching; it is somewhat intermediate between Z_DEFAULT_STRATEGY and Z_HUFFMAN_ONLY. Z_RLE is designed to be almost as fast as Z_HUFFMAN_ONLY, but give better compression for PNG image data. The strategy parameter only affects the compression ratio but not the correctness of the compressed output even if it is not set appropriately. Z_FIXED prevents the use of dynamic Huffman codes, allowing for a simpler decoder for special applications. deflateInit2 returns Z_OK if success, Z_MEM_ERROR if there was not enough memory, Z_STREAM_ERROR if any parameter is invalid (such as an invalid method), or Z_VERSION_ERROR if the zlib library version (zlib_version) is incompatible with the version assumed by the caller (ZLIB_VERSION). msg is set to null if there is no error message. deflateInit2 does not perform any compression: this will be done by deflate(). */ ZEXTERN int ZEXPORT deflateSetDictionary OF((z_streamp strm, const Bytef *dictionary, uInt dictLength)); /* Initializes the compression dictionary from the given byte sequence without producing any compressed output. When using the zlib format, this function must be called immediately after deflateInit, deflateInit2 or deflateReset, and before any call of deflate. When doing raw deflate, this function must be called either before any call of deflate, or immediately after the completion of a deflate block, i.e. after all input has been consumed and all output has been delivered when using any of the flush options Z_BLOCK, Z_PARTIAL_FLUSH, Z_SYNC_FLUSH, or Z_FULL_FLUSH. The compressor and decompressor must use exactly the same dictionary (see inflateSetDictionary). The dictionary should consist of strings (byte sequences) that are likely to be encountered later in the data to be compressed, with the most commonly used strings preferably put towards the end of the dictionary. Using a dictionary is most useful when the data to be compressed is short and can be predicted with good accuracy; the data can then be compressed better than with the default empty dictionary. Depending on the size of the compression data structures selected by deflateInit or deflateInit2, a part of the dictionary may in effect be discarded, for example if the dictionary is larger than the window size provided in deflateInit or deflateInit2. Thus the strings most likely to be useful should be put at the end of the dictionary, not at the front. In addition, the current implementation of deflate will use at most the window size minus 262 bytes of the provided dictionary. Upon return of this function, strm->adler is set to the adler32 value of the dictionary; the decompressor may later use this value to determine which dictionary has been used by the compressor. (The adler32 value applies to the whole dictionary even if only a subset of the dictionary is actually used by the compressor.) If a raw deflate was requested, then the adler32 value is not computed and strm->adler is not set. deflateSetDictionary returns Z_OK if success, or Z_STREAM_ERROR if a parameter is invalid (e.g. dictionary being Z_NULL) or the stream state is inconsistent (for example if deflate has already been called for this stream or if not at a block boundary for raw deflate). deflateSetDictionary does not perform any compression: this will be done by deflate(). */ ZEXTERN int ZEXPORT deflateCopy OF((z_streamp dest, z_streamp source)); /* Sets the destination stream as a complete copy of the source stream. This function can be useful when several compression strategies will be tried, for example when there are several ways of pre-processing the input data with a filter. The streams that will be discarded should then be freed by calling deflateEnd. Note that deflateCopy duplicates the internal compression state which can be quite large, so this strategy is slow and can consume lots of memory. deflateCopy returns Z_OK if success, Z_MEM_ERROR if there was not enough memory, Z_STREAM_ERROR if the source stream state was inconsistent (such as zalloc being Z_NULL). msg is left unchanged in both source and destination. */ ZEXTERN int ZEXPORT deflateReset OF((z_streamp strm)); /* This function is equivalent to deflateEnd followed by deflateInit, but does not free and reallocate all the internal compression state. The stream will keep the same compression level and any other attributes that may have been set by deflateInit2. deflateReset returns Z_OK if success, or Z_STREAM_ERROR if the source stream state was inconsistent (such as zalloc or state being Z_NULL). */ ZEXTERN int ZEXPORT deflateParams OF((z_streamp strm, int level, int strategy)); /* Dynamically update the compression level and compression strategy. The interpretation of level and strategy is as in deflateInit2. This can be used to switch between compression and straight copy of the input data, or to switch to a different kind of input data requiring a different strategy. If the compression level is changed, the input available so far is compressed with the old level (and may be flushed); the new level will take effect only at the next call of deflate(). Before the call of deflateParams, the stream state must be set as for a call of deflate(), since the currently available input may have to be compressed and flushed. In particular, strm->avail_out must be non-zero. deflateParams returns Z_OK if success, Z_STREAM_ERROR if the source stream state was inconsistent or if a parameter was invalid, Z_BUF_ERROR if strm->avail_out was zero. */ ZEXTERN int ZEXPORT deflateTune OF((z_streamp strm, int good_length, int max_lazy, int nice_length, int max_chain)); /* Fine tune deflate's internal compression parameters. This should only be used by someone who understands the algorithm used by zlib's deflate for searching for the best matching string, and even then only by the most fanatic optimizer trying to squeeze out the last compressed bit for their specific input data. Read the deflate.c source code for the meaning of the max_lazy, good_length, nice_length, and max_chain parameters. deflateTune() can be called after deflateInit() or deflateInit2(), and returns Z_OK on success, or Z_STREAM_ERROR for an invalid deflate stream. */ ZEXTERN uLong ZEXPORT deflateBound OF((z_streamp strm, uLong sourceLen)); /* deflateBound() returns an upper bound on the compressed size after deflation of sourceLen bytes. It must be called after deflateInit() or deflateInit2(), and after deflateSetHeader(), if used. This would be used to allocate an output buffer for deflation in a single pass, and so would be called before deflate(). If that first deflate() call is provided the sourceLen input bytes, an output buffer allocated to the size returned by deflateBound(), and the flush value Z_FINISH, then deflate() is guaranteed to return Z_STREAM_END. Note that it is possible for the compressed size to be larger than the value returned by deflateBound() if flush options other than Z_FINISH or Z_NO_FLUSH are used. */ ZEXTERN int ZEXPORT deflatePending OF((z_streamp strm, unsigned *pending, int *bits)); /* deflatePending() returns the number of bytes and bits of output that have been generated, but not yet provided in the available output. The bytes not provided would be due to the available output space having being consumed. The number of bits of output not provided are between 0 and 7, where they await more bits to join them in order to fill out a full byte. If pending or bits are Z_NULL, then those values are not set. deflatePending returns Z_OK if success, or Z_STREAM_ERROR if the source stream state was inconsistent. */ ZEXTERN int ZEXPORT deflatePrime OF((z_streamp strm, int bits, int value)); /* deflatePrime() inserts bits in the deflate output stream. The intent is that this function is used to start off the deflate output with the bits leftover from a previous deflate stream when appending to it. As such, this function can only be used for raw deflate, and must be used before the first deflate() call after a deflateInit2() or deflateReset(). bits must be less than or equal to 16, and that many of the least significant bits of value will be inserted in the output. deflatePrime returns Z_OK if success, Z_BUF_ERROR if there was not enough room in the internal buffer to insert the bits, or Z_STREAM_ERROR if the source stream state was inconsistent. */ ZEXTERN int ZEXPORT deflateSetHeader OF((z_streamp strm, gz_headerp head)); /* deflateSetHeader() provides gzip header information for when a gzip stream is requested by deflateInit2(). deflateSetHeader() may be called after deflateInit2() or deflateReset() and before the first call of deflate(). The text, time, os, extra field, name, and comment information in the provided gz_header structure are written to the gzip header (xflag is ignored -- the extra flags are set according to the compression level). The caller must assure that, if not Z_NULL, name and comment are terminated with a zero byte, and that if extra is not Z_NULL, that extra_len bytes are available there. If hcrc is true, a gzip header crc is included. Note that the current versions of the command-line version of gzip (up through version 1.3.x) do not support header crc's, and will report that it is a "multi-part gzip file" and give up. If deflateSetHeader is not used, the default gzip header has text false, the time set to zero, and os set to 255, with no extra, name, or comment fields. The gzip header is returned to the default state by deflateReset(). deflateSetHeader returns Z_OK if success, or Z_STREAM_ERROR if the source stream state was inconsistent. */ /* ZEXTERN int ZEXPORT inflateInit2 OF((z_streamp strm, int windowBits)); This is another version of inflateInit with an extra parameter. The fields next_in, avail_in, zalloc, zfree and opaque must be initialized before by the caller. The windowBits parameter is the base two logarithm of the maximum window size (the size of the history buffer). It should be in the range 8..15 for this version of the library. The default value is 15 if inflateInit is used instead. windowBits must be greater than or equal to the windowBits value provided to deflateInit2() while compressing, or it must be equal to 15 if deflateInit2() was not used. If a compressed stream with a larger window size is given as input, inflate() will return with the error code Z_DATA_ERROR instead of trying to allocate a larger window. windowBits can also be zero to request that inflate use the window size in the zlib header of the compressed stream. windowBits can also be -8..-15 for raw inflate. In this case, -windowBits determines the window size. inflate() will then process raw deflate data, not looking for a zlib or gzip header, not generating a check value, and not looking for any check values for comparison at the end of the stream. This is for use with other formats that use the deflate compressed data format such as zip. Those formats provide their own check values. If a custom format is developed using the raw deflate format for compressed data, it is recommended that a check value such as an adler32 or a crc32 be applied to the uncompressed data as is done in the zlib, gzip, and zip formats. For most applications, the zlib format should be used as is. Note that comments above on the use in deflateInit2() applies to the magnitude of windowBits. windowBits can also be greater than 15 for optional gzip decoding. Add 32 to windowBits to enable zlib and gzip decoding with automatic header detection, or add 16 to decode only the gzip format (the zlib format will return a Z_DATA_ERROR). If a gzip stream is being decoded, strm->adler is a crc32 instead of an adler32. inflateInit2 returns Z_OK if success, Z_MEM_ERROR if there was not enough memory, Z_VERSION_ERROR if the zlib library version is incompatible with the version assumed by the caller, or Z_STREAM_ERROR if the parameters are invalid, such as a null pointer to the structure. msg is set to null if there is no error message. inflateInit2 does not perform any decompression apart from possibly reading the zlib header if present: actual decompression will be done by inflate(). (So next_in and avail_in may be modified, but next_out and avail_out are unused and unchanged.) The current implementation of inflateInit2() does not process any header information -- that is deferred until inflate() is called. */ ZEXTERN int ZEXPORT inflateSetDictionary OF((z_streamp strm, const Bytef *dictionary, uInt dictLength)); /* Initializes the decompression dictionary from the given uncompressed byte sequence. This function must be called immediately after a call of inflate, if that call returned Z_NEED_DICT. The dictionary chosen by the compressor can be determined from the adler32 value returned by that call of inflate. The compressor and decompressor must use exactly the same dictionary (see deflateSetDictionary). For raw inflate, this function can be called at any time to set the dictionary. If the provided dictionary is smaller than the window and there is already data in the window, then the provided dictionary will amend what's there. The application must insure that the dictionary that was used for compression is provided. inflateSetDictionary returns Z_OK if success, Z_STREAM_ERROR if a parameter is invalid (e.g. dictionary being Z_NULL) or the stream state is inconsistent, Z_DATA_ERROR if the given dictionary doesn't match the expected one (incorrect adler32 value). inflateSetDictionary does not perform any decompression: this will be done by subsequent calls of inflate(). */ ZEXTERN int ZEXPORT inflateGetDictionary OF((z_streamp strm, Bytef *dictionary, uInt *dictLength)); /* Returns the sliding dictionary being maintained by inflate. dictLength is set to the number of bytes in the dictionary, and that many bytes are copied to dictionary. dictionary must have enough space, where 32768 bytes is always enough. If inflateGetDictionary() is called with dictionary equal to Z_NULL, then only the dictionary length is returned, and nothing is copied. Similary, if dictLength is Z_NULL, then it is not set. inflateGetDictionary returns Z_OK on success, or Z_STREAM_ERROR if the stream state is inconsistent. */ ZEXTERN int ZEXPORT inflateSync OF((z_streamp strm)); /* Skips invalid compressed data until a possible full flush point (see above for the description of deflate with Z_FULL_FLUSH) can be found, or until all available input is skipped. No output is provided. inflateSync searches for a 00 00 FF FF pattern in the compressed data. All full flush points have this pattern, but not all occurrences of this pattern are full flush points. inflateSync returns Z_OK if a possible full flush point has been found, Z_BUF_ERROR if no more input was provided, Z_DATA_ERROR if no flush point has been found, or Z_STREAM_ERROR if the stream structure was inconsistent. In the success case, the application may save the current current value of total_in which indicates where valid compressed data was found. In the error case, the application may repeatedly call inflateSync, providing more input each time, until success or end of the input data. */ ZEXTERN int ZEXPORT inflateCopy OF((z_streamp dest, z_streamp source)); /* Sets the destination stream as a complete copy of the source stream. This function can be useful when randomly accessing a large stream. The first pass through the stream can periodically record the inflate state, allowing restarting inflate at those points when randomly accessing the stream. inflateCopy returns Z_OK if success, Z_MEM_ERROR if there was not enough memory, Z_STREAM_ERROR if the source stream state was inconsistent (such as zalloc being Z_NULL). msg is left unchanged in both source and destination. */ ZEXTERN int ZEXPORT inflateReset OF((z_streamp strm)); /* This function is equivalent to inflateEnd followed by inflateInit, but does not free and reallocate all the internal decompression state. The stream will keep attributes that may have been set by inflateInit2. inflateReset returns Z_OK if success, or Z_STREAM_ERROR if the source stream state was inconsistent (such as zalloc or state being Z_NULL). */ ZEXTERN int ZEXPORT inflateReset2 OF((z_streamp strm, int windowBits)); /* This function is the same as inflateReset, but it also permits changing the wrap and window size requests. The windowBits parameter is interpreted the same as it is for inflateInit2. inflateReset2 returns Z_OK if success, or Z_STREAM_ERROR if the source stream state was inconsistent (such as zalloc or state being Z_NULL), or if the windowBits parameter is invalid. */ ZEXTERN int ZEXPORT inflatePrime OF((z_streamp strm, int bits, int value)); /* This function inserts bits in the inflate input stream. The intent is that this function is used to start inflating at a bit position in the middle of a byte. The provided bits will be used before any bytes are used from next_in. This function should only be used with raw inflate, and should be used before the first inflate() call after inflateInit2() or inflateReset(). bits must be less than or equal to 16, and that many of the least significant bits of value will be inserted in the input. If bits is negative, then the input stream bit buffer is emptied. Then inflatePrime() can be called again to put bits in the buffer. This is used to clear out bits leftover after feeding inflate a block description prior to feeding inflate codes. inflatePrime returns Z_OK if success, or Z_STREAM_ERROR if the source stream state was inconsistent. */ ZEXTERN long ZEXPORT inflateMark OF((z_streamp strm)); /* This function returns two values, one in the lower 16 bits of the return value, and the other in the remaining upper bits, obtained by shifting the return value down 16 bits. If the upper value is -1 and the lower value is zero, then inflate() is currently decoding information outside of a block. If the upper value is -1 and the lower value is non-zero, then inflate is in the middle of a stored block, with the lower value equaling the number of bytes from the input remaining to copy. If the upper value is not -1, then it is the number of bits back from the current bit position in the input of the code (literal or length/distance pair) currently being processed. In that case the lower value is the number of bytes already emitted for that code. A code is being processed if inflate is waiting for more input to complete decoding of the code, or if it has completed decoding but is waiting for more output space to write the literal or match data. inflateMark() is used to mark locations in the input data for random access, which may be at bit positions, and to note those cases where the output of a code may span boundaries of random access blocks. The current location in the input stream can be determined from avail_in and data_type as noted in the description for the Z_BLOCK flush parameter for inflate. inflateMark returns the value noted above or -1 << 16 if the provided source stream state was inconsistent. */ ZEXTERN int ZEXPORT inflateGetHeader OF((z_streamp strm, gz_headerp head)); /* inflateGetHeader() requests that gzip header information be stored in the provided gz_header structure. inflateGetHeader() may be called after inflateInit2() or inflateReset(), and before the first call of inflate(). As inflate() processes the gzip stream, head->done is zero until the header is completed, at which time head->done is set to one. If a zlib stream is being decoded, then head->done is set to -1 to indicate that there will be no gzip header information forthcoming. Note that Z_BLOCK or Z_TREES can be used to force inflate() to return immediately after header processing is complete and before any actual data is decompressed. The text, time, xflags, and os fields are filled in with the gzip header contents. hcrc is set to true if there is a header CRC. (The header CRC was valid if done is set to one.) If extra is not Z_NULL, then extra_max contains the maximum number of bytes to write to extra. Once done is true, extra_len contains the actual extra field length, and extra contains the extra field, or that field truncated if extra_max is less than extra_len. If name is not Z_NULL, then up to name_max characters are written there, terminated with a zero unless the length is greater than name_max. If comment is not Z_NULL, then up to comm_max characters are written there, terminated with a zero unless the length is greater than comm_max. When any of extra, name, or comment are not Z_NULL and the respective field is not present in the header, then that field is set to Z_NULL to signal its absence. This allows the use of deflateSetHeader() with the returned structure to duplicate the header. However if those fields are set to allocated memory, then the application will need to save those pointers elsewhere so that they can be eventually freed. If inflateGetHeader is not used, then the header information is simply discarded. The header is always checked for validity, including the header CRC if present. inflateReset() will reset the process to discard the header information. The application would need to call inflateGetHeader() again to retrieve the header from the next gzip stream. inflateGetHeader returns Z_OK if success, or Z_STREAM_ERROR if the source stream state was inconsistent. */ /* ZEXTERN int ZEXPORT inflateBackInit OF((z_streamp strm, int windowBits, unsigned char FAR *window)); Initialize the internal stream state for decompression using inflateBack() calls. The fields zalloc, zfree and opaque in strm must be initialized before the call. If zalloc and zfree are Z_NULL, then the default library- derived memory allocation routines are used. windowBits is the base two logarithm of the window size, in the range 8..15. window is a caller supplied buffer of that size. Except for special applications where it is assured that deflate was used with small window sizes, windowBits must be 15 and a 32K byte window must be supplied to be able to decompress general deflate streams. See inflateBack() for the usage of these routines. inflateBackInit will return Z_OK on success, Z_STREAM_ERROR if any of the parameters are invalid, Z_MEM_ERROR if the internal state could not be allocated, or Z_VERSION_ERROR if the version of the library does not match the version of the header file. */ typedef unsigned (*in_func) OF((void FAR *, z_const unsigned char FAR * FAR *)); typedef int (*out_func) OF((void FAR *, unsigned char FAR *, unsigned)); ZEXTERN int ZEXPORT inflateBack OF((z_streamp strm, in_func in, void FAR *in_desc, out_func out, void FAR *out_desc)); /* inflateBack() does a raw inflate with a single call using a call-back interface for input and output. This is potentially more efficient than inflate() for file i/o applications, in that it avoids copying between the output and the sliding window by simply making the window itself the output buffer. inflate() can be faster on modern CPUs when used with large buffers. inflateBack() trusts the application to not change the output buffer passed by the output function, at least until inflateBack() returns. inflateBackInit() must be called first to allocate the internal state and to initialize the state with the user-provided window buffer. inflateBack() may then be used multiple times to inflate a complete, raw deflate stream with each call. inflateBackEnd() is then called to free the allocated state. A raw deflate stream is one with no zlib or gzip header or trailer. This routine would normally be used in a utility that reads zip or gzip files and writes out uncompressed files. The utility would decode the header and process the trailer on its own, hence this routine expects only the raw deflate stream to decompress. This is different from the normal behavior of inflate(), which expects either a zlib or gzip header and trailer around the deflate stream. inflateBack() uses two subroutines supplied by the caller that are then called by inflateBack() for input and output. inflateBack() calls those routines until it reads a complete deflate stream and writes out all of the uncompressed data, or until it encounters an error. The function's parameters and return types are defined above in the in_func and out_func typedefs. inflateBack() will call in(in_desc, &buf) which should return the number of bytes of provided input, and a pointer to that input in buf. If there is no input available, in() must return zero--buf is ignored in that case--and inflateBack() will return a buffer error. inflateBack() will call out(out_desc, buf, len) to write the uncompressed data buf[0..len-1]. out() should return zero on success, or non-zero on failure. If out() returns non-zero, inflateBack() will return with an error. Neither in() nor out() are permitted to change the contents of the window provided to inflateBackInit(), which is also the buffer that out() uses to write from. The length written by out() will be at most the window size. Any non-zero amount of input may be provided by in(). For convenience, inflateBack() can be provided input on the first call by setting strm->next_in and strm->avail_in. If that input is exhausted, then in() will be called. Therefore strm->next_in must be initialized before calling inflateBack(). If strm->next_in is Z_NULL, then in() will be called immediately for input. If strm->next_in is not Z_NULL, then strm->avail_in must also be initialized, and then if strm->avail_in is not zero, input will initially be taken from strm->next_in[0 .. strm->avail_in - 1]. The in_desc and out_desc parameters of inflateBack() is passed as the first parameter of in() and out() respectively when they are called. These descriptors can be optionally used to pass any information that the caller- supplied in() and out() functions need to do their job. On return, inflateBack() will set strm->next_in and strm->avail_in to pass back any unused input that was provided by the last in() call. The return values of inflateBack() can be Z_STREAM_END on success, Z_BUF_ERROR if in() or out() returned an error, Z_DATA_ERROR if there was a format error in the deflate stream (in which case strm->msg is set to indicate the nature of the error), or Z_STREAM_ERROR if the stream was not properly initialized. In the case of Z_BUF_ERROR, an input or output error can be distinguished using strm->next_in which will be Z_NULL only if in() returned an error. If strm->next_in is not Z_NULL, then the Z_BUF_ERROR was due to out() returning non-zero. (in() will always be called before out(), so strm->next_in is assured to be defined if out() returns non-zero.) Note that inflateBack() cannot return Z_OK. */ ZEXTERN int ZEXPORT inflateBackEnd OF((z_streamp strm)); /* All memory allocated by inflateBackInit() is freed. inflateBackEnd() returns Z_OK on success, or Z_STREAM_ERROR if the stream state was inconsistent. */ ZEXTERN uLong ZEXPORT zlibCompileFlags OF((void)); /* Return flags indicating compile-time options. Type sizes, two bits each, 00 = 16 bits, 01 = 32, 10 = 64, 11 = other: 1.0: size of uInt 3.2: size of uLong 5.4: size of voidpf (pointer) 7.6: size of z_off_t Compiler, assembler, and debug options: 8: DEBUG 9: ASMV or ASMINF -- use ASM code 10: ZLIB_WINAPI -- exported functions use the WINAPI calling convention 11: 0 (reserved) One-time table building (smaller code, but not thread-safe if true): 12: BUILDFIXED -- build static block decoding tables when needed 13: DYNAMIC_CRC_TABLE -- build CRC calculation tables when needed 14,15: 0 (reserved) Library content (indicates missing functionality): 16: NO_GZCOMPRESS -- gz* functions cannot compress (to avoid linking deflate code when not needed) 17: NO_GZIP -- deflate can't write gzip streams, and inflate can't detect and decode gzip streams (to avoid linking crc code) 18-19: 0 (reserved) Operation variations (changes in library functionality): 20: PKZIP_BUG_WORKAROUND -- slightly more permissive inflate 21: FASTEST -- deflate algorithm with only one, lowest compression level 22,23: 0 (reserved) The sprintf variant used by gzprintf (zero is best): 24: 0 = vs*, 1 = s* -- 1 means limited to 20 arguments after the format 25: 0 = *nprintf, 1 = *printf -- 1 means gzprintf() not secure! 26: 0 = returns value, 1 = void -- 1 means inferred string length returned Remainder: 27-31: 0 (reserved) */ #ifndef Z_SOLO /* utility functions */ /* The following utility functions are implemented on top of the basic stream-oriented functions. To simplify the interface, some default options are assumed (compression level and memory usage, standard memory allocation functions). The source code of these utility functions can be modified if you need special options. */ ZEXTERN int ZEXPORT compress OF((Bytef *dest, uLongf *destLen, const Bytef *source, uLong sourceLen)); /* Compresses the source buffer into the destination buffer. sourceLen is the byte length of the source buffer. Upon entry, destLen is the total size of the destination buffer, which must be at least the value returned by compressBound(sourceLen). Upon exit, destLen is the actual size of the compressed buffer. compress returns Z_OK if success, Z_MEM_ERROR if there was not enough memory, Z_BUF_ERROR if there was not enough room in the output buffer. */ ZEXTERN int ZEXPORT compress2 OF((Bytef *dest, uLongf *destLen, const Bytef *source, uLong sourceLen, int level)); /* Compresses the source buffer into the destination buffer. The level parameter has the same meaning as in deflateInit. sourceLen is the byte length of the source buffer. Upon entry, destLen is the total size of the destination buffer, which must be at least the value returned by compressBound(sourceLen). Upon exit, destLen is the actual size of the compressed buffer. compress2 returns Z_OK if success, Z_MEM_ERROR if there was not enough memory, Z_BUF_ERROR if there was not enough room in the output buffer, Z_STREAM_ERROR if the level parameter is invalid. */ ZEXTERN uLong ZEXPORT compressBound OF((uLong sourceLen)); /* compressBound() returns an upper bound on the compressed size after compress() or compress2() on sourceLen bytes. It would be used before a compress() or compress2() call to allocate the destination buffer. */ ZEXTERN int ZEXPORT uncompress OF((Bytef *dest, uLongf *destLen, const Bytef *source, uLong sourceLen)); /* Decompresses the source buffer into the destination buffer. sourceLen is the byte length of the source buffer. Upon entry, destLen is the total size of the destination buffer, which must be large enough to hold the entire uncompressed data. (The size of the uncompressed data must have been saved previously by the compressor and transmitted to the decompressor by some mechanism outside the scope of this compression library.) Upon exit, destLen is the actual size of the uncompressed buffer. uncompress returns Z_OK if success, Z_MEM_ERROR if there was not enough memory, Z_BUF_ERROR if there was not enough room in the output buffer, or Z_DATA_ERROR if the input data was corrupted or incomplete. In the case where there is not enough room, uncompress() will fill the output buffer with the uncompressed data up to that point. */ /* gzip file access functions */ /* This library supports reading and writing files in gzip (.gz) format with an interface similar to that of stdio, using the functions that start with "gz". The gzip format is different from the zlib format. gzip is a gzip wrapper, documented in RFC 1952, wrapped around a deflate stream. */ typedef struct gzFile_s *gzFile; /* semi-opaque gzip file descriptor */ /* ZEXTERN gzFile ZEXPORT gzopen OF((const char *path, const char *mode)); Opens a gzip (.gz) file for reading or writing. The mode parameter is as in fopen ("rb" or "wb") but can also include a compression level ("wb9") or a strategy: 'f' for filtered data as in "wb6f", 'h' for Huffman-only compression as in "wb1h", 'R' for run-length encoding as in "wb1R", or 'F' for fixed code compression as in "wb9F". (See the description of deflateInit2 for more information about the strategy parameter.) 'T' will request transparent writing or appending with no compression and not using the gzip format. "a" can be used instead of "w" to request that the gzip stream that will be written be appended to the file. "+" will result in an error, since reading and writing to the same gzip file is not supported. The addition of "x" when writing will create the file exclusively, which fails if the file already exists. On systems that support it, the addition of "e" when reading or writing will set the flag to close the file on an execve() call. These functions, as well as gzip, will read and decode a sequence of gzip streams in a file. The append function of gzopen() can be used to create such a file. (Also see gzflush() for another way to do this.) When appending, gzopen does not test whether the file begins with a gzip stream, nor does it look for the end of the gzip streams to begin appending. gzopen will simply append a gzip stream to the existing file. gzopen can be used to read a file which is not in gzip format; in this case gzread will directly read from the file without decompression. When reading, this will be detected automatically by looking for the magic two- byte gzip header. gzopen returns NULL if the file could not be opened, if there was insufficient memory to allocate the gzFile state, or if an invalid mode was specified (an 'r', 'w', or 'a' was not provided, or '+' was provided). errno can be checked to determine if the reason gzopen failed was that the file could not be opened. */ ZEXTERN gzFile ZEXPORT gzdopen OF((int fd, const char *mode)); /* gzdopen associates a gzFile with the file descriptor fd. File descriptors are obtained from calls like open, dup, creat, pipe or fileno (if the file has been previously opened with fopen). The mode parameter is as in gzopen. The next call of gzclose on the returned gzFile will also close the file descriptor fd, just like fclose(fdopen(fd, mode)) closes the file descriptor fd. If you want to keep fd open, use fd = dup(fd_keep); gz = gzdopen(fd, mode);. The duplicated descriptor should be saved to avoid a leak, since gzdopen does not close fd if it fails. If you are using fileno() to get the file descriptor from a FILE *, then you will have to use dup() to avoid double-close()ing the file descriptor. Both gzclose() and fclose() will close the associated file descriptor, so they need to have different file descriptors. gzdopen returns NULL if there was insufficient memory to allocate the gzFile state, if an invalid mode was specified (an 'r', 'w', or 'a' was not provided, or '+' was provided), or if fd is -1. The file descriptor is not used until the next gz* read, write, seek, or close operation, so gzdopen will not detect if fd is invalid (unless fd is -1). */ ZEXTERN int ZEXPORT gzbuffer OF((gzFile file, unsigned size)); /* Set the internal buffer size used by this library's functions. The default buffer size is 8192 bytes. This function must be called after gzopen() or gzdopen(), and before any other calls that read or write the file. The buffer memory allocation is always deferred to the first read or write. Two buffers are allocated, either both of the specified size when writing, or one of the specified size and the other twice that size when reading. A larger buffer size of, for example, 64K or 128K bytes will noticeably increase the speed of decompression (reading). The new buffer size also affects the maximum length for gzprintf(). gzbuffer() returns 0 on success, or -1 on failure, such as being called too late. */ ZEXTERN int ZEXPORT gzsetparams OF((gzFile file, int level, int strategy)); /* Dynamically update the compression level or strategy. See the description of deflateInit2 for the meaning of these parameters. gzsetparams returns Z_OK if success, or Z_STREAM_ERROR if the file was not opened for writing. */ ZEXTERN int ZEXPORT gzread OF((gzFile file, voidp buf, unsigned len)); /* Reads the given number of uncompressed bytes from the compressed file. If the input file is not in gzip format, gzread copies the given number of bytes into the buffer directly from the file. After reaching the end of a gzip stream in the input, gzread will continue to read, looking for another gzip stream. Any number of gzip streams may be concatenated in the input file, and will all be decompressed by gzread(). If something other than a gzip stream is encountered after a gzip stream, that remaining trailing garbage is ignored (and no error is returned). gzread can be used to read a gzip file that is being concurrently written. Upon reaching the end of the input, gzread will return with the available data. If the error code returned by gzerror is Z_OK or Z_BUF_ERROR, then gzclearerr can be used to clear the end of file indicator in order to permit gzread to be tried again. Z_OK indicates that a gzip stream was completed on the last gzread. Z_BUF_ERROR indicates that the input file ended in the middle of a gzip stream. Note that gzread does not return -1 in the event of an incomplete gzip stream. This error is deferred until gzclose(), which will return Z_BUF_ERROR if the last gzread ended in the middle of a gzip stream. Alternatively, gzerror can be used before gzclose to detect this case. gzread returns the number of uncompressed bytes actually read, less than len for end of file, or -1 for error. */ ZEXTERN int ZEXPORT gzwrite OF((gzFile file, voidpc buf, unsigned len)); /* Writes the given number of uncompressed bytes into the compressed file. gzwrite returns the number of uncompressed bytes written or 0 in case of error. */ ZEXTERN int ZEXPORTVA gzprintf Z_ARG((gzFile file, const char *format, ...)); /* Converts, formats, and writes the arguments to the compressed file under control of the format string, as in fprintf. gzprintf returns the number of uncompressed bytes actually written, or 0 in case of error. The number of uncompressed bytes written is limited to 8191, or one less than the buffer size given to gzbuffer(). The caller should assure that this limit is not exceeded. If it is exceeded, then gzprintf() will return an error (0) with nothing written. In this case, there may also be a buffer overflow with unpredictable consequences, which is possible only if zlib was compiled with the insecure functions sprintf() or vsprintf() because the secure snprintf() or vsnprintf() functions were not available. This can be determined using zlibCompileFlags(). */ ZEXTERN int ZEXPORT gzputs OF((gzFile file, const char *s)); /* Writes the given null-terminated string to the compressed file, excluding the terminating null character. gzputs returns the number of characters written, or -1 in case of error. */ ZEXTERN char * ZEXPORT gzgets OF((gzFile file, char *buf, int len)); /* Reads bytes from the compressed file until len-1 characters are read, or a newline character is read and transferred to buf, or an end-of-file condition is encountered. If any characters are read or if len == 1, the string is terminated with a null character. If no characters are read due to an end-of-file or len < 1, then the buffer is left untouched. gzgets returns buf which is a null-terminated string, or it returns NULL for end-of-file or in case of error. If there was an error, the contents at buf are indeterminate. */ ZEXTERN int ZEXPORT gzputc OF((gzFile file, int c)); /* Writes c, converted to an unsigned char, into the compressed file. gzputc returns the value that was written, or -1 in case of error. */ ZEXTERN int ZEXPORT gzgetc OF((gzFile file)); /* Reads one byte from the compressed file. gzgetc returns this byte or -1 in case of end of file or error. This is implemented as a macro for speed. As such, it does not do all of the checking the other functions do. I.e. it does not check to see if file is NULL, nor whether the structure file points to has been clobbered or not. */ ZEXTERN int ZEXPORT gzungetc OF((int c, gzFile file)); /* Push one character back onto the stream to be read as the first character on the next read. At least one character of push-back is allowed. gzungetc() returns the character pushed, or -1 on failure. gzungetc() will fail if c is -1, and may fail if a character has been pushed but not read yet. If gzungetc is used immediately after gzopen or gzdopen, at least the output buffer size of pushed characters is allowed. (See gzbuffer above.) The pushed character will be discarded if the stream is repositioned with gzseek() or gzrewind(). */ ZEXTERN int ZEXPORT gzflush OF((gzFile file, int flush)); /* Flushes all pending output into the compressed file. The parameter flush is as in the deflate() function. The return value is the zlib error number (see function gzerror below). gzflush is only permitted when writing. If the flush parameter is Z_FINISH, the remaining data is written and the gzip stream is completed in the output. If gzwrite() is called again, a new gzip stream will be started in the output. gzread() is able to read such concatented gzip streams. gzflush should be called only when strictly necessary because it will degrade compression if called too often. */ /* ZEXTERN z_off_t ZEXPORT gzseek OF((gzFile file, z_off_t offset, int whence)); Sets the starting position for the next gzread or gzwrite on the given compressed file. The offset represents a number of bytes in the uncompressed data stream. The whence parameter is defined as in lseek(2); the value SEEK_END is not supported. If the file is opened for reading, this function is emulated but can be extremely slow. If the file is opened for writing, only forward seeks are supported; gzseek then compresses a sequence of zeroes up to the new starting position. gzseek returns the resulting offset location as measured in bytes from the beginning of the uncompressed stream, or -1 in case of error, in particular if the file is opened for writing and the new starting position would be before the current position. */ ZEXTERN int ZEXPORT gzrewind OF((gzFile file)); /* Rewinds the given file. This function is supported only for reading. gzrewind(file) is equivalent to (int)gzseek(file, 0L, SEEK_SET) */ /* ZEXTERN z_off_t ZEXPORT gztell OF((gzFile file)); Returns the starting position for the next gzread or gzwrite on the given compressed file. This position represents a number of bytes in the uncompressed data stream, and is zero when starting, even if appending or reading a gzip stream from the middle of a file using gzdopen(). gztell(file) is equivalent to gzseek(file, 0L, SEEK_CUR) */ /* ZEXTERN z_off_t ZEXPORT gzoffset OF((gzFile file)); Returns the current offset in the file being read or written. This offset includes the count of bytes that precede the gzip stream, for example when appending or when using gzdopen() for reading. When reading, the offset does not include as yet unused buffered input. This information can be used for a progress indicator. On error, gzoffset() returns -1. */ ZEXTERN int ZEXPORT gzeof OF((gzFile file)); /* Returns true (1) if the end-of-file indicator has been set while reading, false (0) otherwise. Note that the end-of-file indicator is set only if the read tried to go past the end of the input, but came up short. Therefore, just like feof(), gzeof() may return false even if there is no more data to read, in the event that the last read request was for the exact number of bytes remaining in the input file. This will happen if the input file size is an exact multiple of the buffer size. If gzeof() returns true, then the read functions will return no more data, unless the end-of-file indicator is reset by gzclearerr() and the input file has grown since the previous end of file was detected. */ ZEXTERN int ZEXPORT gzdirect OF((gzFile file)); /* Returns true (1) if file is being copied directly while reading, or false (0) if file is a gzip stream being decompressed. If the input file is empty, gzdirect() will return true, since the input does not contain a gzip stream. If gzdirect() is used immediately after gzopen() or gzdopen() it will cause buffers to be allocated to allow reading the file to determine if it is a gzip file. Therefore if gzbuffer() is used, it should be called before gzdirect(). When writing, gzdirect() returns true (1) if transparent writing was requested ("wT" for the gzopen() mode), or false (0) otherwise. (Note: gzdirect() is not needed when writing. Transparent writing must be explicitly requested, so the application already knows the answer. When linking statically, using gzdirect() will include all of the zlib code for gzip file reading and decompression, which may not be desired.) */ ZEXTERN int ZEXPORT gzclose OF((gzFile file)); /* Flushes all pending output if necessary, closes the compressed file and deallocates the (de)compression state. Note that once file is closed, you cannot call gzerror with file, since its structures have been deallocated. gzclose must not be called more than once on the same file, just as free must not be called more than once on the same allocation. gzclose will return Z_STREAM_ERROR if file is not valid, Z_ERRNO on a file operation error, Z_MEM_ERROR if out of memory, Z_BUF_ERROR if the last read ended in the middle of a gzip stream, or Z_OK on success. */ ZEXTERN int ZEXPORT gzclose_r OF((gzFile file)); ZEXTERN int ZEXPORT gzclose_w OF((gzFile file)); /* Same as gzclose(), but gzclose_r() is only for use when reading, and gzclose_w() is only for use when writing or appending. The advantage to using these instead of gzclose() is that they avoid linking in zlib compression or decompression code that is not used when only reading or only writing respectively. If gzclose() is used, then both compression and decompression code will be included the application when linking to a static zlib library. */ ZEXTERN const char * ZEXPORT gzerror OF((gzFile file, int *errnum)); /* Returns the error message for the last error which occurred on the given compressed file. errnum is set to zlib error number. If an error occurred in the file system and not in the compression library, errnum is set to Z_ERRNO and the application may consult errno to get the exact error code. The application must not modify the returned string. Future calls to this function may invalidate the previously returned string. If file is closed, then the string previously returned by gzerror will no longer be available. gzerror() should be used to distinguish errors from end-of-file for those functions above that do not distinguish those cases in their return values. */ ZEXTERN void ZEXPORT gzclearerr OF((gzFile file)); /* Clears the error and end-of-file flags for file. This is analogous to the clearerr() function in stdio. This is useful for continuing to read a gzip file that is being written concurrently. */ #endif /* !Z_SOLO */ /* checksum functions */ /* These functions are not related to compression but are exported anyway because they might be useful in applications using the compression library. */ ZEXTERN uLong ZEXPORT adler32 OF((uLong adler, const Bytef *buf, uInt len)); /* Update a running Adler-32 checksum with the bytes buf[0..len-1] and return the updated checksum. If buf is Z_NULL, this function returns the required initial value for the checksum. An Adler-32 checksum is almost as reliable as a CRC32 but can be computed much faster. Usage example: uLong adler = adler32(0L, Z_NULL, 0); while (read_buffer(buffer, length) != EOF) { adler = adler32(adler, buffer, length); } if (adler != original_adler) error(); */ /* ZEXTERN uLong ZEXPORT adler32_combine OF((uLong adler1, uLong adler2, z_off_t len2)); Combine two Adler-32 checksums into one. For two sequences of bytes, seq1 and seq2 with lengths len1 and len2, Adler-32 checksums were calculated for each, adler1 and adler2. adler32_combine() returns the Adler-32 checksum of seq1 and seq2 concatenated, requiring only adler1, adler2, and len2. Note that the z_off_t type (like off_t) is a signed integer. If len2 is negative, the result has no meaning or utility. */ ZEXTERN uLong ZEXPORT crc32 OF((uLong crc, const Bytef *buf, uInt len)); /* Update a running CRC-32 with the bytes buf[0..len-1] and return the updated CRC-32. If buf is Z_NULL, this function returns the required initial value for the crc. Pre- and post-conditioning (one's complement) is performed within this function so it shouldn't be done by the application. Usage example: uLong crc = crc32(0L, Z_NULL, 0); while (read_buffer(buffer, length) != EOF) { crc = crc32(crc, buffer, length); } if (crc != original_crc) error(); */ /* ZEXTERN uLong ZEXPORT crc32_combine OF((uLong crc1, uLong crc2, z_off_t len2)); Combine two CRC-32 check values into one. For two sequences of bytes, seq1 and seq2 with lengths len1 and len2, CRC-32 check values were calculated for each, crc1 and crc2. crc32_combine() returns the CRC-32 check value of seq1 and seq2 concatenated, requiring only crc1, crc2, and len2. */ /* various hacks, don't look :) */ /* deflateInit and inflateInit are macros to allow checking the zlib version * and the compiler's view of z_stream: */ ZEXTERN int ZEXPORT deflateInit_ OF((z_streamp strm, int level, const char *version, int stream_size)); ZEXTERN int ZEXPORT inflateInit_ OF((z_streamp strm, const char *version, int stream_size)); ZEXTERN int ZEXPORT deflateInit2_ OF((z_streamp strm, int level, int method, int windowBits, int memLevel, int strategy, const char *version, int stream_size)); ZEXTERN int ZEXPORT inflateInit2_ OF((z_streamp strm, int windowBits, const char *version, int stream_size)); ZEXTERN int ZEXPORT inflateBackInit_ OF((z_streamp strm, int windowBits, unsigned char FAR *window, const char *version, int stream_size)); #define deflateInit(strm, level) \ deflateInit_((strm), (level), ZLIB_VERSION, (int)sizeof(z_stream)) #define inflateInit(strm) \ inflateInit_((strm), ZLIB_VERSION, (int)sizeof(z_stream)) #define deflateInit2(strm, level, method, windowBits, memLevel, strategy) \ deflateInit2_((strm),(level),(method),(windowBits),(memLevel),\ (strategy), ZLIB_VERSION, (int)sizeof(z_stream)) #define inflateInit2(strm, windowBits) \ inflateInit2_((strm), (windowBits), ZLIB_VERSION, \ (int)sizeof(z_stream)) #define inflateBackInit(strm, windowBits, window) \ inflateBackInit_((strm), (windowBits), (window), \ ZLIB_VERSION, (int)sizeof(z_stream)) #ifndef Z_SOLO /* gzgetc() macro and its supporting function and exposed data structure. Note * that the real internal state is much larger than the exposed structure. * This abbreviated structure exposes just enough for the gzgetc() macro. The * user should not mess with these exposed elements, since their names or * behavior could change in the future, perhaps even capriciously. They can * only be used by the gzgetc() macro. You have been warned. */ struct gzFile_s { unsigned have; unsigned char *next; z_off64_t pos; }; ZEXTERN int ZEXPORT gzgetc_ OF((gzFile file)); /* backward compatibility */ #ifdef Z_PREFIX_SET # undef z_gzgetc # define z_gzgetc(g) \ ((g)->have ? ((g)->have--, (g)->pos++, *((g)->next)++) : gzgetc(g)) #else # define gzgetc(g) \ ((g)->have ? ((g)->have--, (g)->pos++, *((g)->next)++) : gzgetc(g)) #endif /* provide 64-bit offset functions if _LARGEFILE64_SOURCE defined, and/or * change the regular functions to 64 bits if _FILE_OFFSET_BITS is 64 (if * both are true, the application gets the *64 functions, and the regular * functions are changed to 64 bits) -- in case these are set on systems * without large file support, _LFS64_LARGEFILE must also be true */ #ifdef Z_LARGE64 ZEXTERN gzFile ZEXPORT gzopen64 OF((const char *, const char *)); ZEXTERN z_off64_t ZEXPORT gzseek64 OF((gzFile, z_off64_t, int)); ZEXTERN z_off64_t ZEXPORT gztell64 OF((gzFile)); ZEXTERN z_off64_t ZEXPORT gzoffset64 OF((gzFile)); ZEXTERN uLong ZEXPORT adler32_combine64 OF((uLong, uLong, z_off64_t)); ZEXTERN uLong ZEXPORT crc32_combine64 OF((uLong, uLong, z_off64_t)); #endif #if !defined(ZLIB_INTERNAL) && defined(Z_WANT64) # ifdef Z_PREFIX_SET # define z_gzopen z_gzopen64 # define z_gzseek z_gzseek64 # define z_gztell z_gztell64 # define z_gzoffset z_gzoffset64 # define z_adler32_combine z_adler32_combine64 # define z_crc32_combine z_crc32_combine64 # else # define gzopen gzopen64 # define gzseek gzseek64 # define gztell gztell64 # define gzoffset gzoffset64 # define adler32_combine adler32_combine64 # define crc32_combine crc32_combine64 # endif # ifndef Z_LARGE64 ZEXTERN gzFile ZEXPORT gzopen64 OF((const char *, const char *)); ZEXTERN z_off_t ZEXPORT gzseek64 OF((gzFile, z_off_t, int)); ZEXTERN z_off_t ZEXPORT gztell64 OF((gzFile)); ZEXTERN z_off_t ZEXPORT gzoffset64 OF((gzFile)); ZEXTERN uLong ZEXPORT adler32_combine64 OF((uLong, uLong, z_off_t)); ZEXTERN uLong ZEXPORT crc32_combine64 OF((uLong, uLong, z_off_t)); # endif #else ZEXTERN gzFile ZEXPORT gzopen OF((const char *, const char *)); ZEXTERN z_off_t ZEXPORT gzseek OF((gzFile, z_off_t, int)); ZEXTERN z_off_t ZEXPORT gztell OF((gzFile)); ZEXTERN z_off_t ZEXPORT gzoffset OF((gzFile)); ZEXTERN uLong ZEXPORT adler32_combine OF((uLong, uLong, z_off_t)); ZEXTERN uLong ZEXPORT crc32_combine OF((uLong, uLong, z_off_t)); #endif #else /* Z_SOLO */ ZEXTERN uLong ZEXPORT adler32_combine OF((uLong, uLong, z_off_t)); ZEXTERN uLong ZEXPORT crc32_combine OF((uLong, uLong, z_off_t)); #endif /* !Z_SOLO */ /* hack for buggy compilers */ #if !defined(ZUTIL_H) && !defined(NO_DUMMY_DECL) struct internal_state {int dummy;}; #endif /* undocumented functions */ ZEXTERN const char * ZEXPORT zError OF((int)); ZEXTERN int ZEXPORT inflateSyncPoint OF((z_streamp)); ZEXTERN const z_crc_t FAR * ZEXPORT get_crc_table OF((void)); ZEXTERN int ZEXPORT inflateUndermine OF((z_streamp, int)); ZEXTERN int ZEXPORT inflateResetKeep OF((z_streamp)); ZEXTERN int ZEXPORT deflateResetKeep OF((z_streamp)); #if defined(_WIN32) && !defined(Z_SOLO) ZEXTERN gzFile ZEXPORT gzopen_w OF((const wchar_t *path, const char *mode)); #endif #if defined(STDC) || defined(Z_HAVE_STDARG_H) # ifndef Z_SOLO ZEXTERN int ZEXPORTVA gzvprintf Z_ARG((gzFile file, const char *format, va_list va)); # endif #endif #ifdef __cplusplus } #endif #endif /* ZLIB_H */ c-blosc-1.7.0/internal-complibs/zlib-1.2.8/zutil.c000066400000000000000000000163661254622165100214220ustar00rootroot00000000000000/* zutil.c -- target dependent utility functions for the compression library * Copyright (C) 1995-2005, 2010, 2011, 2012 Jean-loup Gailly. * For conditions of distribution and use, see copyright notice in zlib.h */ /* @(#) $Id$ */ #include "zutil.h" #ifndef Z_SOLO # include "gzguts.h" #endif #ifndef NO_DUMMY_DECL struct internal_state {int dummy;}; /* for buggy compilers */ #endif z_const char * const z_errmsg[10] = { "need dictionary", /* Z_NEED_DICT 2 */ "stream end", /* Z_STREAM_END 1 */ "", /* Z_OK 0 */ "file error", /* Z_ERRNO (-1) */ "stream error", /* Z_STREAM_ERROR (-2) */ "data error", /* Z_DATA_ERROR (-3) */ "insufficient memory", /* Z_MEM_ERROR (-4) */ "buffer error", /* Z_BUF_ERROR (-5) */ "incompatible version",/* Z_VERSION_ERROR (-6) */ ""}; const char * ZEXPORT zlibVersion() { return ZLIB_VERSION; } uLong ZEXPORT zlibCompileFlags() { uLong flags; flags = 0; switch ((int)(sizeof(uInt))) { case 2: break; case 4: flags += 1; break; case 8: flags += 2; break; default: flags += 3; } switch ((int)(sizeof(uLong))) { case 2: break; case 4: flags += 1 << 2; break; case 8: flags += 2 << 2; break; default: flags += 3 << 2; } switch ((int)(sizeof(voidpf))) { case 2: break; case 4: flags += 1 << 4; break; case 8: flags += 2 << 4; break; default: flags += 3 << 4; } switch ((int)(sizeof(z_off_t))) { case 2: break; case 4: flags += 1 << 6; break; case 8: flags += 2 << 6; break; default: flags += 3 << 6; } #ifdef DEBUG flags += 1 << 8; #endif #if defined(ASMV) || defined(ASMINF) flags += 1 << 9; #endif #ifdef ZLIB_WINAPI flags += 1 << 10; #endif #ifdef BUILDFIXED flags += 1 << 12; #endif #ifdef DYNAMIC_CRC_TABLE flags += 1 << 13; #endif #ifdef NO_GZCOMPRESS flags += 1L << 16; #endif #ifdef NO_GZIP flags += 1L << 17; #endif #ifdef PKZIP_BUG_WORKAROUND flags += 1L << 20; #endif #ifdef FASTEST flags += 1L << 21; #endif #if defined(STDC) || defined(Z_HAVE_STDARG_H) # ifdef NO_vsnprintf flags += 1L << 25; # ifdef HAS_vsprintf_void flags += 1L << 26; # endif # else # ifdef HAS_vsnprintf_void flags += 1L << 26; # endif # endif #else flags += 1L << 24; # ifdef NO_snprintf flags += 1L << 25; # ifdef HAS_sprintf_void flags += 1L << 26; # endif # else # ifdef HAS_snprintf_void flags += 1L << 26; # endif # endif #endif return flags; } #ifdef DEBUG # ifndef verbose # define verbose 0 # endif int ZLIB_INTERNAL z_verbose = verbose; void ZLIB_INTERNAL z_error (m) char *m; { fprintf(stderr, "%s\n", m); exit(1); } #endif /* exported to allow conversion of error code to string for compress() and * uncompress() */ const char * ZEXPORT zError(err) int err; { return ERR_MSG(err); } #if defined(_WIN32_WCE) /* The Microsoft C Run-Time Library for Windows CE doesn't have * errno. We define it as a global variable to simplify porting. * Its value is always 0 and should not be used. */ int errno = 0; #endif #ifndef HAVE_MEMCPY void ZLIB_INTERNAL zmemcpy(dest, source, len) Bytef* dest; const Bytef* source; uInt len; { if (len == 0) return; do { *dest++ = *source++; /* ??? to be unrolled */ } while (--len != 0); } int ZLIB_INTERNAL zmemcmp(s1, s2, len) const Bytef* s1; const Bytef* s2; uInt len; { uInt j; for (j = 0; j < len; j++) { if (s1[j] != s2[j]) return 2*(s1[j] > s2[j])-1; } return 0; } void ZLIB_INTERNAL zmemzero(dest, len) Bytef* dest; uInt len; { if (len == 0) return; do { *dest++ = 0; /* ??? to be unrolled */ } while (--len != 0); } #endif #ifndef Z_SOLO #ifdef SYS16BIT #ifdef __TURBOC__ /* Turbo C in 16-bit mode */ # define MY_ZCALLOC /* Turbo C malloc() does not allow dynamic allocation of 64K bytes * and farmalloc(64K) returns a pointer with an offset of 8, so we * must fix the pointer. Warning: the pointer must be put back to its * original form in order to free it, use zcfree(). */ #define MAX_PTR 10 /* 10*64K = 640K */ local int next_ptr = 0; typedef struct ptr_table_s { voidpf org_ptr; voidpf new_ptr; } ptr_table; local ptr_table table[MAX_PTR]; /* This table is used to remember the original form of pointers * to large buffers (64K). Such pointers are normalized with a zero offset. * Since MSDOS is not a preemptive multitasking OS, this table is not * protected from concurrent access. This hack doesn't work anyway on * a protected system like OS/2. Use Microsoft C instead. */ voidpf ZLIB_INTERNAL zcalloc (voidpf opaque, unsigned items, unsigned size) { voidpf buf = opaque; /* just to make some compilers happy */ ulg bsize = (ulg)items*size; /* If we allocate less than 65520 bytes, we assume that farmalloc * will return a usable pointer which doesn't have to be normalized. */ if (bsize < 65520L) { buf = farmalloc(bsize); if (*(ush*)&buf != 0) return buf; } else { buf = farmalloc(bsize + 16L); } if (buf == NULL || next_ptr >= MAX_PTR) return NULL; table[next_ptr].org_ptr = buf; /* Normalize the pointer to seg:0 */ *((ush*)&buf+1) += ((ush)((uch*)buf-0) + 15) >> 4; *(ush*)&buf = 0; table[next_ptr++].new_ptr = buf; return buf; } void ZLIB_INTERNAL zcfree (voidpf opaque, voidpf ptr) { int n; if (*(ush*)&ptr != 0) { /* object < 64K */ farfree(ptr); return; } /* Find the original pointer */ for (n = 0; n < next_ptr; n++) { if (ptr != table[n].new_ptr) continue; farfree(table[n].org_ptr); while (++n < next_ptr) { table[n-1] = table[n]; } next_ptr--; return; } ptr = opaque; /* just to make some compilers happy */ Assert(0, "zcfree: ptr not found"); } #endif /* __TURBOC__ */ #ifdef M_I86 /* Microsoft C in 16-bit mode */ # define MY_ZCALLOC #if (!defined(_MSC_VER) || (_MSC_VER <= 600)) # define _halloc halloc # define _hfree hfree #endif voidpf ZLIB_INTERNAL zcalloc (voidpf opaque, uInt items, uInt size) { if (opaque) opaque = 0; /* to make compiler happy */ return _halloc((long)items, size); } void ZLIB_INTERNAL zcfree (voidpf opaque, voidpf ptr) { if (opaque) opaque = 0; /* to make compiler happy */ _hfree(ptr); } #endif /* M_I86 */ #endif /* SYS16BIT */ #ifndef MY_ZCALLOC /* Any system without a special alloc function */ #ifndef STDC extern voidp malloc OF((uInt size)); extern voidp calloc OF((uInt items, uInt size)); extern void free OF((voidpf ptr)); #endif voidpf ZLIB_INTERNAL zcalloc (opaque, items, size) voidpf opaque; unsigned items; unsigned size; { if (opaque) items += size - size; /* make compiler happy */ return sizeof(uInt) > 2 ? (voidpf)malloc(items * size) : (voidpf)calloc(items, size); } void ZLIB_INTERNAL zcfree (opaque, ptr) voidpf opaque; voidpf ptr; { free(ptr); if (opaque) return; /* make compiler happy */ } #endif /* MY_ZCALLOC */ #endif /* !Z_SOLO */ c-blosc-1.7.0/internal-complibs/zlib-1.2.8/zutil.h000066400000000000000000000151561254622165100214230ustar00rootroot00000000000000/* zutil.h -- internal interface and configuration of the compression library * Copyright (C) 1995-2013 Jean-loup Gailly. * For conditions of distribution and use, see copyright notice in zlib.h */ /* WARNING: this file should *not* be used by applications. It is part of the implementation of the compression library and is subject to change. Applications should only use zlib.h. */ /* @(#) $Id$ */ #ifndef ZUTIL_H #define ZUTIL_H #ifdef HAVE_HIDDEN # define ZLIB_INTERNAL __attribute__((visibility ("hidden"))) #else # define ZLIB_INTERNAL #endif #include "zlib.h" #if defined(STDC) && !defined(Z_SOLO) # if !(defined(_WIN32_WCE) && defined(_MSC_VER)) # include # endif # include # include #endif #ifdef Z_SOLO typedef long ptrdiff_t; /* guess -- will be caught if guess is wrong */ #endif #ifndef local # define local static #endif /* compile with -Dlocal if your debugger can't find static symbols */ typedef unsigned char uch; typedef uch FAR uchf; typedef unsigned short ush; typedef ush FAR ushf; typedef unsigned long ulg; extern z_const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ /* (size given to avoid silly warnings with Visual C++) */ #define ERR_MSG(err) z_errmsg[Z_NEED_DICT-(err)] #define ERR_RETURN(strm,err) \ return (strm->msg = ERR_MSG(err), (err)) /* To be used only when the state is known to be valid */ /* common constants */ #ifndef DEF_WBITS # define DEF_WBITS MAX_WBITS #endif /* default windowBits for decompression. MAX_WBITS is for compression only */ #if MAX_MEM_LEVEL >= 8 # define DEF_MEM_LEVEL 8 #else # define DEF_MEM_LEVEL MAX_MEM_LEVEL #endif /* default memLevel */ #define STORED_BLOCK 0 #define STATIC_TREES 1 #define DYN_TREES 2 /* The three kinds of block type */ #define MIN_MATCH 3 #define MAX_MATCH 258 /* The minimum and maximum match lengths */ #define PRESET_DICT 0x20 /* preset dictionary flag in zlib header */ /* target dependencies */ #if defined(MSDOS) || (defined(WINDOWS) && !defined(WIN32)) # define OS_CODE 0x00 # ifndef Z_SOLO # if defined(__TURBOC__) || defined(__BORLANDC__) # if (__STDC__ == 1) && (defined(__LARGE__) || defined(__COMPACT__)) /* Allow compilation with ANSI keywords only enabled */ void _Cdecl farfree( void *block ); void *_Cdecl farmalloc( unsigned long nbytes ); # else # include # endif # else /* MSC or DJGPP */ # include # endif # endif #endif #ifdef AMIGA # define OS_CODE 0x01 #endif #if defined(VAXC) || defined(VMS) # define OS_CODE 0x02 # define F_OPEN(name, mode) \ fopen((name), (mode), "mbc=60", "ctx=stm", "rfm=fix", "mrs=512") #endif #if defined(ATARI) || defined(atarist) # define OS_CODE 0x05 #endif #ifdef OS2 # define OS_CODE 0x06 # if defined(M_I86) && !defined(Z_SOLO) # include # endif #endif #if defined(MACOS) || defined(TARGET_OS_MAC) # define OS_CODE 0x07 # ifndef Z_SOLO # if defined(__MWERKS__) && __dest_os != __be_os && __dest_os != __win32_os # include /* for fdopen */ # else # ifndef fdopen # define fdopen(fd,mode) NULL /* No fdopen() */ # endif # endif # endif #endif #ifdef TOPS20 # define OS_CODE 0x0a #endif #ifdef WIN32 # ifndef __CYGWIN__ /* Cygwin is Unix, not Win32 */ # define OS_CODE 0x0b # endif #endif #ifdef __50SERIES /* Prime/PRIMOS */ # define OS_CODE 0x0f #endif #if defined(_BEOS_) || defined(RISCOS) # define fdopen(fd,mode) NULL /* No fdopen() */ #endif #if (defined(_MSC_VER) && (_MSC_VER > 600)) && !defined __INTERIX # if defined(_WIN32_WCE) # define fdopen(fd,mode) NULL /* No fdopen() */ # ifndef _PTRDIFF_T_DEFINED typedef int ptrdiff_t; # define _PTRDIFF_T_DEFINED # endif # else # define fdopen(fd,type) _fdopen(fd,type) # endif #endif #if defined(__BORLANDC__) && !defined(MSDOS) #pragma warn -8004 #pragma warn -8008 #pragma warn -8066 #endif /* provide prototypes for these when building zlib without LFS */ #if !defined(_WIN32) && \ (!defined(_LARGEFILE64_SOURCE) || _LFS64_LARGEFILE-0 == 0) ZEXTERN uLong ZEXPORT adler32_combine64 OF((uLong, uLong, z_off_t)); ZEXTERN uLong ZEXPORT crc32_combine64 OF((uLong, uLong, z_off_t)); #endif /* common defaults */ #ifndef OS_CODE # define OS_CODE 0x03 /* assume Unix */ #endif #ifndef F_OPEN # define F_OPEN(name, mode) fopen((name), (mode)) #endif /* functions */ #if defined(pyr) || defined(Z_SOLO) # define NO_MEMCPY #endif #if defined(SMALL_MEDIUM) && !defined(_MSC_VER) && !defined(__SC__) /* Use our own functions for small and medium model with MSC <= 5.0. * You may have to use the same strategy for Borland C (untested). * The __SC__ check is for Symantec. */ # define NO_MEMCPY #endif #if defined(STDC) && !defined(HAVE_MEMCPY) && !defined(NO_MEMCPY) # define HAVE_MEMCPY #endif #ifdef HAVE_MEMCPY # ifdef SMALL_MEDIUM /* MSDOS small or medium model */ # define zmemcpy _fmemcpy # define zmemcmp _fmemcmp # define zmemzero(dest, len) _fmemset(dest, 0, len) # else # define zmemcpy memcpy # define zmemcmp memcmp # define zmemzero(dest, len) memset(dest, 0, len) # endif #else void ZLIB_INTERNAL zmemcpy OF((Bytef* dest, const Bytef* source, uInt len)); int ZLIB_INTERNAL zmemcmp OF((const Bytef* s1, const Bytef* s2, uInt len)); void ZLIB_INTERNAL zmemzero OF((Bytef* dest, uInt len)); #endif /* Diagnostic functions */ #ifdef DEBUG # include extern int ZLIB_INTERNAL z_verbose; extern void ZLIB_INTERNAL z_error OF((char *m)); # define Assert(cond,msg) {if(!(cond)) z_error(msg);} # define Trace(x) {if (z_verbose>=0) fprintf x ;} # define Tracev(x) {if (z_verbose>0) fprintf x ;} # define Tracevv(x) {if (z_verbose>1) fprintf x ;} # define Tracec(c,x) {if (z_verbose>0 && (c)) fprintf x ;} # define Tracecv(c,x) {if (z_verbose>1 && (c)) fprintf x ;} #else # define Assert(cond,msg) # define Trace(x) # define Tracev(x) # define Tracevv(x) # define Tracec(c,x) # define Tracecv(c,x) #endif #ifndef Z_SOLO voidpf ZLIB_INTERNAL zcalloc OF((voidpf opaque, unsigned items, unsigned size)); void ZLIB_INTERNAL zcfree OF((voidpf opaque, voidpf ptr)); #endif #define ZALLOC(strm, items, size) \ (*((strm)->zalloc))((strm)->opaque, (items), (size)) #define ZFREE(strm, addr) (*((strm)->zfree))((strm)->opaque, (voidpf)(addr)) #define TRY_FREE(s, p) {if (p) ZFREE(s, p);} /* Reverse the bytes in a 32-bit value */ #define ZSWAP32(q) ((((q) >> 24) & 0xff) + (((q) >> 8) & 0xff00) + \ (((q) & 0xff00) << 8) + (((q) & 0xff) << 24)) #endif /* ZUTIL_H */ c-blosc-1.7.0/scripts/000077500000000000000000000000001254622165100145345ustar00rootroot00000000000000c-blosc-1.7.0/scripts/travis-before-install.sh000077500000000000000000000006171254622165100213130ustar00rootroot00000000000000#/bin/sh -f # things to do for travis-ci in the before_install section if ( test "`uname -s`" = "Darwin" ) then #cmake v2.8.12 is installed on the Mac workers now #brew update #brew install cmake echo else #install a newer cmake since at this time Travis only has version 2.8.7 sudo add-apt-repository --yes ppa:kalakris/cmake sudo apt-get update -qq sudo apt-get install cmake fi c-blosc-1.7.0/tests/000077500000000000000000000000001254622165100142075ustar00rootroot00000000000000c-blosc-1.7.0/tests/.gitignore000066400000000000000000000000061254622165100161730ustar00rootroot00000000000000*.exe c-blosc-1.7.0/tests/CMakeLists.txt000066400000000000000000000073571254622165100167630ustar00rootroot00000000000000# sources #aux_source_directory(. SOURCES) file(GLOB SOURCES test_*.c) # flags link_directories(${PROJECT_BINARY_DIR}/blosc) # targets and tests foreach(source ${SOURCES}) # Enable support for testing accelerated shuffles if(COMPILER_SUPPORT_SSE2) # Define a symbol so tests for SSE2 shuffle/unshuffle will be compiled in. set_property( SOURCE ${source} APPEND PROPERTY COMPILE_DEFINITIONS SHUFFLE_SSE2_ENABLED) endif(COMPILER_SUPPORT_SSE2) # if(COMPILER_SUPPORT_AVX2) # # Define a symbol so tests for AVX2 shuffle/unshuffle will be compiled in. # set_property( # SOURCE ${source} # APPEND PROPERTY COMPILE_DEFINITIONS SHUFFLE_AVX2_ENABLED) # endif(COMPILER_SUPPORT_AVX2) get_filename_component(target ${source} NAME_WE) add_executable(${target} ${source}) # Define the BLOSC_TESTING symbol so normally-hidden functions # aren't hidden from the view of the test programs. set_property( TARGET ${target} APPEND PROPERTY COMPILE_DEFINITIONS BLOSC_TESTING) # have to copy dlls for Visual Studio if(MSVC) add_custom_command( TARGET ${target} POST_BUILD COMMAND ${CMAKE_COMMAND} ARGS -E copy_if_different "${PROJECT_BINARY_DIR}/blosc/\$\(Configuration\)/blosc_testing.dll" "${CMAKE_CURRENT_BINARY_DIR}/\$\(Configuration\)/blosc_testing.dll") elseif(MINGW) add_custom_command( TARGET ${target} POST_BUILD COMMAND ${CMAKE_COMMAND} ARGS -E copy_if_different "${PROJECT_BINARY_DIR}/blosc/libblosc_testing.dll" "${CMAKE_CURRENT_BINARY_DIR}/libblosc_testing.dll") endif() target_link_libraries(${target} blosc_testing) # If there's a CSV file present for this test, read it to get the list # of test parameters then add a test for each parameter set. # Otherwise, this is a simple test so just add it once. get_filename_component(source_extension ${source} EXT) string(REGEX REPLACE "${source_extension}$" ".csv" test_params_file ${source}) if (EXISTS "${test_params_file}") # Read the file contents into a CMake list file(READ "${test_params_file}" test_params_contents) string(REGEX REPLACE ";" "\\\\;" test_params_contents "${test_params_contents}") string(REGEX REPLACE "\n" ";" test_params_contents "${test_params_contents}") # How many parameter sets for this test? # If there's not at least one (accounting for the CSV header line), # that's probably not correct so emit an error and stop configuring. list(LENGTH test_params_contents test_params_count) if ("${test_params_count}" LESS 2) message(ERROR "Invalid test parameters file: ${test_params_file}") endif() # Remove the header line. list(REMOVE_AT test_params_contents 0) # Add a test for each parameter set in the file. foreach(test_params_raw ${test_params_contents}) string(REGEX REPLACE "," " " test_params "${test_params_raw}") # Create the test name. # NOTE: The documentation for add_test says the test name "may not contain # spaces, quotes, or other characters special in CMake syntax." string(REGEX REPLACE "\"| " "_" test_name_params "${test_params}") set(test_name "${target}_${test_name_params}") separate_arguments(test_params) add_test(${test_name} ${target} ${test_params}) endforeach() else() add_test(${target} ${target}) endif() endforeach(source) c-blosc-1.7.0/tests/Makefile000066400000000000000000000020621254622165100156470ustar00rootroot00000000000000CC=gcc CFLAGS=-O3 -msse2 -Wall -pthread LDFLAGS=-pthread BLOSC_LIB= $(wildcard ../blosc/*.c) # The list of executables # Generated PNG (intermediate) files SOURCES := $(wildcard *.c) EXECUTABLES := $(patsubst %.c, %.exe, $(SOURCES)) # Support for internal LZ4 and LZ4HC LZ4_DIR = ../internal-complibs/lz4-1.7.0 CFLAGS += -DHAVE_LZ4 -I$(LZ4_DIR) BLOSC_LIB += $(wildcard $(LZ4_DIR)/*.c) # Support for external LZ4 and LZ4HC #LDFLAGS += -DHAVE_LZ4 -llz4 # Support for internal Snappy #SNAPPY_DIR = ../internal-complibs/snappy-1.1.1 #CFLAGS += -DHAVE_SNAPPY -I$(SNAPPY_DIR) #BLOSC_LIB += $(wildcard $(SNAPPY_DIR)/*.cc) # Support for external Snappy LDFLAGS += -DHAVE_SNAPPY -lsnappy # Support for external Zlib LDFLAGS += -DHAVE_ZLIB -lz # Support for internal Zlib #ZLIB_DIR = ../internal-complibs/zlib-1.2.8 #CFLAGS += -DHAVE_ZLIB -I$(ZLIB_DIR) #BLOSC_LIB += $(wildcard $(ZLIB_DIR)/*.c) .PHONY: all all: $(EXECUTABLES) test: $(EXECUTABLES) sh test_all.sh %.exe: %.c $(BLOSC_LIB) $(CC) $(CFLAGS) $(LDFLAGS) "$<" $(BLOSC_LIB) -o "$@" clean: rm -rf $(EXECUTABLES) c-blosc-1.7.0/tests/print_versions.c000066400000000000000000000020531254622165100174370ustar00rootroot00000000000000/********************************************************************* Print versions for Blosc and all its internal compressors. *********************************************************************/ #include #include #include #include "../blosc/blosc.h" int main(int argc, char *argv[]) { char *name = NULL, *version = NULL; int ret; printf("Blosc version: %s (%s)\n", BLOSC_VERSION_STRING, BLOSC_VERSION_DATE); printf("List of supported compressors in this build: %s\n", blosc_list_compressors()); printf("Supported compression libraries:\n"); ret = blosc_get_complib_info("blosclz", &name, &version); if (ret >= 0) printf(" %s: %s\n", name, version); ret = blosc_get_complib_info("lz4", &name, &version); if (ret >= 0) printf(" %s: %s\n", name, version); ret = blosc_get_complib_info("snappy", &name, &version); if (ret >= 0) printf(" %s: %s\n", name, version); ret = blosc_get_complib_info("zlib", &name, &version); if (ret >= 0) printf(" %s: %s\n", name, version); return(0); } c-blosc-1.7.0/tests/test_all.sh000066400000000000000000000006661254622165100163620ustar00rootroot00000000000000#********************************************************************* # Blosc - Blocked Shuffling and Compression Library # # Unit tests for basic features in Blosc. # # Creation date: 2010-06-07 # Author: Francesc Alted # # See LICENSES/BLOSC.txt for details about copyright and rights to use. #********************************************************************** for exe in $(ls *.exe); do ./$exe done c-blosc-1.7.0/tests/test_api.c000066400000000000000000000055741254622165100161760ustar00rootroot00000000000000/********************************************************************* Blosc - Blocked Shuffling and Compression Library Unit tests for Blosc API. Creation date: 2010-06-07 Author: Francesc Alted See LICENSES/BLOSC.txt for details about copyright and rights to use. **********************************************************************/ #include "test_common.h" int tests_run = 0; /* Global vars */ void *src, *srccpy, *dest, *dest2; size_t nbytes, cbytes; int clevel = 3; int doshuffle = 1; size_t typesize = 4; size_t size = 1*MB; static char *test_cbuffer_sizes() { size_t nbytes_, cbytes_, blocksize; blosc_cbuffer_sizes(dest, &nbytes_, &cbytes_, &blocksize); mu_assert("ERROR: nbytes incorrect(1)", nbytes == size); mu_assert("ERROR: nbytes incorrect(2)", nbytes_ == nbytes); mu_assert("ERROR: cbytes incorrect", cbytes == cbytes_); mu_assert("ERROR: blocksize incorrect", blocksize >= 128); return 0; } static char *test_cbuffer_metainfo() { size_t typesize_; int flags; blosc_cbuffer_metainfo(dest, &typesize_, &flags); mu_assert("ERROR: typesize incorrect", typesize_ == typesize); mu_assert("ERROR: shuffle incorrect", (flags & BLOSC_DOSHUFFLE) == doshuffle); return 0; } static char *test_cbuffer_versions() { int version_; int versionlz_; blosc_cbuffer_versions(dest, &version_, &versionlz_); mu_assert("ERROR: version incorrect", version_ == BLOSC_VERSION_FORMAT); mu_assert("ERROR: versionlz incorrect", versionlz_ == BLOSC_BLOSCLZ_VERSION_FORMAT); return 0; } static char *test_cbuffer_complib() { char *complib; complib = blosc_cbuffer_complib(dest); mu_assert("ERROR: complib incorrect", strcmp(complib, "BloscLZ") == 0); return 0; } static char *all_tests() { mu_run_test(test_cbuffer_sizes); mu_run_test(test_cbuffer_metainfo); mu_run_test(test_cbuffer_versions); mu_run_test(test_cbuffer_complib); return 0; } #define BUFFER_ALIGN_SIZE 8 int main(int argc, char **argv) { char *result; printf("STARTING TESTS for %s", argv[0]); blosc_init(); blosc_set_nthreads(1); /* Initialize buffers */ src = blosc_test_malloc(BUFFER_ALIGN_SIZE, size); srccpy = blosc_test_malloc(BUFFER_ALIGN_SIZE, size); dest = blosc_test_malloc(BUFFER_ALIGN_SIZE, size); dest2 = blosc_test_malloc(BUFFER_ALIGN_SIZE, size); memset(src, 0, size); memcpy(srccpy, src, size); /* Get a compressed buffer */ cbytes = blosc_compress(clevel, doshuffle, typesize, size, src, dest, size); /* Get a decompressed buffer */ nbytes = blosc_decompress(dest, dest2, size); /* Run all the suite */ result = all_tests(); if (result != 0) { printf(" (%s)\n", result); } else { printf(" ALL TESTS PASSED"); } printf("\tTests run: %d\n", tests_run); blosc_test_free(src); blosc_test_free(srccpy); blosc_test_free(dest); blosc_test_free(dest2); blosc_destroy(); return result != 0; } c-blosc-1.7.0/tests/test_common.h000066400000000000000000000076651254622165100167250ustar00rootroot00000000000000/********************************************************************* Blosc - Blocked Shuffling and Compression Library Unit tests for basic features in Blosc. Creation date: 2010-06-07 Author: Francesc Alted See LICENSES/BLOSC.txt for details about copyright and rights to use. **********************************************************************/ #ifndef BLOSC_TEST_COMMON_H #define BLOSC_TEST_COMMON_H #include #include #include #include #include #include #include #if defined(_WIN32) && !defined(__MINGW32__) #include #include "win32/stdint-windows.h" #else #include #include #include #endif #include #include "../blosc/blosc.h" /* This is MinUnit in action (http://www.jera.com/techinfo/jtns/jtn002.html) */ #define mu_assert(message, test) do { if (!(test)) return message; } while (0) #define mu_run_test(test) do \ { char *message = test(); tests_run++; \ if (message) { printf("%c", 'F'); return message;} \ else printf("%c", '.'); } while (0) extern int tests_run; #define KB 1024 #define MB (1024*KB) #define GB (1024*MB) /* Memory functions. */ /** Allocates a block of memory with the specified size and alignment. The allocated memory is 'cleaned' before returning to avoid accidental re-use of data within or between tests. */ static void* blosc_test_malloc(const size_t alignment, const size_t size) { const int32_t clean_value = 0x99; void *block = NULL; int32_t res = 0; #if _ISOC11_SOURCE /* C11 aligned allocation. 'size' must be a multiple of the alignment. */ block = aligned_alloc(alignment, size); #elif defined(_WIN32) /* A (void *) cast needed for avoiding a warning with MINGW :-/ */ block = (void *)_aligned_malloc(size, alignment); #elif _POSIX_C_SOURCE >= 200112L || _XOPEN_SOURCE >= 600 /* Platform does have an implementation of posix_memalign */ res = posix_memalign(&block, alignment, size); #elif defined(__APPLE__) /* Mac OS X guarantees 16-byte alignment in small allocs */ block = malloc(size); #else #error Cannot determine how to allocate aligned memory on the target platform. #endif if (block == NULL || res != 0) { fprintf(stderr, "Error allocating memory!"); return NULL; } /* Clean the allocated memory before returning. */ memset(block, clean_value, size); return block; } /** Frees memory allocated by blosc_test_malloc. */ static void blosc_test_free(void* ptr) { #if defined(_WIN32) _aligned_free(ptr); #else free(ptr); #endif /* _WIN32 */ } /** Fills a buffer with random values. */ static void blosc_test_fill_random(void* const ptr, const size_t size) { size_t k; uint8_t* const byte_ptr = (uint8_t*)ptr; for (k = 0; k < size; k++) { byte_ptr[k] = rand(); } } /* Argument parsing. */ /** Parse a `int32_t` value from a string, checking for overflow. */ static bool blosc_test_parse_uint32_t(const char* const str, uint32_t* value) { char* str_end; int32_t signed_value = strtol(str, &str_end, 10); if (signed_value < 0 || *str_end) { return false; } else { *value = (uint32_t)signed_value; return true; } } /* Error message functions. */ /** Print an error message when a test program has been invoked with an invalid number of arguments. */ static void blosc_test_print_bad_argcount_msg( const int32_t num_expected_args, const int32_t num_actual_args) { fprintf(stderr, "Invalid number of arguments specified.\nExpected %d arguments but was given %d.", num_expected_args, num_actual_args); } /** Print an error message when a test program has been invoked with an invalid argument value. */ static void blosc_test_print_bad_arg_msg(const int32_t arg_index) { fprintf(stderr, "Invalid value specified for argument at index %d.\n", arg_index); } #endif /* !defined(BLOSC_TEST_COMMON_H) */ c-blosc-1.7.0/tests/test_compress_roundtrip.c000066400000000000000000000075651254622165100213700ustar00rootroot00000000000000/********************************************************************* Blosc - Blocked Shuffling and Compression Library Roundtrip compression/decompression tests. Creation date: 2010-06-07 Author: Francesc Alted See LICENSES/BLOSC.txt for details about copyright and rights to use. **********************************************************************/ #include "test_common.h" #include "../blosc/shuffle.h" #include "../blosc/shuffle-generic.h" /** Perform a compress + decompress round trip. */ static int test_compress_roundtrip(size_t type_size, size_t num_elements, size_t buffer_alignment, int compression_level, bool do_shuffle) { size_t buffer_size = type_size * num_elements; /* Allocate memory for the test. */ void* original = blosc_test_malloc(buffer_alignment, buffer_size); void* intermediate = blosc_test_malloc(buffer_alignment, buffer_size + BLOSC_MAX_OVERHEAD); void* result = blosc_test_malloc(buffer_alignment, buffer_size); /* Fill the input data buffer with random values. */ blosc_test_fill_random(original, buffer_size); /* Compress the input data and store it in an intermediate buffer. Decompress the data from the intermediate buffer into a result buffer. */ blosc_compress(compression_level, do_shuffle, type_size, buffer_size, original, intermediate, buffer_size + BLOSC_MAX_OVERHEAD); blosc_decompress(intermediate, result, buffer_size); /* The round-tripped data matches the original data when the result of memcmp is 0. */ int exit_code = memcmp(original, result, buffer_size) ? EXIT_FAILURE : EXIT_SUCCESS; /* Free allocated memory. */ blosc_test_free(original); blosc_test_free(intermediate); blosc_test_free(result); return exit_code; } /** Required number of arguments to this test, including the executable name. */ #define TEST_ARG_COUNT 7 int main(int argc, char **argv) { /* argv[1]: sizeof(element type) argv[2]: number of elements argv[3]: buffer alignment argv[4]: compression level argv[5]: shuffle enabled argv[6]: thread count */ /* Verify the correct number of command-line args have been specified. */ if (TEST_ARG_COUNT != argc) { blosc_test_print_bad_argcount_msg(TEST_ARG_COUNT, argc); return EXIT_FAILURE; } /* Parse arguments */ uint32_t type_size; if (!blosc_test_parse_uint32_t(argv[1], &type_size) || (type_size < 1)) { blosc_test_print_bad_arg_msg(1); return EXIT_FAILURE; } uint32_t num_elements; if (!blosc_test_parse_uint32_t(argv[2], &num_elements) || (num_elements < 1)) { blosc_test_print_bad_arg_msg(2); return EXIT_FAILURE; } uint32_t buffer_align_size; if (!blosc_test_parse_uint32_t(argv[3], &buffer_align_size) || (buffer_align_size & (buffer_align_size - 1)) || (buffer_align_size < sizeof(void*))) { blosc_test_print_bad_arg_msg(3); return EXIT_FAILURE; } uint32_t compression_level; if (!blosc_test_parse_uint32_t(argv[4], &compression_level) || (compression_level > 9)) { blosc_test_print_bad_arg_msg(4); return EXIT_FAILURE; } bool shuffle_enabled; { uint32_t shuffle_enabled_raw; if (!blosc_test_parse_uint32_t(argv[5], &shuffle_enabled_raw) || (shuffle_enabled_raw > 1)) { blosc_test_print_bad_arg_msg(5); return EXIT_FAILURE; } shuffle_enabled = shuffle_enabled_raw == 0 ? false : true; } uint32_t blosc_thread_count; if (!blosc_test_parse_uint32_t(argv[6], &blosc_thread_count) || (blosc_thread_count < 1)) { blosc_test_print_bad_arg_msg(6); return EXIT_FAILURE; } /* Initialize blosc before running tests. */ blosc_init(); blosc_set_nthreads(blosc_thread_count); /* Run the test. */ int result = test_compress_roundtrip(type_size, num_elements, buffer_align_size, compression_level, shuffle_enabled); /* Cleanup blosc resources. */ blosc_destroy(); return result; } c-blosc-1.7.0/tests/test_compress_roundtrip.csv000066400000000000000000000106431254622165100217300ustar00rootroot00000000000000"Size of element type (bytes)","Number of elements","Buffer alignment size (bytes)","Compression level","Shuffle enabled","Blosc thread count" 1,7,32,5,0,1 1,192,32,5,0,1 1,1792,32,5,0,1 1,500,32,5,0,1 1,8000,32,5,0,1 1,100000,32,5,0,1 1,702713,32,5,0,1 2,7,32,5,0,1 2,192,32,5,0,1 2,1792,32,5,0,1 2,500,32,5,0,1 2,8000,32,5,0,1 2,100000,32,5,0,1 2,702713,32,5,0,1 3,7,32,5,0,1 3,192,32,5,0,1 3,1792,32,5,0,1 3,500,32,5,0,1 3,8000,32,5,0,1 3,100000,32,5,0,1 3,702713,32,5,0,1 4,7,32,5,0,1 4,192,32,5,0,1 4,1792,32,5,0,1 4,500,32,5,0,1 4,8000,32,5,0,1 4,100000,32,5,0,1 4,702713,32,5,0,1 5,7,32,5,0,1 5,192,32,5,0,1 5,1792,32,5,0,1 5,500,32,5,0,1 5,8000,32,5,0,1 5,100000,32,5,0,1 5,702713,32,5,0,1 6,7,32,5,0,1 6,192,32,5,0,1 6,1792,32,5,0,1 6,500,32,5,0,1 6,8000,32,5,0,1 6,100000,32,5,0,1 6,702713,32,5,0,1 7,7,32,5,0,1 7,192,32,5,0,1 7,1792,32,5,0,1 7,500,32,5,0,1 7,8000,32,5,0,1 7,100000,32,5,0,1 7,702713,32,5,0,1 8,7,32,5,0,1 8,192,32,5,0,1 8,1792,32,5,0,1 8,500,32,5,0,1 8,8000,32,5,0,1 8,100000,32,5,0,1 8,702713,32,5,0,1 11,7,32,5,0,1 11,192,32,5,0,1 11,1792,32,5,0,1 11,500,32,5,0,1 11,8000,32,5,0,1 11,100000,32,5,0,1 11,702713,32,5,0,1 16,7,32,5,0,1 16,192,32,5,0,1 16,1792,32,5,0,1 16,500,32,5,0,1 16,8000,32,5,0,1 16,100000,32,5,0,1 16,702713,32,5,0,1 22,7,32,5,0,1 22,192,32,5,0,1 22,1792,32,5,0,1 22,500,32,5,0,1 22,8000,32,5,0,1 22,100000,32,5,0,1 22,702713,32,5,0,1 30,7,32,5,0,1 30,192,32,5,0,1 30,1792,32,5,0,1 30,500,32,5,0,1 30,8000,32,5,0,1 30,100000,32,5,0,1 30,702713,32,5,0,1 32,7,32,5,0,1 32,192,32,5,0,1 32,1792,32,5,0,1 32,500,32,5,0,1 32,8000,32,5,0,1 32,100000,32,5,0,1 32,702713,32,5,0,1 42,7,32,5,0,1 42,192,32,5,0,1 42,1792,32,5,0,1 42,500,32,5,0,1 42,8000,32,5,0,1 42,100000,32,5,0,1 42,702713,32,5,0,1 48,7,32,5,0,1 48,192,32,5,0,1 48,1792,32,5,0,1 48,500,32,5,0,1 48,8000,32,5,0,1 48,100000,32,5,0,1 48,702713,32,5,0,1 52,7,32,5,0,1 52,192,32,5,0,1 52,1792,32,5,0,1 52,500,32,5,0,1 52,8000,32,5,0,1 52,100000,32,5,0,1 52,702713,32,5,0,1 53,7,32,5,0,1 53,192,32,5,0,1 53,1792,32,5,0,1 53,500,32,5,0,1 53,8000,32,5,0,1 53,100000,32,5,0,1 53,702713,32,5,0,1 64,7,32,5,0,1 64,192,32,5,0,1 64,1792,32,5,0,1 64,500,32,5,0,1 64,8000,32,5,0,1 64,100000,32,5,0,1 64,702713,32,5,0,1 80,7,32,5,0,1 80,192,32,5,0,1 80,1792,32,5,0,1 80,500,32,5,0,1 80,8000,32,5,0,1 80,100000,32,5,0,1 80,702713,32,5,0,1 1,7,32,5,1,1 1,192,32,5,1,1 1,1792,32,5,1,1 1,500,32,5,1,1 1,8000,32,5,1,1 1,100000,32,5,1,1 1,702713,32,5,1,1 2,7,32,5,1,1 2,192,32,5,1,1 2,1792,32,5,1,1 2,500,32,5,1,1 2,8000,32,5,1,1 2,100000,32,5,1,1 2,702713,32,5,1,1 3,7,32,5,1,1 3,192,32,5,1,1 3,1792,32,5,1,1 3,500,32,5,1,1 3,8000,32,5,1,1 3,100000,32,5,1,1 3,702713,32,5,1,1 4,7,32,5,1,1 4,192,32,5,1,1 4,1792,32,5,1,1 4,500,32,5,1,1 4,8000,32,5,1,1 4,100000,32,5,1,1 4,702713,32,5,1,1 5,7,32,5,1,1 5,192,32,5,1,1 5,1792,32,5,1,1 5,500,32,5,1,1 5,8000,32,5,1,1 5,100000,32,5,1,1 5,702713,32,5,1,1 6,7,32,5,1,1 6,192,32,5,1,1 6,1792,32,5,1,1 6,500,32,5,1,1 6,8000,32,5,1,1 6,100000,32,5,1,1 6,702713,32,5,1,1 7,7,32,5,1,1 7,192,32,5,1,1 7,1792,32,5,1,1 7,500,32,5,1,1 7,8000,32,5,1,1 7,100000,32,5,1,1 7,702713,32,5,1,1 8,7,32,5,1,1 8,192,32,5,1,1 8,1792,32,5,1,1 8,500,32,5,1,1 8,8000,32,5,1,1 8,100000,32,5,1,1 8,702713,32,5,1,1 11,7,32,5,1,1 11,192,32,5,1,1 11,1792,32,5,1,1 11,500,32,5,1,1 11,8000,32,5,1,1 11,100000,32,5,1,1 11,702713,32,5,1,1 16,7,32,5,1,1 16,192,32,5,1,1 16,1792,32,5,1,1 16,500,32,5,1,1 16,8000,32,5,1,1 16,100000,32,5,1,1 16,702713,32,5,1,1 22,7,32,5,1,1 22,192,32,5,1,1 22,1792,32,5,1,1 22,500,32,5,1,1 22,8000,32,5,1,1 22,100000,32,5,1,1 22,702713,32,5,1,1 30,7,32,5,1,1 30,192,32,5,1,1 30,1792,32,5,1,1 30,500,32,5,1,1 30,8000,32,5,1,1 30,100000,32,5,1,1 30,702713,32,5,1,1 32,7,32,5,1,1 32,192,32,5,1,1 32,1792,32,5,1,1 32,500,32,5,1,1 32,8000,32,5,1,1 32,100000,32,5,1,1 32,702713,32,5,1,1 42,7,32,5,1,1 42,192,32,5,1,1 42,1792,32,5,1,1 42,500,32,5,1,1 42,8000,32,5,1,1 42,100000,32,5,1,1 42,702713,32,5,1,1 48,7,32,5,1,1 48,192,32,5,1,1 48,1792,32,5,1,1 48,500,32,5,1,1 48,8000,32,5,1,1 48,100000,32,5,1,1 48,702713,32,5,1,1 52,7,32,5,1,1 52,192,32,5,1,1 52,1792,32,5,1,1 52,500,32,5,1,1 52,8000,32,5,1,1 52,100000,32,5,1,1 52,702713,32,5,1,1 53,7,32,5,1,1 53,192,32,5,1,1 53,1792,32,5,1,1 53,500,32,5,1,1 53,8000,32,5,1,1 53,100000,32,5,1,1 53,702713,32,5,1,1 64,7,32,5,1,1 64,192,32,5,1,1 64,1792,32,5,1,1 64,500,32,5,1,1 64,8000,32,5,1,1 64,100000,32,5,1,1 64,702713,32,5,1,1 80,7,32,5,1,1 80,192,32,5,1,1 80,1792,32,5,1,1 80,500,32,5,1,1 80,8000,32,5,1,1 80,100000,32,5,1,1 80,702713,32,5,1,1 c-blosc-1.7.0/tests/test_getitem.c000066400000000000000000000073731254622165100170620ustar00rootroot00000000000000/********************************************************************* Blosc - Blocked Shuffling and Compression Library Unit tests for the blosc_getitem() function. Creation date: 2010-06-07 Author: Francesc Alted See LICENSES/BLOSC.txt for details about copyright and rights to use. **********************************************************************/ #include "test_common.h" /** Test the blosc_getitem function. */ static int test_getitem(size_t type_size, size_t num_elements, size_t buffer_alignment, int compression_level, bool do_shuffle) { size_t buffer_size = type_size * num_elements; /* Allocate memory for the test. */ void* original = blosc_test_malloc(buffer_alignment, buffer_size); void* intermediate = blosc_test_malloc(buffer_alignment, buffer_size + BLOSC_MAX_OVERHEAD); void* result = blosc_test_malloc(buffer_alignment, buffer_size); /* Fill the input data buffer with random values. */ blosc_test_fill_random(original, buffer_size); /* Compress the input data, then use blosc_getitem to extract (decompress) a range of elements into a new buffer. */ blosc_compress(compression_level, do_shuffle, type_size, buffer_size, original, intermediate, buffer_size + BLOSC_MAX_OVERHEAD); blosc_getitem(intermediate, 0, num_elements, result); /* The round-tripped data matches the original data when the result of memcmp is 0. */ int exit_code = memcmp(original, result, buffer_size) ? EXIT_FAILURE : EXIT_SUCCESS; /* Free allocated memory. */ blosc_test_free(original); blosc_test_free(intermediate); blosc_test_free(result); return exit_code; } /** Required number of arguments to this test, including the executable name. */ #define TEST_ARG_COUNT 7 int main(int argc, char **argv) { /* argv[1]: sizeof(element type) argv[2]: number of elements argv[3]: buffer alignment argv[4]: compression level argv[5]: shuffle enabled argv[6]: thread count */ /* Verify the correct number of command-line args have been specified. */ if (TEST_ARG_COUNT != argc) { blosc_test_print_bad_argcount_msg(TEST_ARG_COUNT, argc); return EXIT_FAILURE; } /* Parse arguments */ uint32_t type_size; if (!blosc_test_parse_uint32_t(argv[1], &type_size) || (type_size < 1)) { blosc_test_print_bad_arg_msg(1); return EXIT_FAILURE; } uint32_t num_elements; if (!blosc_test_parse_uint32_t(argv[2], &num_elements) || (num_elements < 1)) { blosc_test_print_bad_arg_msg(2); return EXIT_FAILURE; } uint32_t buffer_align_size; if (!blosc_test_parse_uint32_t(argv[3], &buffer_align_size) || (buffer_align_size & (buffer_align_size - 1)) || (buffer_align_size < sizeof(void*))) { blosc_test_print_bad_arg_msg(3); return EXIT_FAILURE; } uint32_t compression_level; if (!blosc_test_parse_uint32_t(argv[4], &compression_level) || (compression_level > 9)) { blosc_test_print_bad_arg_msg(4); return EXIT_FAILURE; } bool shuffle_enabled; { uint32_t shuffle_enabled_raw; if (!blosc_test_parse_uint32_t(argv[5], &shuffle_enabled_raw) || (shuffle_enabled_raw > 1)) { blosc_test_print_bad_arg_msg(5); return EXIT_FAILURE; } shuffle_enabled = shuffle_enabled_raw == 0 ? false : true; } uint32_t blosc_thread_count; if (!blosc_test_parse_uint32_t(argv[6], &blosc_thread_count) || (blosc_thread_count < 1)) { blosc_test_print_bad_arg_msg(6); return EXIT_FAILURE; } /* Initialize blosc before running tests. */ blosc_init(); blosc_set_nthreads(blosc_thread_count); /* Run the test. */ int result = test_getitem(type_size, num_elements, buffer_align_size, compression_level, shuffle_enabled); /* Cleanup blosc resources. */ blosc_destroy(); return result; } c-blosc-1.7.0/tests/test_getitem.csv000066400000000000000000000106431254622165100174250ustar00rootroot00000000000000"Size of element type (bytes)","Number of elements","Buffer alignment size (bytes)","Compression level","Shuffle enabled","Blosc thread count" 1,7,32,5,0,1 1,192,32,5,0,1 1,1792,32,5,0,1 1,500,32,5,0,1 1,8000,32,5,0,1 1,100000,32,5,0,1 1,702713,32,5,0,1 2,7,32,5,0,1 2,192,32,5,0,1 2,1792,32,5,0,1 2,500,32,5,0,1 2,8000,32,5,0,1 2,100000,32,5,0,1 2,702713,32,5,0,1 3,7,32,5,0,1 3,192,32,5,0,1 3,1792,32,5,0,1 3,500,32,5,0,1 3,8000,32,5,0,1 3,100000,32,5,0,1 3,702713,32,5,0,1 4,7,32,5,0,1 4,192,32,5,0,1 4,1792,32,5,0,1 4,500,32,5,0,1 4,8000,32,5,0,1 4,100000,32,5,0,1 4,702713,32,5,0,1 5,7,32,5,0,1 5,192,32,5,0,1 5,1792,32,5,0,1 5,500,32,5,0,1 5,8000,32,5,0,1 5,100000,32,5,0,1 5,702713,32,5,0,1 6,7,32,5,0,1 6,192,32,5,0,1 6,1792,32,5,0,1 6,500,32,5,0,1 6,8000,32,5,0,1 6,100000,32,5,0,1 6,702713,32,5,0,1 7,7,32,5,0,1 7,192,32,5,0,1 7,1792,32,5,0,1 7,500,32,5,0,1 7,8000,32,5,0,1 7,100000,32,5,0,1 7,702713,32,5,0,1 8,7,32,5,0,1 8,192,32,5,0,1 8,1792,32,5,0,1 8,500,32,5,0,1 8,8000,32,5,0,1 8,100000,32,5,0,1 8,702713,32,5,0,1 11,7,32,5,0,1 11,192,32,5,0,1 11,1792,32,5,0,1 11,500,32,5,0,1 11,8000,32,5,0,1 11,100000,32,5,0,1 11,702713,32,5,0,1 16,7,32,5,0,1 16,192,32,5,0,1 16,1792,32,5,0,1 16,500,32,5,0,1 16,8000,32,5,0,1 16,100000,32,5,0,1 16,702713,32,5,0,1 22,7,32,5,0,1 22,192,32,5,0,1 22,1792,32,5,0,1 22,500,32,5,0,1 22,8000,32,5,0,1 22,100000,32,5,0,1 22,702713,32,5,0,1 30,7,32,5,0,1 30,192,32,5,0,1 30,1792,32,5,0,1 30,500,32,5,0,1 30,8000,32,5,0,1 30,100000,32,5,0,1 30,702713,32,5,0,1 32,7,32,5,0,1 32,192,32,5,0,1 32,1792,32,5,0,1 32,500,32,5,0,1 32,8000,32,5,0,1 32,100000,32,5,0,1 32,702713,32,5,0,1 42,7,32,5,0,1 42,192,32,5,0,1 42,1792,32,5,0,1 42,500,32,5,0,1 42,8000,32,5,0,1 42,100000,32,5,0,1 42,702713,32,5,0,1 48,7,32,5,0,1 48,192,32,5,0,1 48,1792,32,5,0,1 48,500,32,5,0,1 48,8000,32,5,0,1 48,100000,32,5,0,1 48,702713,32,5,0,1 52,7,32,5,0,1 52,192,32,5,0,1 52,1792,32,5,0,1 52,500,32,5,0,1 52,8000,32,5,0,1 52,100000,32,5,0,1 52,702713,32,5,0,1 53,7,32,5,0,1 53,192,32,5,0,1 53,1792,32,5,0,1 53,500,32,5,0,1 53,8000,32,5,0,1 53,100000,32,5,0,1 53,702713,32,5,0,1 64,7,32,5,0,1 64,192,32,5,0,1 64,1792,32,5,0,1 64,500,32,5,0,1 64,8000,32,5,0,1 64,100000,32,5,0,1 64,702713,32,5,0,1 80,7,32,5,0,1 80,192,32,5,0,1 80,1792,32,5,0,1 80,500,32,5,0,1 80,8000,32,5,0,1 80,100000,32,5,0,1 80,702713,32,5,0,1 1,7,32,5,1,1 1,192,32,5,1,1 1,1792,32,5,1,1 1,500,32,5,1,1 1,8000,32,5,1,1 1,100000,32,5,1,1 1,702713,32,5,1,1 2,7,32,5,1,1 2,192,32,5,1,1 2,1792,32,5,1,1 2,500,32,5,1,1 2,8000,32,5,1,1 2,100000,32,5,1,1 2,702713,32,5,1,1 3,7,32,5,1,1 3,192,32,5,1,1 3,1792,32,5,1,1 3,500,32,5,1,1 3,8000,32,5,1,1 3,100000,32,5,1,1 3,702713,32,5,1,1 4,7,32,5,1,1 4,192,32,5,1,1 4,1792,32,5,1,1 4,500,32,5,1,1 4,8000,32,5,1,1 4,100000,32,5,1,1 4,702713,32,5,1,1 5,7,32,5,1,1 5,192,32,5,1,1 5,1792,32,5,1,1 5,500,32,5,1,1 5,8000,32,5,1,1 5,100000,32,5,1,1 5,702713,32,5,1,1 6,7,32,5,1,1 6,192,32,5,1,1 6,1792,32,5,1,1 6,500,32,5,1,1 6,8000,32,5,1,1 6,100000,32,5,1,1 6,702713,32,5,1,1 7,7,32,5,1,1 7,192,32,5,1,1 7,1792,32,5,1,1 7,500,32,5,1,1 7,8000,32,5,1,1 7,100000,32,5,1,1 7,702713,32,5,1,1 8,7,32,5,1,1 8,192,32,5,1,1 8,1792,32,5,1,1 8,500,32,5,1,1 8,8000,32,5,1,1 8,100000,32,5,1,1 8,702713,32,5,1,1 11,7,32,5,1,1 11,192,32,5,1,1 11,1792,32,5,1,1 11,500,32,5,1,1 11,8000,32,5,1,1 11,100000,32,5,1,1 11,702713,32,5,1,1 16,7,32,5,1,1 16,192,32,5,1,1 16,1792,32,5,1,1 16,500,32,5,1,1 16,8000,32,5,1,1 16,100000,32,5,1,1 16,702713,32,5,1,1 22,7,32,5,1,1 22,192,32,5,1,1 22,1792,32,5,1,1 22,500,32,5,1,1 22,8000,32,5,1,1 22,100000,32,5,1,1 22,702713,32,5,1,1 30,7,32,5,1,1 30,192,32,5,1,1 30,1792,32,5,1,1 30,500,32,5,1,1 30,8000,32,5,1,1 30,100000,32,5,1,1 30,702713,32,5,1,1 32,7,32,5,1,1 32,192,32,5,1,1 32,1792,32,5,1,1 32,500,32,5,1,1 32,8000,32,5,1,1 32,100000,32,5,1,1 32,702713,32,5,1,1 42,7,32,5,1,1 42,192,32,5,1,1 42,1792,32,5,1,1 42,500,32,5,1,1 42,8000,32,5,1,1 42,100000,32,5,1,1 42,702713,32,5,1,1 48,7,32,5,1,1 48,192,32,5,1,1 48,1792,32,5,1,1 48,500,32,5,1,1 48,8000,32,5,1,1 48,100000,32,5,1,1 48,702713,32,5,1,1 52,7,32,5,1,1 52,192,32,5,1,1 52,1792,32,5,1,1 52,500,32,5,1,1 52,8000,32,5,1,1 52,100000,32,5,1,1 52,702713,32,5,1,1 53,7,32,5,1,1 53,192,32,5,1,1 53,1792,32,5,1,1 53,500,32,5,1,1 53,8000,32,5,1,1 53,100000,32,5,1,1 53,702713,32,5,1,1 64,7,32,5,1,1 64,192,32,5,1,1 64,1792,32,5,1,1 64,500,32,5,1,1 64,8000,32,5,1,1 64,100000,32,5,1,1 64,702713,32,5,1,1 80,7,32,5,1,1 80,192,32,5,1,1 80,1792,32,5,1,1 80,500,32,5,1,1 80,8000,32,5,1,1 80,100000,32,5,1,1 80,702713,32,5,1,1 c-blosc-1.7.0/tests/test_maxout.c000066400000000000000000000054031254622165100167310ustar00rootroot00000000000000/********************************************************************* Blosc - Blocked Shuffling and Compression Library Unit tests for basic features in Blosc. Creation date: 2010-06-07 Author: Francesc Alted See LICENSES/BLOSC.txt for details about copyright and rights to use. **********************************************************************/ #include "test_common.h" int tests_run = 0; /* Global vars */ void *src, *srccpy, *dest, *dest2; size_t nbytes, cbytes; int clevel = 1; int doshuffle = 0; size_t typesize = 4; size_t size = 1000; /* must be divisible by 4 */ /* Check maxout with maxout < size */ static char *test_maxout_less() { /* Get a compressed buffer */ cbytes = blosc_compress(clevel, doshuffle, typesize, size, src, dest, size+15); mu_assert("ERROR: cbytes is not 0", cbytes == 0); return 0; } /* Check maxout with maxout == size */ static char *test_maxout_equal() { /* Get a compressed buffer */ cbytes = blosc_compress(clevel, doshuffle, typesize, size, src, dest, size+16); mu_assert("ERROR: cbytes is not correct", cbytes == size+16); /* Decompress the buffer */ nbytes = blosc_decompress(dest, dest2, size); mu_assert("ERROR: nbytes incorrect(1)", nbytes == size); return 0; } /* Check maxout with maxout > size */ static char *test_maxout_great() { /* Get a compressed buffer */ cbytes = blosc_compress(clevel, doshuffle, typesize, size, src, dest, size+17); mu_assert("ERROR: cbytes is not 0", cbytes == size+16); /* Decompress the buffer */ nbytes = blosc_decompress(dest, dest2, size); mu_assert("ERROR: nbytes incorrect(1)", nbytes == size); return 0; } static char *all_tests() { mu_run_test(test_maxout_less); mu_run_test(test_maxout_equal); mu_run_test(test_maxout_great); return 0; } #define BUFFER_ALIGN_SIZE 32 int main(int argc, char **argv) { int32_t *_src; char *result; size_t i; printf("STARTING TESTS for %s", argv[0]); blosc_init(); blosc_set_nthreads(1); /* Initialize buffers */ src = blosc_test_malloc(BUFFER_ALIGN_SIZE, size); srccpy = blosc_test_malloc(BUFFER_ALIGN_SIZE, size); dest = blosc_test_malloc(BUFFER_ALIGN_SIZE, size + 16); dest2 = blosc_test_malloc(BUFFER_ALIGN_SIZE, size); _src = (int32_t *)src; for (i=0; i < (size/4); i++) { _src[i] = (int32_t)i; } memcpy(srccpy, src, size); /* Run all the suite */ result = all_tests(); if (result != 0) { printf(" (%s)\n", result); } else { printf(" ALL TESTS PASSED"); } printf("\tTests run: %d\n", tests_run); blosc_test_free(src); blosc_test_free(srccpy); blosc_test_free(dest); blosc_test_free(dest2); blosc_destroy(); return result != 0; } c-blosc-1.7.0/tests/test_shuffle_roundtrip_avx2.c000066400000000000000000000077571254622165100221340ustar00rootroot00000000000000/********************************************************************* Blosc - Blocked Shuffling and Compression Library Roundtrip tests for the AVX2-accelerated shuffle/unshuffle. Creation date: 2010-06-07 Author: Francesc Alted See LICENSES/BLOSC.txt for details about copyright and rights to use. **********************************************************************/ #include "test_common.h" #include "../blosc/shuffle.h" #include "../blosc/shuffle-generic.h" /* Include accelerated shuffles if supported by this compiler. TODO: Need to also do run-time CPU feature support here. */ #if defined(SHUFFLE_AVX2_ENABLED) #include "../blosc/shuffle-avx2.h" #else #if defined(_MSC_VER) #pragma message("AVX2 shuffle tests not enabled.") #else #warning AVX2 shuffle tests not enabled. #endif #endif /* defined(SHUFFLE_AVX2_ENABLED) */ /** Roundtrip tests for the AVX2-accelerated shuffle/unshuffle. */ static int test_shuffle_roundtrip_avx2(size_t type_size, size_t num_elements, size_t buffer_alignment, int test_type) { #if defined(SHUFFLE_AVX2_ENABLED) size_t buffer_size = type_size * num_elements; /* Allocate memory for the test. */ void* original = blosc_test_malloc(buffer_alignment, buffer_size); void* shuffled = blosc_test_malloc(buffer_alignment, buffer_size); void* unshuffled = blosc_test_malloc(buffer_alignment, buffer_size); /* Fill the input data buffer with random values. */ blosc_test_fill_random(original, buffer_size); /* Shuffle/unshuffle, selecting the implementations based on the test type. */ switch(test_type) { case 0: /* avx2/avx2 */ shuffle_avx2(type_size, buffer_size, original, shuffled); unshuffle_avx2(type_size, buffer_size, shuffled, unshuffled); break; case 1: /* generic/avx2 */ shuffle_generic(type_size, buffer_size, original, shuffled); unshuffle_avx2(type_size, buffer_size, shuffled, unshuffled); break; case 2: /* avx2/generic */ shuffle_avx2(type_size, buffer_size, original, shuffled); unshuffle_generic(type_size, buffer_size, shuffled, unshuffled); break; default: fprintf(stderr, "Invalid test type specified (%d).", test_type); return EXIT_FAILURE; } /* The round-tripped data matches the original data when the result of memcmp is 0. */ int exit_code = memcmp(original, unshuffled, buffer_size) ? EXIT_FAILURE : EXIT_SUCCESS; /* Free allocated memory. */ blosc_test_free(original); blosc_test_free(shuffled); blosc_test_free(unshuffled); return exit_code; #else return EXIT_SUCCESS; #endif /* defined(SHUFFLE_AVX2_ENABLED) */ } /** Required number of arguments to this test, including the executable name. */ #define TEST_ARG_COUNT 5 int main(int argc, char **argv) { /* argv[1]: sizeof(element type) argv[2]: number of elements argv[3]: buffer alignment argv[4]: test type */ /* Verify the correct number of command-line args have been specified. */ if (TEST_ARG_COUNT != argc) { blosc_test_print_bad_argcount_msg(TEST_ARG_COUNT, argc); return EXIT_FAILURE; } /* Parse arguments */ uint32_t type_size; if (!blosc_test_parse_uint32_t(argv[1], &type_size) || (type_size < 1)) { blosc_test_print_bad_arg_msg(1); return EXIT_FAILURE; } uint32_t num_elements; if (!blosc_test_parse_uint32_t(argv[2], &num_elements) || (num_elements < 1)) { blosc_test_print_bad_arg_msg(2); return EXIT_FAILURE; } uint32_t buffer_align_size; if (!blosc_test_parse_uint32_t(argv[3], &buffer_align_size) || (buffer_align_size & (buffer_align_size - 1)) || (buffer_align_size < sizeof(void*))) { blosc_test_print_bad_arg_msg(3); return EXIT_FAILURE; } uint32_t test_type; if (!blosc_test_parse_uint32_t(argv[4], &test_type) || (test_type > 2)) { blosc_test_print_bad_arg_msg(4); return EXIT_FAILURE; } /* Run the test. */ return test_shuffle_roundtrip_avx2(type_size, num_elements, buffer_align_size, test_type); } c-blosc-1.7.0/tests/test_shuffle_roundtrip_avx2.csv000066400000000000000000000117011254622165100224650ustar00rootroot00000000000000"Size of element type (bytes)","Number of elements","Buffer alignment size (bytes)","Test type" 1,7,32,0 1,7,32,1 1,7,32,2 1,192,32,0 1,192,32,1 1,192,32,2 1,1792,32,0 1,1792,32,1 1,1792,32,2 1,500,32,0 1,500,32,1 1,500,32,2 1,8000,32,0 1,8000,32,1 1,8000,32,2 1,100000,32,0 1,100000,32,1 1,100000,32,2 1,702713,32,0 1,702713,32,1 1,702713,32,2 2,7,32,0 2,7,32,1 2,7,32,2 2,192,32,0 2,192,32,1 2,192,32,2 2,1792,32,0 2,1792,32,1 2,1792,32,2 2,500,32,0 2,500,32,1 2,500,32,2 2,8000,32,0 2,8000,32,1 2,8000,32,2 2,100000,32,0 2,100000,32,1 2,100000,32,2 2,702713,32,0 2,702713,32,1 2,702713,32,2 3,7,32,0 3,7,32,1 3,7,32,2 3,192,32,0 3,192,32,1 3,192,32,2 3,1792,32,0 3,1792,32,1 3,1792,32,2 3,500,32,0 3,500,32,1 3,500,32,2 3,8000,32,0 3,8000,32,1 3,8000,32,2 3,100000,32,0 3,100000,32,1 3,100000,32,2 3,702713,32,0 3,702713,32,1 3,702713,32,2 4,7,32,0 4,7,32,1 4,7,32,2 4,192,32,0 4,192,32,1 4,192,32,2 4,1792,32,0 4,1792,32,1 4,1792,32,2 4,500,32,0 4,500,32,1 4,500,32,2 4,8000,32,0 4,8000,32,1 4,8000,32,2 4,100000,32,0 4,100000,32,1 4,100000,32,2 4,702713,32,0 4,702713,32,1 4,702713,32,2 5,7,32,0 5,7,32,1 5,7,32,2 5,192,32,0 5,192,32,1 5,192,32,2 5,1792,32,0 5,1792,32,1 5,1792,32,2 5,500,32,0 5,500,32,1 5,500,32,2 5,8000,32,0 5,8000,32,1 5,8000,32,2 5,100000,32,0 5,100000,32,1 5,100000,32,2 5,702713,32,0 5,702713,32,1 5,702713,32,2 6,7,32,0 6,7,32,1 6,7,32,2 6,192,32,0 6,192,32,1 6,192,32,2 6,1792,32,0 6,1792,32,1 6,1792,32,2 6,500,32,0 6,500,32,1 6,500,32,2 6,8000,32,0 6,8000,32,1 6,8000,32,2 6,100000,32,0 6,100000,32,1 6,100000,32,2 6,702713,32,0 6,702713,32,1 6,702713,32,2 7,7,32,0 7,7,32,1 7,7,32,2 7,192,32,0 7,192,32,1 7,192,32,2 7,1792,32,0 7,1792,32,1 7,1792,32,2 7,500,32,0 7,500,32,1 7,500,32,2 7,8000,32,0 7,8000,32,1 7,8000,32,2 7,100000,32,0 7,100000,32,1 7,100000,32,2 7,702713,32,0 7,702713,32,1 7,702713,32,2 8,7,32,0 8,7,32,1 8,7,32,2 8,192,32,0 8,192,32,1 8,192,32,2 8,1792,32,0 8,1792,32,1 8,1792,32,2 8,500,32,0 8,500,32,1 8,500,32,2 8,8000,32,0 8,8000,32,1 8,8000,32,2 8,100000,32,0 8,100000,32,1 8,100000,32,2 8,702713,32,0 8,702713,32,1 8,702713,32,2 11,7,32,0 11,7,32,1 11,7,32,2 11,192,32,0 11,192,32,1 11,192,32,2 11,1792,32,0 11,1792,32,1 11,1792,32,2 11,500,32,0 11,500,32,1 11,500,32,2 11,8000,32,0 11,8000,32,1 11,8000,32,2 11,100000,32,0 11,100000,32,1 11,100000,32,2 11,702713,32,0 11,702713,32,1 11,702713,32,2 16,7,32,0 16,7,32,1 16,7,32,2 16,192,32,0 16,192,32,1 16,192,32,2 16,1792,32,0 16,1792,32,1 16,1792,32,2 16,500,32,0 16,500,32,1 16,500,32,2 16,8000,32,0 16,8000,32,1 16,8000,32,2 16,100000,32,0 16,100000,32,1 16,100000,32,2 16,702713,32,0 16,702713,32,1 16,702713,32,2 22,7,32,0 22,7,32,1 22,7,32,2 22,192,32,0 22,192,32,1 22,192,32,2 22,1792,32,0 22,1792,32,1 22,1792,32,2 22,500,32,0 22,500,32,1 22,500,32,2 22,8000,32,0 22,8000,32,1 22,8000,32,2 22,100000,32,0 22,100000,32,1 22,100000,32,2 22,702713,32,0 22,702713,32,1 22,702713,32,2 30,7,32,0 30,7,32,1 30,7,32,2 30,192,32,0 30,192,32,1 30,192,32,2 30,1792,32,0 30,1792,32,1 30,1792,32,2 30,500,32,0 30,500,32,1 30,500,32,2 30,8000,32,0 30,8000,32,1 30,8000,32,2 30,100000,32,0 30,100000,32,1 30,100000,32,2 30,702713,32,0 30,702713,32,1 30,702713,32,2 32,7,32,0 32,7,32,1 32,7,32,2 32,192,32,0 32,192,32,1 32,192,32,2 32,1792,32,0 32,1792,32,1 32,1792,32,2 32,500,32,0 32,500,32,1 32,500,32,2 32,8000,32,0 32,8000,32,1 32,8000,32,2 32,100000,32,0 32,100000,32,1 32,100000,32,2 32,702713,32,0 32,702713,32,1 32,702713,32,2 42,7,32,0 42,7,32,1 42,7,32,2 42,192,32,0 42,192,32,1 42,192,32,2 42,1792,32,0 42,1792,32,1 42,1792,32,2 42,500,32,0 42,500,32,1 42,500,32,2 42,8000,32,0 42,8000,32,1 42,8000,32,2 42,100000,32,0 42,100000,32,1 42,100000,32,2 42,702713,32,0 42,702713,32,1 42,702713,32,2 48,7,32,0 48,7,32,1 48,7,32,2 48,192,32,0 48,192,32,1 48,192,32,2 48,1792,32,0 48,1792,32,1 48,1792,32,2 48,500,32,0 48,500,32,1 48,500,32,2 48,8000,32,0 48,8000,32,1 48,8000,32,2 48,100000,32,0 48,100000,32,1 48,100000,32,2 48,702713,32,0 48,702713,32,1 48,702713,32,2 52,7,32,0 52,7,32,1 52,7,32,2 52,192,32,0 52,192,32,1 52,192,32,2 52,1792,32,0 52,1792,32,1 52,1792,32,2 52,500,32,0 52,500,32,1 52,500,32,2 52,8000,32,0 52,8000,32,1 52,8000,32,2 52,100000,32,0 52,100000,32,1 52,100000,32,2 52,702713,32,0 52,702713,32,1 52,702713,32,2 53,7,32,0 53,7,32,1 53,7,32,2 53,192,32,0 53,192,32,1 53,192,32,2 53,1792,32,0 53,1792,32,1 53,1792,32,2 53,500,32,0 53,500,32,1 53,500,32,2 53,8000,32,0 53,8000,32,1 53,8000,32,2 53,100000,32,0 53,100000,32,1 53,100000,32,2 53,702713,32,0 53,702713,32,1 53,702713,32,2 64,7,32,0 64,7,32,1 64,7,32,2 64,192,32,0 64,192,32,1 64,192,32,2 64,1792,32,0 64,1792,32,1 64,1792,32,2 64,500,32,0 64,500,32,1 64,500,32,2 64,8000,32,0 64,8000,32,1 64,8000,32,2 64,100000,32,0 64,100000,32,1 64,100000,32,2 64,702713,32,0 64,702713,32,1 64,702713,32,2 80,7,32,0 80,7,32,1 80,7,32,2 80,192,32,0 80,192,32,1 80,192,32,2 80,1792,32,0 80,1792,32,1 80,1792,32,2 80,500,32,0 80,500,32,1 80,500,32,2 80,8000,32,0 80,8000,32,1 80,8000,32,2 80,100000,32,0 80,100000,32,1 80,100000,32,2 80,702713,32,0 80,702713,32,1 80,702713,32,2c-blosc-1.7.0/tests/test_shuffle_roundtrip_generic.c000066400000000000000000000052531254622165100226550ustar00rootroot00000000000000/********************************************************************* Blosc - Blocked Shuffling and Compression Library Roundtrip tests Creation date: 2010-06-07 Author: Francesc Alted See LICENSES/BLOSC.txt for details about copyright and rights to use. **********************************************************************/ #include "test_common.h" #include "../blosc/shuffle.h" #include "../blosc/shuffle-generic.h" /** Roundtrip tests for the generic shuffle/unshuffle. */ static int test_shuffle_roundtrip_generic(size_t type_size, size_t num_elements, size_t buffer_alignment) { size_t buffer_size = type_size * num_elements; /* Allocate memory for the test. */ void* original = blosc_test_malloc(buffer_alignment, buffer_size); void* shuffled = blosc_test_malloc(buffer_alignment, buffer_size); void* unshuffled = blosc_test_malloc(buffer_alignment, buffer_size); /* Fill the input data buffer with random values. */ blosc_test_fill_random(original, buffer_size); /* Generic shuffle, then generic unshuffle. */ shuffle_generic(type_size, buffer_size, original, shuffled); unshuffle_generic(type_size, buffer_size, shuffled, unshuffled); /* The round-tripped data matches the original data when the result of memcmp is 0. */ int exit_code = memcmp(original, unshuffled, buffer_size) ? EXIT_FAILURE : EXIT_SUCCESS; /* Free allocated memory. */ blosc_test_free(original); blosc_test_free(shuffled); blosc_test_free(unshuffled); return exit_code; } /** Required number of arguments to this test, including the executable name. */ #define TEST_ARG_COUNT 4 int main(int argc, char **argv) { /* argv[1]: sizeof(element type) argv[2]: number of elements argv[3]: buffer alignment */ /* Verify the correct number of command-line args have been specified. */ if (TEST_ARG_COUNT != argc) { blosc_test_print_bad_argcount_msg(TEST_ARG_COUNT, argc); return EXIT_FAILURE; } /* Parse arguments */ uint32_t type_size; if (!blosc_test_parse_uint32_t(argv[1], &type_size) || (type_size < 1)) { blosc_test_print_bad_arg_msg(1); return EXIT_FAILURE; } uint32_t num_elements; if (!blosc_test_parse_uint32_t(argv[2], &num_elements) || (num_elements < 1)) { blosc_test_print_bad_arg_msg(2); return EXIT_FAILURE; } uint32_t buffer_align_size; if (!blosc_test_parse_uint32_t(argv[3], &buffer_align_size) || (buffer_align_size & (buffer_align_size - 1)) || (buffer_align_size < sizeof(void*))) { blosc_test_print_bad_arg_msg(3); return EXIT_FAILURE; } /* Run the test. */ return test_shuffle_roundtrip_generic(type_size, num_elements, buffer_align_size); } c-blosc-1.7.0/tests/test_shuffle_roundtrip_generic.csv000066400000000000000000000024721254622165100232260ustar00rootroot00000000000000"Size of element type (bytes)","Number of elements","Buffer alignment size (bytes)" 1,7,8 1,192,8 1,1792,8 1,500,8 1,8000,8 1,100000,8 1,702713,8 2,7,8 2,192,8 2,1792,8 2,500,8 2,8000,8 2,100000,8 2,702713,8 3,7,8 3,192,8 3,1792,8 3,500,8 3,8000,8 3,100000,8 3,702713,8 4,7,8 4,192,8 4,1792,8 4,500,8 4,8000,8 4,100000,8 4,702713,8 5,7,8 5,192,8 5,1792,8 5,500,8 5,8000,8 5,100000,8 5,702713,8 6,7,8 6,192,8 6,1792,8 6,500,8 6,8000,8 6,100000,8 6,702713,8 7,7,8 7,192,8 7,1792,8 7,500,8 7,8000,8 7,100000,8 7,702713,8 8,7,8 8,192,8 8,1792,8 8,500,8 8,8000,8 8,100000,8 8,702713,8 11,7,8 11,192,8 11,1792,8 11,500,8 11,8000,8 11,100000,8 11,702713,8 16,7,8 16,192,8 16,1792,8 16,500,8 16,8000,8 16,100000,8 16,702713,8 22,7,8 22,192,8 22,1792,8 22,500,8 22,8000,8 22,100000,8 22,702713,8 30,7,8 30,192,8 30,1792,8 30,500,8 30,8000,8 30,100000,8 30,702713,8 32,7,8 32,192,8 32,1792,8 32,500,8 32,8000,8 32,100000,8 32,702713,8 42,7,8 42,192,8 42,1792,8 42,500,8 42,8000,8 42,100000,8 42,702713,8 48,7,8 48,192,8 48,1792,8 48,500,8 48,8000,8 48,100000,8 48,702713,8 52,7,8 52,192,8 52,1792,8 52,500,8 52,8000,8 52,100000,8 52,702713,8 53,7,8 53,192,8 53,1792,8 53,500,8 53,8000,8 53,100000,8 53,702713,8 64,7,8 64,192,8 64,1792,8 64,500,8 64,8000,8 64,100000,8 64,702713,8 80,7,8 80,192,8 80,1792,8 80,500,8 80,8000,8 80,100000,8 80,702713,8c-blosc-1.7.0/tests/test_shuffle_roundtrip_sse2.c000066400000000000000000000100021254622165100221010ustar00rootroot00000000000000/********************************************************************* Blosc - Blocked Shuffling and Compression Library Roundtrip tests for the SSE2-accelerated shuffle/unshuffle. Creation date: 2010-06-07 Author: Francesc Alted See LICENSES/BLOSC.txt for details about copyright and rights to use. **********************************************************************/ #include "test_common.h" #include "../blosc/shuffle.h" #include "../blosc/shuffle-generic.h" /* Include SSE2-accelerated shuffle implementation if supported by this compiler. TODO: Need to also do run-time CPU feature support here. */ #if defined(SHUFFLE_SSE2_ENABLED) #include "../blosc/shuffle-sse2.h" #else #if defined(_MSC_VER) #pragma message("SSE2 shuffle tests not enabled.") #else #warning SSE2 shuffle tests not enabled. #endif #endif /* defined(SHUFFLE_SSE2_ENABLED) */ /** Roundtrip tests for the SSE2-accelerated shuffle/unshuffle. */ static int test_shuffle_roundtrip_sse2(size_t type_size, size_t num_elements, size_t buffer_alignment, int test_type) { #if defined(SHUFFLE_SSE2_ENABLED) size_t buffer_size = type_size * num_elements; /* Allocate memory for the test. */ void* original = blosc_test_malloc(buffer_alignment, buffer_size); void* shuffled = blosc_test_malloc(buffer_alignment, buffer_size); void* unshuffled = blosc_test_malloc(buffer_alignment, buffer_size); /* Fill the input data buffer with random values. */ blosc_test_fill_random(original, buffer_size); /* Shuffle/unshuffle, selecting the implementations based on the test type. */ switch(test_type) { case 0: /* sse2/sse2 */ shuffle_sse2(type_size, buffer_size, original, shuffled); unshuffle_sse2(type_size, buffer_size, shuffled, unshuffled); break; case 1: /* generic/sse2 */ shuffle_generic(type_size, buffer_size, original, shuffled); unshuffle_sse2(type_size, buffer_size, shuffled, unshuffled); break; case 2: /* sse2/generic */ shuffle_sse2(type_size, buffer_size, original, shuffled); unshuffle_generic(type_size, buffer_size, shuffled, unshuffled); break; default: fprintf(stderr, "Invalid test type specified (%d).", test_type); return EXIT_FAILURE; } /* The round-tripped data matches the original data when the result of memcmp is 0. */ int exit_code = memcmp(original, unshuffled, buffer_size) ? EXIT_FAILURE : EXIT_SUCCESS; /* Free allocated memory. */ blosc_test_free(original); blosc_test_free(shuffled); blosc_test_free(unshuffled); return exit_code; #else return EXIT_SUCCESS; #endif /* defined(SHUFFLE_SSE2_ENABLED) */ } /** Required number of arguments to this test, including the executable name. */ #define TEST_ARG_COUNT 5 int main(int argc, char **argv) { /* argv[1]: sizeof(element type) argv[2]: number of elements argv[3]: buffer alignment argv[4]: test type */ /* Verify the correct number of command-line args have been specified. */ if (TEST_ARG_COUNT != argc) { blosc_test_print_bad_argcount_msg(TEST_ARG_COUNT, argc); return EXIT_FAILURE; } /* Parse arguments */ uint32_t type_size; if (!blosc_test_parse_uint32_t(argv[1], &type_size) || (type_size < 1)) { blosc_test_print_bad_arg_msg(1); return EXIT_FAILURE; } uint32_t num_elements; if (!blosc_test_parse_uint32_t(argv[2], &num_elements) || (num_elements < 1)) { blosc_test_print_bad_arg_msg(2); return EXIT_FAILURE; } uint32_t buffer_align_size; if (!blosc_test_parse_uint32_t(argv[3], &buffer_align_size) || (buffer_align_size & (buffer_align_size - 1)) || (buffer_align_size < sizeof(void*))) { blosc_test_print_bad_arg_msg(3); return EXIT_FAILURE; } uint32_t test_type; if (!blosc_test_parse_uint32_t(argv[4], &test_type) || (test_type > 2)) { blosc_test_print_bad_arg_msg(4); return EXIT_FAILURE; } /* Run the test. */ return test_shuffle_roundtrip_sse2(type_size, num_elements, buffer_align_size, test_type); } c-blosc-1.7.0/tests/test_shuffle_roundtrip_sse2.csv000066400000000000000000000117011254622165100224610ustar00rootroot00000000000000"Size of element type (bytes)","Number of elements","Buffer alignment size (bytes)","Test type" 1,7,32,0 1,7,32,1 1,7,32,2 1,192,32,0 1,192,32,1 1,192,32,2 1,1792,32,0 1,1792,32,1 1,1792,32,2 1,500,32,0 1,500,32,1 1,500,32,2 1,8000,32,0 1,8000,32,1 1,8000,32,2 1,100000,32,0 1,100000,32,1 1,100000,32,2 1,702713,32,0 1,702713,32,1 1,702713,32,2 2,7,32,0 2,7,32,1 2,7,32,2 2,192,32,0 2,192,32,1 2,192,32,2 2,1792,32,0 2,1792,32,1 2,1792,32,2 2,500,32,0 2,500,32,1 2,500,32,2 2,8000,32,0 2,8000,32,1 2,8000,32,2 2,100000,32,0 2,100000,32,1 2,100000,32,2 2,702713,32,0 2,702713,32,1 2,702713,32,2 3,7,32,0 3,7,32,1 3,7,32,2 3,192,32,0 3,192,32,1 3,192,32,2 3,1792,32,0 3,1792,32,1 3,1792,32,2 3,500,32,0 3,500,32,1 3,500,32,2 3,8000,32,0 3,8000,32,1 3,8000,32,2 3,100000,32,0 3,100000,32,1 3,100000,32,2 3,702713,32,0 3,702713,32,1 3,702713,32,2 4,7,32,0 4,7,32,1 4,7,32,2 4,192,32,0 4,192,32,1 4,192,32,2 4,1792,32,0 4,1792,32,1 4,1792,32,2 4,500,32,0 4,500,32,1 4,500,32,2 4,8000,32,0 4,8000,32,1 4,8000,32,2 4,100000,32,0 4,100000,32,1 4,100000,32,2 4,702713,32,0 4,702713,32,1 4,702713,32,2 5,7,32,0 5,7,32,1 5,7,32,2 5,192,32,0 5,192,32,1 5,192,32,2 5,1792,32,0 5,1792,32,1 5,1792,32,2 5,500,32,0 5,500,32,1 5,500,32,2 5,8000,32,0 5,8000,32,1 5,8000,32,2 5,100000,32,0 5,100000,32,1 5,100000,32,2 5,702713,32,0 5,702713,32,1 5,702713,32,2 6,7,32,0 6,7,32,1 6,7,32,2 6,192,32,0 6,192,32,1 6,192,32,2 6,1792,32,0 6,1792,32,1 6,1792,32,2 6,500,32,0 6,500,32,1 6,500,32,2 6,8000,32,0 6,8000,32,1 6,8000,32,2 6,100000,32,0 6,100000,32,1 6,100000,32,2 6,702713,32,0 6,702713,32,1 6,702713,32,2 7,7,32,0 7,7,32,1 7,7,32,2 7,192,32,0 7,192,32,1 7,192,32,2 7,1792,32,0 7,1792,32,1 7,1792,32,2 7,500,32,0 7,500,32,1 7,500,32,2 7,8000,32,0 7,8000,32,1 7,8000,32,2 7,100000,32,0 7,100000,32,1 7,100000,32,2 7,702713,32,0 7,702713,32,1 7,702713,32,2 8,7,32,0 8,7,32,1 8,7,32,2 8,192,32,0 8,192,32,1 8,192,32,2 8,1792,32,0 8,1792,32,1 8,1792,32,2 8,500,32,0 8,500,32,1 8,500,32,2 8,8000,32,0 8,8000,32,1 8,8000,32,2 8,100000,32,0 8,100000,32,1 8,100000,32,2 8,702713,32,0 8,702713,32,1 8,702713,32,2 11,7,32,0 11,7,32,1 11,7,32,2 11,192,32,0 11,192,32,1 11,192,32,2 11,1792,32,0 11,1792,32,1 11,1792,32,2 11,500,32,0 11,500,32,1 11,500,32,2 11,8000,32,0 11,8000,32,1 11,8000,32,2 11,100000,32,0 11,100000,32,1 11,100000,32,2 11,702713,32,0 11,702713,32,1 11,702713,32,2 16,7,32,0 16,7,32,1 16,7,32,2 16,192,32,0 16,192,32,1 16,192,32,2 16,1792,32,0 16,1792,32,1 16,1792,32,2 16,500,32,0 16,500,32,1 16,500,32,2 16,8000,32,0 16,8000,32,1 16,8000,32,2 16,100000,32,0 16,100000,32,1 16,100000,32,2 16,702713,32,0 16,702713,32,1 16,702713,32,2 22,7,32,0 22,7,32,1 22,7,32,2 22,192,32,0 22,192,32,1 22,192,32,2 22,1792,32,0 22,1792,32,1 22,1792,32,2 22,500,32,0 22,500,32,1 22,500,32,2 22,8000,32,0 22,8000,32,1 22,8000,32,2 22,100000,32,0 22,100000,32,1 22,100000,32,2 22,702713,32,0 22,702713,32,1 22,702713,32,2 30,7,32,0 30,7,32,1 30,7,32,2 30,192,32,0 30,192,32,1 30,192,32,2 30,1792,32,0 30,1792,32,1 30,1792,32,2 30,500,32,0 30,500,32,1 30,500,32,2 30,8000,32,0 30,8000,32,1 30,8000,32,2 30,100000,32,0 30,100000,32,1 30,100000,32,2 30,702713,32,0 30,702713,32,1 30,702713,32,2 32,7,32,0 32,7,32,1 32,7,32,2 32,192,32,0 32,192,32,1 32,192,32,2 32,1792,32,0 32,1792,32,1 32,1792,32,2 32,500,32,0 32,500,32,1 32,500,32,2 32,8000,32,0 32,8000,32,1 32,8000,32,2 32,100000,32,0 32,100000,32,1 32,100000,32,2 32,702713,32,0 32,702713,32,1 32,702713,32,2 42,7,32,0 42,7,32,1 42,7,32,2 42,192,32,0 42,192,32,1 42,192,32,2 42,1792,32,0 42,1792,32,1 42,1792,32,2 42,500,32,0 42,500,32,1 42,500,32,2 42,8000,32,0 42,8000,32,1 42,8000,32,2 42,100000,32,0 42,100000,32,1 42,100000,32,2 42,702713,32,0 42,702713,32,1 42,702713,32,2 48,7,32,0 48,7,32,1 48,7,32,2 48,192,32,0 48,192,32,1 48,192,32,2 48,1792,32,0 48,1792,32,1 48,1792,32,2 48,500,32,0 48,500,32,1 48,500,32,2 48,8000,32,0 48,8000,32,1 48,8000,32,2 48,100000,32,0 48,100000,32,1 48,100000,32,2 48,702713,32,0 48,702713,32,1 48,702713,32,2 52,7,32,0 52,7,32,1 52,7,32,2 52,192,32,0 52,192,32,1 52,192,32,2 52,1792,32,0 52,1792,32,1 52,1792,32,2 52,500,32,0 52,500,32,1 52,500,32,2 52,8000,32,0 52,8000,32,1 52,8000,32,2 52,100000,32,0 52,100000,32,1 52,100000,32,2 52,702713,32,0 52,702713,32,1 52,702713,32,2 53,7,32,0 53,7,32,1 53,7,32,2 53,192,32,0 53,192,32,1 53,192,32,2 53,1792,32,0 53,1792,32,1 53,1792,32,2 53,500,32,0 53,500,32,1 53,500,32,2 53,8000,32,0 53,8000,32,1 53,8000,32,2 53,100000,32,0 53,100000,32,1 53,100000,32,2 53,702713,32,0 53,702713,32,1 53,702713,32,2 64,7,32,0 64,7,32,1 64,7,32,2 64,192,32,0 64,192,32,1 64,192,32,2 64,1792,32,0 64,1792,32,1 64,1792,32,2 64,500,32,0 64,500,32,1 64,500,32,2 64,8000,32,0 64,8000,32,1 64,8000,32,2 64,100000,32,0 64,100000,32,1 64,100000,32,2 64,702713,32,0 64,702713,32,1 64,702713,32,2 80,7,32,0 80,7,32,1 80,7,32,2 80,192,32,0 80,192,32,1 80,192,32,2 80,1792,32,0 80,1792,32,1 80,1792,32,2 80,500,32,0 80,500,32,1 80,500,32,2 80,8000,32,0 80,8000,32,1 80,8000,32,2 80,100000,32,0 80,100000,32,1 80,100000,32,2 80,702713,32,0 80,702713,32,1 80,702713,32,2