pax_global_header00006660000000000000000000000064142325425620014517gustar00rootroot0000000000000052 comment=fd7032a3045f268f84d26baa7cc9fb7f3cfec99d msgpack-numpy-0.4.8/000077500000000000000000000000001423254256200143235ustar00rootroot00000000000000msgpack-numpy-0.4.8/.gitignore000066400000000000000000000040471423254256200163200ustar00rootroot00000000000000# Created by https://www.toptal.com/developers/gitignore/api/python # Edit at https://www.toptal.com/developers/gitignore?templates=python ### Python ### # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class # C extensions *.so # Distribution / packaging .Python build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ wheels/ pip-wheel-metadata/ share/python-wheels/ *.egg-info/ .installed.cfg *.egg MANIFEST # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .nox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *.cover *.py,cover .hypothesis/ .pytest_cache/ pytestdebug.log # Translations *.mo *.pot # Django stuff: *.log local_settings.py db.sqlite3 db.sqlite3-journal # Flask stuff: instance/ .webassets-cache # Scrapy stuff: .scrapy # Sphinx documentation docs/_build/ doc/_build/ # PyBuilder target/ # Jupyter Notebook .ipynb_checkpoints # IPython profile_default/ ipython_config.py # pyenv .python-version # pipenv # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. # However, in case of collaboration, if having platform-specific dependencies or dependencies # having no cross-platform support, pipenv may install dependencies that don't work, or not # install all needed dependencies. #Pipfile.lock # PEP 582; used by e.g. github.com/David-OConnor/pyflow __pypackages__/ # Celery stuff celerybeat-schedule celerybeat.pid # SageMath parsed files *.sage.py # Environments .env .venv env/ venv/ ENV/ env.bak/ venv.bak/ # Spyder project settings .spyderproject .spyproject # Rope project settings .ropeproject # mkdocs documentation /site # mypy .mypy_cache/ .dmypy.json dmypy.json # Pyre type checker .pyre/ # pytype static type analyzer .pytype/ # End of https://www.toptal.com/developers/gitignore/api/python msgpack-numpy-0.4.8/.travis.yml000066400000000000000000000002261423254256200164340ustar00rootroot00000000000000dist: xenial language: python os: - linux python: - 2.7 - 3.5 - 3.6 - 3.7 - 3.8 install: pip install tox-travis script: tox msgpack-numpy-0.4.8/AUTHORS.md000066400000000000000000000013511423254256200157720ustar00rootroot00000000000000 Authors ======= This software was written and packaged by [Lev E. Givon](https://github.com/lebedov) Special thanks are due to the following parties for their contributions: - [Alex Ford](https://github.com/asford) - bug fix. - [Colin Jermain](https://github.com/cjermain) - Python 3 support. - [Etienne Wodey](https://github.com/airwoodix) - support for ndarrays with dtype='O' - [John Tyree](https://github.com/johntyree) - support for numpy scalar booleans. - [Mehdi Sadeghi](https://github.com/mehdisadeghi) - bug reports. - [Sujoy Roy](https://github.com/tvkpz) - bug reports. - [Jose Coutinho](https://github.com/tiagocoutinho) - make msgpack-numpy params consistent with msgpack-python. msgpack-numpy-0.4.8/CHANGES.md000066400000000000000000000110211423254256200157100ustar00rootroot00000000000000 Change Log ========== Release 0.4.8 (April 28, 2022) ------------------------------ * Add support for ndarrays with dtype=object (#46). Release 0.4.7.1 (September 30, 2020) ------------------------------------ * Fix Python 2.7 regression (#45). Release 0.4.7 (September 16, 2020) ---------------------------------- * Fix bug unpacking nested, structured dtypes (#42). Release 0.4.6.1 (July 28, 2020) ------------------------------- * Deprecate support for msgpack < 1.0.0 (#41). Release 0.4.6 (May 25, 2020) ---------------------------- * Set default use_bin_type and raw params of Packer and Unpacker classes the same as in msgpack (#39). * Convert docs to Markdown. Release 0.4.5 (April 12, 2020) ------------------------------ * Set defaults appropriate to msgpack 1.0.0 (#38). Release 0.4.4.3 (May 16, 2019) ------------------------------ * Configure contiguous integration with Travis. * Move unit tests out of msgpack_numpy module. * Add workaround for issue handling n-dim arrays on MacOS (#35). Release 0.4.4.2 (November 8, 2018) ---------------------------------- * Fix regression handling noncontiguous arrays (#34). Release 0.4.4.1 (September 26, 2018) ------------------------------------ * Check Py version before defining encode and tostr funcs. * Eliminate deprecation warnings, raise minimum required msgpack version (#31). * Fix Python 2 support (#32). Release 0.4.4 (September 25, 2018) ---------------------------------- * Access ndarray memory view directly to slightly speed up encoding (#30). Release 0.4.3.2 - (September 17, 2018) -------------------------------------- * Update classifiers to list Py3 support (#29). Release 0.4.3.1 - (July 11, 2018) --------------------------------- * Switch to numpy.frombuffer to avoid deprecation warning (#27). Release 0.4.3 - (February 25, 2018) ----------------------------------- * Change dependency name due to package name change (#25). Release 0.4.2 - (December 2, 2017) ---------------------------------- * Make decoding/encoding defaults identical to those of msgpack-python 0.4.* (#19). * Fix handling of nested arrays in Py3 (#21). * Improve decoding of complex values in Py3. * Use numpy.testing functions to make unit test errors more informative. * Make object_hook params consistent with those of msgack-python (#23). Release 0.4.1 - (July 6, 2017) ------------------------------ * Improve ability to decode data serialized with versions before 0.3.9 (#20). Release 0.4.0 - (May 24, 2017) ------------------------------ * Remove deprecated ez_setup. * Add numpy >= 1.9.0 dependency (#18). Release 0.3.9 - (February 7, 2017) ---------------------------------- * Complex type handling fixes. * Handle structured dtypes (#15). * Note: data serialized with earlier versions cannot be deserialized with 0.3.9 (#17). Release 0.3.8 - (Februrary 4, 2017) ----------------------------------- * Fix deserialization with Python 3.5 (#13). Release 0.3.7 - (March 12, 2016) -------------------------------- * Add Python 3 support (#11). Release 0.3.6 - (July 8, 2015) ------------------------------ * Fall back to pure Python msgpack if compiled extension is unavailable (#6). Release 0.3.5 - (February 12, 2015) ----------------------------------- * Add support for numpy scalar booleans (#8). Release 0.3.4 - (December 12, 2014) ----------------------------------- * Add badges, update README. Release 0.3.3 - (October 21, 2014) ---------------------------------- * Add missing ez_setup.py file. Release 0.3.2 - (May 6, 2014) ----------------------------- * Simplify encoding/decoding of numpy scalars. Release 0.3.1.1 - (March 2, 2014) --------------------------------- * Make package a simple module to fix dependency installation issues. Release 0.3.1 - (March 2, 2014) ------------------------------- * Switch to PEP 440 version numbering. * Update to setuptools 2.2 ez_setup.py Release 0.03 - (October 30, 2013) --------------------------------- * Add support for msgpack 0.4.0. * Rename to msgpack-numpy. Release 0.022 - (September 10, 2013) ------------------------------------ * Fix decoding of string arrays (#4). * Fix decoding of dicts containing ndarrays (#5). Release 0.021 - (May 29, 2013) ------------------------------ * Improve encoding/decoding performance for arrays. * Fix numpy type support on different platforms (#3). Release 0.02 - (February 21, 2013) ---------------------------------- * Add support for msgpack 0.3.0 (contributed by crispamares). Release 0.01 - (February 07, 2013) ---------------------------------- * First public release. msgpack-numpy-0.4.8/LICENSE.md000066400000000000000000000030171423254256200157300ustar00rootroot00000000000000 License ======= Copyright (c) 2013-2022, Lev E. Givon. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Lev E. Givon nor the names of any contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. msgpack-numpy-0.4.8/MANIFEST.in000066400000000000000000000000561423254256200160620ustar00rootroot00000000000000include *.py include *.md exclude MANIFEST.in msgpack-numpy-0.4.8/Makefile000066400000000000000000000007171423254256200157700ustar00rootroot00000000000000PYTHON := `which python` NAME = msgpack-numpy VERSION = $(shell $(PYTHON) -c 'import setup; print setup.VERSION') .PHONY: package build develop install test clean package: $(PYTHON) setup.py sdist --formats=gztar bdist_wheel upload: | package twine upload dist/* build: $(PYTHON) setup.py build develop: $(PYTHON) setup.py develop install: $(PYTHON) setup.py install test: $(PYTHON) msgpack_numpy.py clean: $(PYTHON) setup.py clean rm -f dist/* msgpack-numpy-0.4.8/README.md000066400000000000000000000077611423254256200156150ustar00rootroot00000000000000 Numpy Data Type Serialization Using Msgpack =========================================== Package Description ------------------- This package provides encoding and decoding routines that enable the serialization and deserialization of numerical and array data types provided by [numpy](http://www.numpy.org/) using the highly efficient [msgpack](http://msgpack.org/) format. Serialization of Python's native complex data types is also supported. [![Latest Version](https://img.shields.io/pypi/v/msgpack-numpy.svg)](https://pypi.python.org/pypi/msgpack-numpy) [![Build Status](https://travis-ci.org/lebedov/msgpack-numpy.svg?branch=master)](https://travis-ci.org/lebedov/msgpack-numpy) Installation ------------ msgpack-numpy requires msgpack-python and numpy. If you have [pip](http://www.pip-installer.org/) installed on your system, run pip install msgpack-numpy to install the package and all dependencies. You can also download the source tarball, unpack it, and run python setup.py install from within the source directory. Usage ----- The easiest way to use msgpack-numpy is to call its monkey patching function after importing the Python msgpack package: import msgpack import msgpack_numpy as m m.patch() This will automatically force all msgpack serialization and deserialization routines (and other packages that use them) to become numpy-aware. Of course, one can also manually pass the encoder and decoder provided by msgpack-numpy to the msgpack routines: import msgpack import msgpack_numpy as m import numpy as np x = np.random.rand(5) x_enc = msgpack.packb(x, default=m.encode) x_rec = msgpack.unpackb(x_enc, object_hook=m.decode) msgpack-numpy will try to use the binary (fast) extension in msgpack by default. If msgpack was not compiled with Cython (or if the ``MSGPACK_PUREPYTHON`` variable is set), it will fall back to using the slower pure Python msgpack implementation. Notes ----- The primary design goal of msgpack-numpy is ensuring preservation of numerical data types during msgpack serialization and deserialization. Inclusion of type information in the serialized data necessarily incurs some storage overhead; if preservation of type information is not needed, one may be able to avoid some of this overhead by writing a custom encoder/decoder pair that produces more efficient serializations for those specific use cases. Numpy arrays with a dtype of 'O' are serialized/deserialized using pickle as a fallback solution to enable msgpack-numpy to handle such arrays. As the additional overhead of pickle serialization negates one of the reasons to use msgpack, it may be advisable to either write a custom encoder/decoder to handle the specific use case efficiently or else not bother using msgpack-numpy. Note that numpy arrays deserialized by msgpack-numpy are read-only and must be copied if they are to be modified. Development ----------- The latest source code can be obtained from [GitHub](https://github.com/lebedov/msgpack-numpy/). msgpack-numpy maintains compatibility with python versions 2.7 and 3.5+. Install [`tox`](https://tox.readthedocs.io/en/latest/) to support testing across multiple python versions in your development environment. If you use [`conda`](https://docs.conda.io/en/latest/) to install `python` use [`tox-conda`](https://github.com/tox-dev/tox-conda) to automatically manage testing across all supported python versions. # Using a system python pip install tox # Additionally, using a conda-provided python pip install tox tox-conda Execute tests across supported python versions: tox Authors ------- See the included [AUTHORS.md](https://github.com/lebedov/msgpack-numpy/blob/master/AUTHORS.md) file for more information. License ------- This software is licensed under the [BSD License](http://www.opensource.org/licenses/bsd-license). See the included [LICENSE.md](https://github.com/lebedov/msgpack-numpy/blob/master/LICENSE.md) file for more information. msgpack-numpy-0.4.8/msgpack_numpy.py000066400000000000000000000270161423254256200175600ustar00rootroot00000000000000#!/usr/bin/env python """ Support for serialization of numpy data types with msgpack. """ # Copyright (c) 2013-2022, Lev E. Givon # All rights reserved. # Distributed under the terms of the BSD license: # http://www.opensource.org/licenses/bsd-license import sys import functools import pickle import warnings import msgpack from msgpack import Packer as _Packer, Unpacker as _Unpacker, \ unpack as _unpack, unpackb as _unpackb import numpy as np if sys.version_info >= (3, 0): def ndarray_to_bytes(obj): if obj.dtype == 'O': return obj.dumps() else: if sys.platform == 'darwin': return obj.tobytes() else: return obj.data if obj.flags['C_CONTIGUOUS'] else obj.tobytes() num_to_bytes = lambda obj: obj.data def tostr(x): if isinstance(x, bytes): return x.decode() else: return str(x) else: def ndarray_to_bytes(obj): if obj.dtype == 'O': return obj.dumps() else: if sys.platform == 'darwin': return obj.tobytes() else: return memoryview(obj.data) if obj.flags['C_CONTIGUOUS'] else obj.tobytes() num_to_bytes = lambda obj: memoryview(obj.data) def tostr(x): return x def encode(obj, chain=None): """ Data encoder for serializing numpy data types. """ if isinstance(obj, np.ndarray): # If the dtype is structured, store the interface description; # otherwise, store the corresponding array protocol type string: if obj.dtype.kind in ('V', 'O'): kind = bytes(obj.dtype.kind, 'ascii') descr = obj.dtype.descr else: kind = b'' descr = obj.dtype.str return {b'nd': True, b'type': descr, b'kind': kind, b'shape': obj.shape, b'data': ndarray_to_bytes(obj)} elif isinstance(obj, (np.bool_, np.number)): return {b'nd': False, b'type': obj.dtype.str, b'data': num_to_bytes(obj)} elif isinstance(obj, complex): return {b'complex': True, b'data': obj.__repr__()} else: return obj if chain is None else chain(obj) def decode(obj, chain=None): """ Decoder for deserializing numpy data types. """ try: if b'nd' in obj: if obj[b'nd'] is True: # Check if b'kind' is in obj to enable decoding of data # serialized with older versions (#20) or data # that had dtype == 'O' (#46): if b'kind' in obj and obj[b'kind'] == b'V': descr = [tuple(tostr(t) if type(t) is bytes else t for t in d) \ for d in obj[b'type']] elif b'kind' in obj and obj[b'kind'] == b'O': return pickle.loads(obj[b'data']) else: descr = obj[b'type'] return np.ndarray(buffer=obj[b'data'], dtype=_unpack_dtype(descr), shape=obj[b'shape']) else: descr = obj[b'type'] return np.frombuffer(obj[b'data'], dtype=_unpack_dtype(descr))[0] elif b'complex' in obj: return complex(tostr(obj[b'data'])) else: return obj if chain is None else chain(obj) except KeyError: return obj if chain is None else chain(obj) def _unpack_dtype(dtype): """ Unpack dtype descr, recursively unpacking nested structured dtypes. """ if isinstance(dtype, (list, tuple)): # Unpack structured dtypes of the form: (name, type, *shape) dtype = [ (subdtype[0], _unpack_dtype(subdtype[1])) + tuple(subdtype[2:]) for subdtype in dtype ] return np.dtype(dtype) if msgpack.version < (1, 0, 0): warnings.warn('support for msgpack < 1.0.0 will be removed in a future release', DeprecationWarning) # Maintain support for msgpack < 0.4.0: if msgpack.version < (0, 4, 0): class Packer(_Packer): def __init__(self, default=None, encoding='utf-8', unicode_errors='strict', use_single_float=False, autoreset=1): default = functools.partial(encode, chain=default) super(Packer, self).__init__(default=default, encoding=encoding, unicode_errors=unicode_errors, use_single_float=use_single_float, autoreset=autoreset) class Unpacker(_Unpacker): def __init__(self, file_like=None, read_size=0, use_list=None, object_hook=None, object_pairs_hook=None, list_hook=None, encoding='utf-8', unicode_errors='strict', max_buffer_size=0): object_hook = functools.partial(decode, chain=object_hook) super(Unpacker, self).__init__(file_like=file_like, read_size=read_size, use_list=use_list, object_hook=object_hook, object_pairs_hook=object_pairs_hook, list_hook=list_hook, encoding=encoding, unicode_errors=unicode_errors, max_buffer_size=max_buffer_size) elif msgpack.version < (1, 0, 0): class Packer(_Packer): def __init__(self, default=None, unicode_errors='strict', use_single_float=False, autoreset=1, use_bin_type=True, strict_types=False): default = functools.partial(encode, chain=default) super(Packer, self).__init__(default=default, unicode_errors=unicode_errors, use_single_float=use_single_float, autoreset=autoreset, use_bin_type=use_bin_type, strict_types=strict_types) class Unpacker(_Unpacker): def __init__(self, file_like=None, read_size=0, use_list=None, raw=False, object_hook=None, object_pairs_hook=None, list_hook=None, unicode_errors='strict', max_buffer_size=0, ext_hook=msgpack.ExtType): object_hook = functools.partial(decode, chain=object_hook) super(Unpacker, self).__init__(file_like=file_like, read_size=read_size, use_list=use_list, raw=raw, object_hook=object_hook, object_pairs_hook=object_pairs_hook, list_hook=list_hook, unicode_errors=unicode_errors, max_buffer_size=max_buffer_size, ext_hook=ext_hook) else: class Packer(_Packer): def __init__(self, default=None, use_single_float=False, autoreset=True, use_bin_type=True, strict_types=False, datetime=False, unicode_errors=None): default = functools.partial(encode, chain=default) super(Packer, self).__init__(default=default, use_single_float=use_single_float, autoreset=autoreset, use_bin_type=use_bin_type, strict_types=strict_types, datetime=datetime, unicode_errors=unicode_errors) class Unpacker(_Unpacker): def __init__(self, file_like=None, read_size=0, use_list=True, raw=False, timestamp=0, strict_map_key=True, object_hook=None, object_pairs_hook=None, list_hook=None, unicode_errors=None, max_buffer_size=100 * 1024 * 1024, ext_hook=msgpack.ExtType, max_str_len=-1, max_bin_len=-1, max_array_len=-1, max_map_len=-1, max_ext_len=-1): object_hook = functools.partial(decode, chain=object_hook) super(Unpacker, self).__init__(file_like=file_like, read_size=read_size, use_list=use_list, raw=raw, timestamp=timestamp, strict_map_key=strict_map_key, object_hook=object_hook, object_pairs_hook=object_pairs_hook, list_hook=list_hook, unicode_errors=unicode_errors, max_buffer_size=max_buffer_size, ext_hook=ext_hook, max_str_len=max_str_len, max_bin_len=max_bin_len, max_array_len=max_array_len, max_map_len=max_map_len, max_ext_len=max_ext_len) def pack(o, stream, **kwargs): """ Pack an object and write it to a stream. """ packer = Packer(**kwargs) stream.write(packer.pack(o)) def packb(o, **kwargs): """ Pack an object and return the packed bytes. """ return Packer(**kwargs).pack(o) def unpack(stream, **kwargs): """ Unpack a packed object from a stream. """ object_hook = kwargs.get('object_hook') kwargs['object_hook'] = functools.partial(decode, chain=object_hook) return _unpack(stream, **kwargs) def unpackb(packed, **kwargs): """ Unpack a packed object. """ object_hook = kwargs.get('object_hook') kwargs['object_hook'] = functools.partial(decode, chain=object_hook) return _unpackb(packed, **kwargs) load = unpack loads = unpackb dump = pack dumps = packb def patch(): """ Monkey patch msgpack module to enable support for serializing numpy types. """ setattr(msgpack, 'Packer', Packer) setattr(msgpack, 'Unpacker', Unpacker) setattr(msgpack, 'load', unpack) setattr(msgpack, 'loads', unpackb) setattr(msgpack, 'dump', pack) setattr(msgpack, 'dumps', packb) setattr(msgpack, 'pack', pack) setattr(msgpack, 'packb', packb) setattr(msgpack, 'unpack', unpack) setattr(msgpack, 'unpackb', unpackb) msgpack-numpy-0.4.8/requirements.txt000066400000000000000000000000341423254256200176040ustar00rootroot00000000000000numpy>=1.9.0 msgpack>=0.5.2 msgpack-numpy-0.4.8/setup.cfg000066400000000000000000000000341423254256200161410ustar00rootroot00000000000000[bdist_wheel] universal = 1 msgpack-numpy-0.4.8/setup.py000066400000000000000000000035031423254256200160360ustar00rootroot00000000000000#!/usr/bin/env python import os import re from setuptools import setup NAME = 'msgpack-numpy' VERSION = '0.4.8' AUTHOR = 'Lev E. Givon' AUTHOR_EMAIL = 'lev@columbia.edu' URL = 'https://github.com/lebedov/msgpack-numpy' DESCRIPTION = 'Numpy data serialization using msgpack' with open('README.md', 'r') as f: LONG_DESCRIPTION = f.read() LONG_DESCRIPTION = re.search('.*(^Package Description.*)', LONG_DESCRIPTION, re.MULTILINE|re.DOTALL).group(1) LONG_DESCRIPTION_CONTENT_TYPE = 'text/markdown' DOWNLOAD_URL = URL LICENSE = 'BSD' CLASSIFIERS = [ 'Development Status :: 3 - Alpha', 'Intended Audience :: Developers', 'Intended Audience :: Science/Research', 'License :: OSI Approved :: BSD License', 'Operating System :: OS Independent', 'Programming Language :: Python', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Topic :: Scientific/Engineering', 'Topic :: Software Development'] if __name__ == "__main__": if os.path.exists('MANIFEST'): os.remove('MANIFEST') setup( name = NAME, version = VERSION, author = AUTHOR, author_email = AUTHOR_EMAIL, license = LICENSE, classifiers = CLASSIFIERS, description = DESCRIPTION, long_description = LONG_DESCRIPTION, long_description_content_type = LONG_DESCRIPTION_CONTENT_TYPE, url = URL, py_modules = ['msgpack_numpy'], install_requires = ['numpy>=1.9.0', 'msgpack>=0.5.2'] ) msgpack-numpy-0.4.8/tests.py000066400000000000000000000254351423254256200160500ustar00rootroot00000000000000#!/usr/bin/env python import sys from unittest import main, TestCase import msgpack import numpy as np from numpy.testing import assert_equal, assert_array_equal from msgpack_numpy import patch try: range = xrange # Python 2 except NameError: pass # Python 3 class ThirdParty(object): def __init__(self, foo=b'bar'): self.foo = foo def __eq__(self, other): return isinstance(other, ThirdParty) and self.foo == other.foo class test_numpy_msgpack(TestCase): def setUp(self): patch() def encode_decode(self, x, use_list=True, max_bin_len=-1): x_enc = msgpack.packb(x) return msgpack.unpackb(x_enc, use_list=use_list, max_bin_len=max_bin_len) def encode_thirdparty(self, obj): return {b'__thirdparty__': True, b'foo': obj.foo} def decode_thirdparty(self, obj): if b'__thirdparty__' in obj: return ThirdParty(foo=obj[b'foo']) return obj def encode_decode_thirdparty(self, x, use_list=True, max_bin_len=-1): x_enc = msgpack.packb(x, default=self.encode_thirdparty) return msgpack.unpackb(x_enc, object_hook=self.decode_thirdparty, use_list=use_list, max_bin_len=max_bin_len) def test_bin(self): # str == bytes on Python 2: if sys.version_info.major == 2: assert_equal(type(self.encode_decode(b'foo')), str) else: assert_equal(type(self.encode_decode(b'foo')), bytes) def test_str(self): # str != unicode on Python 2: if sys.version_info.major == 2: assert_equal(type(self.encode_decode('foo')), str) assert_equal(type(self.encode_decode(u'foo')), unicode) else: assert_equal(type(self.encode_decode(u'foo')), str) def test_numpy_scalar_bool(self): x = np.bool_(True) x_rec = self.encode_decode(x) assert_equal(x, x_rec) assert_equal(type(x), type(x_rec)) x = np.bool_(False) x_rec = self.encode_decode(x) assert_equal(x, x_rec) assert_equal(type(x), type(x_rec)) def test_numpy_scalar_float(self): x = np.float32(np.random.rand()) x_rec = self.encode_decode(x) assert_equal(x, x_rec) assert_equal(type(x), type(x_rec)) def test_numpy_scalar_complex(self): x = np.complex64(np.random.rand()+1j*np.random.rand()) x_rec = self.encode_decode(x) assert_equal(x, x_rec) assert_equal(type(x), type(x_rec)) def test_scalar_float(self): x = np.random.rand() x_rec = self.encode_decode(x) assert_equal(x, x_rec) assert_equal(type(x), type(x_rec)) def test_scalar_complex(self): x = np.random.rand()+1j*np.random.rand() x_rec = self.encode_decode(x) assert_equal(x, x_rec) assert_equal(type(x), type(x_rec)) def test_list_numpy_float(self): x = [np.float32(np.random.rand()) for i in range(5)] x_rec = self.encode_decode(x) assert_array_equal(x, x_rec) assert_array_equal([type(e) for e in x], [type(e) for e in x_rec]) def test_list_numpy_float_complex(self): x = [np.float32(np.random.rand()) for i in range(5)] + \ [np.complex128(np.random.rand()+1j*np.random.rand()) for i in range(5)] x_rec = self.encode_decode(x) assert_array_equal(x, x_rec) assert_array_equal([type(e) for e in x], [type(e) for e in x_rec]) def test_list_float(self): x = [np.random.rand() for i in range(5)] x_rec = self.encode_decode(x) assert_array_equal(x, x_rec) assert_array_equal([type(e) for e in x], [type(e) for e in x_rec]) def test_list_float_complex(self): x = [(np.random.rand()+1j*np.random.rand()) for i in range(5)] x_rec = self.encode_decode(x) assert_array_equal(x, x_rec) assert_array_equal([type(e) for e in x], [type(e) for e in x_rec]) def test_list_str(self): x = [b'x'*i for i in range(5)] x_rec = self.encode_decode(x) assert_array_equal(x, x_rec) assert_array_equal([type(e) for e in x_rec], [bytes]*5) def test_dict_float(self): x = {b'foo': 1.0, b'bar': 2.0} x_rec = self.encode_decode(x) assert_array_equal(sorted(x.values()), sorted(x_rec.values())) assert_array_equal([type(e) for e in sorted(x.values())], [type(e) for e in sorted(x_rec.values())]) assert_array_equal(sorted(x.keys()), sorted(x_rec.keys())) assert_array_equal([type(e) for e in sorted(x.keys())], [type(e) for e in sorted(x_rec.keys())]) def test_dict_complex(self): x = {b'foo': 1.0+1.0j, b'bar': 2.0+2.0j} x_rec = self.encode_decode(x) assert_array_equal(sorted(x.values(), key=np.linalg.norm), sorted(x_rec.values(), key=np.linalg.norm)) assert_array_equal([type(e) for e in sorted(x.values(), key=np.linalg.norm)], [type(e) for e in sorted(x_rec.values(), key=np.linalg.norm)]) assert_array_equal(sorted(x.keys()), sorted(x_rec.keys())) assert_array_equal([type(e) for e in sorted(x.keys())], [type(e) for e in sorted(x_rec.keys())]) def test_dict_str(self): x = {b'foo': b'xxx', b'bar': b'yyyy'} x_rec = self.encode_decode(x) assert_array_equal(sorted(x.values()), sorted(x_rec.values())) assert_array_equal([type(e) for e in sorted(x.values())], [type(e) for e in sorted(x_rec.values())]) assert_array_equal(sorted(x.keys()), sorted(x_rec.keys())) assert_array_equal([type(e) for e in sorted(x.keys())], [type(e) for e in sorted(x_rec.keys())]) def test_dict_numpy_float(self): x = {b'foo': np.float32(1.0), b'bar': np.float32(2.0)} x_rec = self.encode_decode(x) assert_array_equal(sorted(x.values()), sorted(x_rec.values())) assert_array_equal([type(e) for e in sorted(x.values())], [type(e) for e in sorted(x_rec.values())]) assert_array_equal(sorted(x.keys()), sorted(x_rec.keys())) assert_array_equal([type(e) for e in sorted(x.keys())], [type(e) for e in sorted(x_rec.keys())]) def test_dict_numpy_complex(self): x = {b'foo': np.complex128(1.0+1.0j), b'bar': np.complex128(2.0+2.0j)} x_rec = self.encode_decode(x) assert_array_equal(sorted(x.values(), key=np.linalg.norm), sorted(x_rec.values(), key=np.linalg.norm)) assert_array_equal([type(e) for e in sorted(x.values(), key=np.linalg.norm)], [type(e) for e in sorted(x_rec.values(), key=np.linalg.norm)]) assert_array_equal(sorted(x.keys()), sorted(x_rec.keys())) assert_array_equal([type(e) for e in sorted(x.keys())], [type(e) for e in sorted(x_rec.keys())]) def test_numpy_array_float(self): x = np.random.rand(5).astype(np.float32) x_rec = self.encode_decode(x) assert_array_equal(x, x_rec) assert_equal(x.dtype, x_rec.dtype) def test_numpy_array_object(self): x = np.random.rand(5).astype(object) x_rec = self.encode_decode(x) assert_array_equal(x, x_rec) assert_equal(x.dtype, x_rec.dtype) def test_numpy_array_complex(self): x = (np.random.rand(5)+1j*np.random.rand(5)).astype(np.complex128) x_rec = self.encode_decode(x) assert_array_equal(x, x_rec) assert_equal(x.dtype, x_rec.dtype) def test_numpy_array_float_2d(self): x = np.random.rand(5,5).astype(np.float32) x_rec = self.encode_decode(x) assert_array_equal(x, x_rec) assert_equal(x.dtype, x_rec.dtype) def test_numpy_array_float_2d_macos(self): """ Unit test for weird data loss error on MacOS (#35). """ x = np.random.rand(5, 5).astype(np.float32) x_rec = self.encode_decode(x, use_list=False, max_bin_len=50000000) assert_array_equal(x, x_rec) assert_equal(x.dtype, x_rec.dtype) def test_numpy_array_str(self): x = np.array([b'aaa', b'bbbb', b'ccccc']) x_rec = self.encode_decode(x) assert_array_equal(x, x_rec) assert_equal(x.dtype, x_rec.dtype) def test_numpy_array_mixed(self): x = np.array([(1, 2, b'a', [1.0, 2.0])], np.dtype([('arg0', np.uint32), ('arg1', np.uint32), ('arg2', 'S1'), ('arg3', np.float32, (2,))])) x_rec = self.encode_decode(x) assert_array_equal(x, x_rec) assert_equal(x.dtype, x_rec.dtype) def test_numpy_array_noncontiguous(self): x = np.ones((10, 10), np.uint32)[0:5, 0:5] x_rec = self.encode_decode(x) assert_array_equal(x, x_rec) assert_equal(x.dtype, x_rec.dtype) def test_list_mixed(self): x = [1.0, np.float32(3.5), np.complex128(4.25), b'foo'] x_rec = self.encode_decode(x) assert_array_equal(x, x_rec) assert_array_equal([type(e) for e in x], [type(e) for e in x_rec]) def test_chain(self): x = ThirdParty(foo=b'test marshal/unmarshal') x_rec = self.encode_decode_thirdparty(x) self.assertEqual(x, x_rec) def test_numpy_structured_array(self): structured_dtype = np.dtype([("a", float), ("b", int)]) x = np.empty((10,), dtype=structured_dtype) x["a"] = np.arange(10) x["b"] = np.arange(10) x_rec = self.encode_decode(x) assert_array_equal(x, x_rec) self.assertEqual(x.dtype, x_rec.dtype) def test_numpy_shaped_structured_array(self): shaped_structured_dtype = np.dtype([("a", float, 3), ("b", int)]) x = np.empty((10,), dtype=shaped_structured_dtype) x["a"] = np.arange(30).reshape(10, 3) x["b"] = np.arange(10) x_rec = self.encode_decode(x) assert_array_equal(x, x_rec) self.assertEqual(x.dtype, x_rec.dtype) def test_numpy_nested_structured_array(self): structured_dtype = np.dtype([("a", float), ("b", int)]) nested_dtype = np.dtype([("foo", structured_dtype), ("bar", structured_dtype)]) x = np.empty((10,), dtype=nested_dtype) x["foo"]["a"] = np.arange(10) x["foo"]["b"] = np.arange(10) x["bar"]["a"] = np.arange(10) + 10 x["bar"]["b"] = np.arange(10) + 10 x_rec = self.encode_decode(x) assert_array_equal(x, x_rec) self.assertEqual(x.dtype, x_rec.dtype) if __name__ == '__main__': main() msgpack-numpy-0.4.8/tox.ini000066400000000000000000000002031423254256200156310ustar00rootroot00000000000000[tox] envlist = py{27,35,36,37,38} recreate = True [testenv] deps = -r{toxinidir}/requirements.txt commands = python tests.py