pax_global_header00006660000000000000000000000064133525614340014520gustar00rootroot0000000000000052 comment=0816a7a41623db9cd2502b7cba43a1d49ac16c24 msgpack-numpy-0.4.4/000077500000000000000000000000001335256143400143205ustar00rootroot00000000000000msgpack-numpy-0.4.4/AUTHORS.rst000066400000000000000000000011071335256143400161760ustar00rootroot00000000000000.. -*- rst -*- Authors ======= This software was written and packaged by `Lev E. Givon `_. Special thanks are due to the following parties for their contributions: - `Colin Jermain `_ - Python 3 support. - `John Tyree `_ - support for numpy scalar booleans. - `Mehdi Sadeghi `_ - bug reports. - `Sujoy Roy `_ - bug reports. - `Jose Coutinho `_ - make msgpack-numpy params consistent with msgpack-python. msgpack-numpy-0.4.4/CHANGES.rst000066400000000000000000000062451335256143400161310ustar00rootroot00000000000000.. -*- rst -*- Change Log ========== Release 0.4.4 (September 25, 2018) ---------------------------------- * Access ndarray memory view directly to slightly speed up encoding (#30). Release 0.4.3.2 - (September 17, 2018) -------------------------------------- * Update classifiers to list Py3 support (#29). Release 0.4.3.1 - (July 11, 2018) --------------------------------- * Switch to numpy.frombuffer to avoid deprecation warning (#27). Release 0.4.3 - (February 25, 2018) ----------------------------------- * Change dependency name due to package name change (#25). Release 0.4.2 - (December 2, 2017) ---------------------------------- * Make decoding/encoding defaults identical to those of msgpack-python 0.4.* (#19). * Fix handling of nested arrays in Py3 (#21). * Improve decoding of complex values in Py3. * Use numpy.testing functions to make unit test errors more informative. * Make object_hook params consistent with those of msgack-python (#23). Release 0.4.1 - (July 6, 2017) ------------------------------ * Improve ability to decode data serialized with versions before 0.3.9 (#20). Release 0.4.0 - (May 24, 2017) ------------------------------ * Remove deprecated ez_setup. * Add numpy >= 1.9.0 dependency (#18). Release 0.3.9 - (February 7, 2017) ---------------------------------- * Complex type handling fixes. * Handle structured dtypes (#15). * Note: data serialized with earlier versions cannot be deserialized with 0.3.9 (#17). Release 0.3.8 - (Februrary 4, 2017) ----------------------------------- * Fix deserialization with Python 3.5 (#13). Release 0.3.7 - (March 12, 2016) -------------------------------- * Add Python 3 support (#11). Release 0.3.6 - (July 8, 2015) ------------------------------ * Fall back to pure Python msgpack if compiled extension is unavailable (#6). Release 0.3.5 - (February 12, 2015) ----------------------------------- * Add support for numpy scalar booleans (#8). Release 0.3.4 - (December 12, 2014) ----------------------------------- * Add badges, update README. Release 0.3.3 - (October 21, 2014) ---------------------------------- * Add missing ez_setup.py file. Release 0.3.2 - (May 6, 2014) ----------------------------- * Simplify encoding/decoding of numpy scalars. Release 0.3.1.1 - (March 2, 2014) --------------------------------- * Make package a simple module to fix dependency installation issues. Release 0.3.1 - (March 2, 2014) ------------------------------- * Switch to PEP 440 version numbering. * Update to setuptools 2.2 ez_setup.py Release 0.03 - (October 30, 2013) --------------------------------- * Add support for msgpack 0.4.0. * Rename to msgpack-numpy. Release 0.022 - (September 10, 2013) ------------------------------------ * Fix decoding of string arrays (#4). * Fix decoding of dicts containing ndarrays (#5). Release 0.021 - (May 29, 2013) ------------------------------ * Improve encoding/decoding performance for arrays. * Fix numpy type support on different platforms (#3). Release 0.02 - (February 21, 2013) ---------------------------------- * Add support for msgpack 0.3.0 (contributed by crispamares). Release 0.01 - (February 07, 2013) ---------------------------------- * First public release. msgpack-numpy-0.4.4/LICENSE.rst000066400000000000000000000027601335256143400161410ustar00rootroot00000000000000.. -*- rst -*- License ======= Copyright (c) 2013-2018, Lev E. Givon. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of Lev E. Givon nor the names of any contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. msgpack-numpy-0.4.4/MANIFEST.in000066400000000000000000000000571335256143400160600ustar00rootroot00000000000000include *.py include *.rst exclude MANIFEST.in msgpack-numpy-0.4.4/Makefile000066400000000000000000000007171335256143400157650ustar00rootroot00000000000000PYTHON := `which python` NAME = msgpack-numpy VERSION = $(shell $(PYTHON) -c 'import setup; print setup.VERSION') .PHONY: package build develop install test clean package: $(PYTHON) setup.py sdist --formats=gztar bdist_wheel upload: | package twine upload dist/* build: $(PYTHON) setup.py build develop: $(PYTHON) setup.py develop install: $(PYTHON) setup.py install test: $(PYTHON) msgpack_numpy.py clean: $(PYTHON) setup.py clean rm -f dist/* msgpack-numpy-0.4.4/README.rst000066400000000000000000000045521335256143400160150ustar00rootroot00000000000000.. -*- rst -*- Numpy Data Type Serialization Using Msgpack =========================================== Package Description ------------------- This package provides encoding and decoding routines that enable the serialization and deserialization of numerical and array data types provided by `numpy `_ using the highly efficient `msgpack `_ format. Serialization of Python's native complex data types is also supported. .. image:: https://img.shields.io/pypi/v/msgpack-numpy.svg :target: https://pypi.python.org/pypi/msgpack-numpy :alt: Latest Version Installation ------------ msgpack-numpy requires msgpack-python and numpy. If you have `pip `_ installed on your system, run :: pip install msgpack-numpy to install the package and all dependencies. You can also download the source tarball, unpack it, and run :: python setup.py install from within the source directory. Usage ----- The easiest way to use msgpack-numpy is to call its monkey patching function after importing the Python msgpack package: :: import msgpack import msgpack_numpy as m m.patch() This will automatically force all msgpack serialization and deserialization routines (and other packages that use them) to become numpy-aware. Of course, one can also manually pass the encoder and decoder provided by msgpack-numpy to the msgpack routines: :: import msgpack import msgpack_numpy as m import numpy as np x = np.random.rand(5) x_enc = msgpack.packb(x, default=m.encode) x_rec = msgpack.unpackb(x_enc, object_hook=m.decode) msgpack-numpy will try to use the binary (fast) extension in msgpack by default. If msgpack was not compiled with Cython (or if the ``MSGPACK_PUREPYTHON`` variable is set), it will fall back to using the slower pure Python msgpack implementation. Development ----------- The latest source code can be obtained from `GitHub `_. Authors ------- See the included `AUTHORS.rst `_ file for more information. License ------- This software is licensed under the `BSD License `_. See the included `LICENSE.rst `_ file for more information. msgpack-numpy-0.4.4/msgpack_numpy.py000066400000000000000000000412561335256143400175570ustar00rootroot00000000000000#!/usr/bin/env python """ Support for serialization of numpy data types with msgpack. """ # Copyright (c) 2013-2018, Lev E. Givon # All rights reserved. # Distributed under the terms of the BSD license: # http://www.opensource.org/licenses/bsd-license import sys import functools import numpy as np import msgpack from msgpack import Packer as _Packer, Unpacker as _Unpacker, \ unpack as _unpack, unpackb as _unpackb def encode(obj, chain=None): """ Data encoder for serializing numpy data types. """ if isinstance(obj, np.ndarray): # If the dtype is structured, store the interface description; # otherwise, store the corresponding array protocol type string: if obj.dtype.kind == 'V': kind = b'V' descr = obj.dtype.descr else: kind = b'' descr = obj.dtype.str return {b'nd': True, b'type': descr, b'kind': kind, b'shape': obj.shape, b'data': obj.data} elif isinstance(obj, (np.bool_, np.number)): return {b'nd': False, b'type': obj.dtype.str, b'data': obj.data} elif isinstance(obj, complex): return {b'complex': True, b'data': obj.__repr__()} else: return obj if chain is None else chain(obj) def tostr(x): if sys.version_info >= (3, 0): if isinstance(x, bytes): return x.decode() else: return str(x) else: return x def decode(obj, chain=None): """ Decoder for deserializing numpy data types. """ try: if b'nd' in obj: if obj[b'nd'] is True: # Check if b'kind' is in obj to enable decoding of data # serialized with older versions (#20): if b'kind' in obj and obj[b'kind'] == b'V': descr = [tuple(tostr(t) if type(t) is bytes else t for t in d) \ for d in obj[b'type']] else: descr = obj[b'type'] return np.frombuffer(obj[b'data'], dtype=np.dtype(descr)).reshape(obj[b'shape']) else: descr = obj[b'type'] return np.frombuffer(obj[b'data'], dtype=np.dtype(descr))[0] elif b'complex' in obj: return complex(tostr(obj[b'data'])) else: return obj if chain is None else chain(obj) except KeyError: return obj if chain is None else chain(obj) # Maintain support for msgpack < 0.4.0: if msgpack.version < (0, 4, 0): class Packer(_Packer): def __init__(self, default=None, encoding='utf-8', unicode_errors='strict', use_single_float=False, autoreset=1): default = functools.partial(encode, chain=default) super(Packer, self).__init__(default=default, encoding=encoding, unicode_errors=unicode_errors, use_single_float=use_single_float, autoreset=autoreset) class Unpacker(_Unpacker): def __init__(self, file_like=None, read_size=0, use_list=None, object_hook=None, object_pairs_hook=None, list_hook=None, encoding='utf-8', unicode_errors='strict', max_buffer_size=0): object_hook = functools.partial(decode, chain=object_hook) super(Unpacker, self).__init__(file_like=file_like, read_size=read_size, use_list=use_list, object_hook=object_hook, object_pairs_hook=object_pairs_hook, list_hook=list_hook, encoding=encoding, unicode_errors=unicode_errors, max_buffer_size=max_buffer_size) else: class Packer(_Packer): def __init__(self, default=None, encoding='utf-8', unicode_errors='strict', use_single_float=False, autoreset=1, use_bin_type=0): default = functools.partial(encode, chain=default) super(Packer, self).__init__(default=default, encoding=encoding, unicode_errors=unicode_errors, use_single_float=use_single_float, autoreset=autoreset, use_bin_type=use_bin_type) class Unpacker(_Unpacker): def __init__(self, file_like=None, read_size=0, use_list=None, object_hook=None, object_pairs_hook=None, list_hook=None, encoding=None, unicode_errors='strict', max_buffer_size=0, ext_hook=msgpack.ExtType): object_hook = functools.partial(decode, chain=object_hook) super(Unpacker, self).__init__(file_like=file_like, read_size=read_size, use_list=use_list, object_hook=object_hook, object_pairs_hook=object_pairs_hook, list_hook=list_hook, encoding=encoding, unicode_errors=unicode_errors, max_buffer_size=max_buffer_size, ext_hook=ext_hook) def pack(o, stream, **kwargs): """ Pack an object and write it to a stream. """ packer = Packer(**kwargs) stream.write(packer.pack(o)) def packb(o, **kwargs): """ Pack an object and return the packed bytes. """ return Packer(**kwargs).pack(o) def unpack(stream, **kwargs): """ Unpack a packed object from a stream. """ object_hook = kwargs.get('object_hook') kwargs['object_hook'] = functools.partial(decode, chain=object_hook) return _unpack(stream, **kwargs) def unpackb(packed, **kwargs): """ Unpack a packed object. """ object_hook = kwargs.get('object_hook') kwargs['object_hook'] = functools.partial(decode, chain=object_hook) return _unpackb(packed, **kwargs) load = unpack loads = unpackb dump = pack dumps = packb def patch(): """ Monkey patch msgpack module to enable support for serializing numpy types. """ setattr(msgpack, 'Packer', Packer) setattr(msgpack, 'Unpacker', Unpacker) setattr(msgpack, 'load', unpack) setattr(msgpack, 'loads', unpackb) setattr(msgpack, 'dump', pack) setattr(msgpack, 'dumps', packb) setattr(msgpack, 'pack', pack) setattr(msgpack, 'packb', packb) setattr(msgpack, 'unpack', unpack) setattr(msgpack, 'unpackb', unpackb) if __name__ == '__main__': try: range = xrange # Python 2 except NameError: pass # Python 3 from unittest import main, TestCase, TestSuite from numpy.testing import assert_equal, assert_array_equal class ThirdParty(object): def __init__(self, foo=b'bar'): self.foo = foo def __eq__(self, other): return isinstance(other, ThirdParty) and self.foo == other.foo class test_numpy_msgpack(TestCase): def setUp(self): patch() def encode_decode(self, x, use_bin_type=False, encoding=None): x_enc = msgpack.packb(x, use_bin_type=use_bin_type) return msgpack.unpackb(x_enc, encoding=encoding) def encode_thirdparty(self, obj): return dict(__thirdparty__=True, foo=obj.foo) def decode_thirdparty(self, obj): if b'__thirdparty__' in obj: return ThirdParty(foo=obj[b'foo']) return obj def encode_decode_thirdparty(self, x, use_bin_type=False, encoding=None): x_enc = msgpack.packb(x, default=self.encode_thirdparty, use_bin_type=use_bin_type) return msgpack.unpackb(x_enc, object_hook=self.decode_thirdparty, encoding=encoding) def test_bin(self): # Since bytes == str in Python 2.7, the following # should pass on both 2.7 and 3.* assert_equal(type(self.encode_decode(b'foo')), bytes) def test_str(self): assert_equal(type(self.encode_decode('foo')), bytes) if sys.version_info.major == 2: assert_equal(type(self.encode_decode(u'foo')), str) # Test non-default string encoding/decoding: assert_equal(type(self.encode_decode(u'foo', True, 'utf=8')), unicode) def test_numpy_scalar_bool(self): x = np.bool_(True) x_rec = self.encode_decode(x) assert_equal(x, x_rec) assert_equal(type(x), type(x_rec)) x = np.bool_(False) x_rec = self.encode_decode(x) assert_equal(x, x_rec) assert_equal(type(x), type(x_rec)) def test_numpy_scalar_float(self): x = np.float32(np.random.rand()) x_rec = self.encode_decode(x) assert_equal(x, x_rec) assert_equal(type(x), type(x_rec)) def test_numpy_scalar_complex(self): x = np.complex64(np.random.rand()+1j*np.random.rand()) x_rec = self.encode_decode(x) assert_equal(x, x_rec) assert_equal(type(x), type(x_rec)) def test_scalar_float(self): x = np.random.rand() x_rec = self.encode_decode(x) assert_equal(x, x_rec) assert_equal(type(x), type(x_rec)) def test_scalar_complex(self): x = np.random.rand()+1j*np.random.rand() x_rec = self.encode_decode(x) assert_equal(x, x_rec) assert_equal(type(x), type(x_rec)) def test_list_numpy_float(self): x = [np.float32(np.random.rand()) for i in range(5)] x_rec = self.encode_decode(x) assert_array_equal(x, x_rec) assert_array_equal([type(e) for e in x], [type(e) for e in x_rec]) def test_list_numpy_float_complex(self): x = [np.float32(np.random.rand()) for i in range(5)] + \ [np.complex128(np.random.rand()+1j*np.random.rand()) for i in range(5)] x_rec = self.encode_decode(x) assert_array_equal(x, x_rec) assert_array_equal([type(e) for e in x], [type(e) for e in x_rec]) def test_list_float(self): x = [np.random.rand() for i in range(5)] x_rec = self.encode_decode(x) assert_array_equal(x, x_rec) assert_array_equal([type(e) for e in x], [type(e) for e in x_rec]) def test_list_float_complex(self): x = [(np.random.rand()+1j*np.random.rand()) for i in range(5)] x_rec = self.encode_decode(x) assert_array_equal(x, x_rec) assert_array_equal([type(e) for e in x], [type(e) for e in x_rec]) def test_list_str(self): x = [b'x'*i for i in range(5)] x_rec = self.encode_decode(x) assert_array_equal(x, x_rec) assert_array_equal([type(e) for e in x_rec], [bytes]*5) def test_dict_float(self): x = {b'foo': 1.0, b'bar': 2.0} x_rec = self.encode_decode(x) assert_array_equal(sorted(x.values()), sorted(x_rec.values())) assert_array_equal([type(e) for e in sorted(x.values())], [type(e) for e in sorted(x_rec.values())]) assert_array_equal(sorted(x.keys()), sorted(x_rec.keys())) assert_array_equal([type(e) for e in sorted(x.keys())], [type(e) for e in sorted(x_rec.keys())]) def test_dict_complex(self): x = {b'foo': 1.0+1.0j, b'bar': 2.0+2.0j} x_rec = self.encode_decode(x) assert_array_equal(sorted(x.values(), key=np.linalg.norm), sorted(x_rec.values(), key=np.linalg.norm)) assert_array_equal([type(e) for e in sorted(x.values(), key=np.linalg.norm)], [type(e) for e in sorted(x_rec.values(), key=np.linalg.norm)]) assert_array_equal(sorted(x.keys()), sorted(x_rec.keys())) assert_array_equal([type(e) for e in sorted(x.keys())], [type(e) for e in sorted(x_rec.keys())]) def test_dict_str(self): x = {b'foo': b'xxx', b'bar': b'yyyy'} x_rec = self.encode_decode(x) assert_array_equal(sorted(x.values()), sorted(x_rec.values())) assert_array_equal([type(e) for e in sorted(x.values())], [type(e) for e in sorted(x_rec.values())]) assert_array_equal(sorted(x.keys()), sorted(x_rec.keys())) assert_array_equal([type(e) for e in sorted(x.keys())], [type(e) for e in sorted(x_rec.keys())]) def test_dict_numpy_float(self): x = {b'foo': np.float32(1.0), b'bar': np.float32(2.0)} x_rec = self.encode_decode(x) assert_array_equal(sorted(x.values()), sorted(x_rec.values())) assert_array_equal([type(e) for e in sorted(x.values())], [type(e) for e in sorted(x_rec.values())]) assert_array_equal(sorted(x.keys()), sorted(x_rec.keys())) assert_array_equal([type(e) for e in sorted(x.keys())], [type(e) for e in sorted(x_rec.keys())]) def test_dict_numpy_complex(self): x = {b'foo': np.complex128(1.0+1.0j), b'bar': np.complex128(2.0+2.0j)} x_rec = self.encode_decode(x) assert_array_equal(sorted(x.values(), key=np.linalg.norm), sorted(x_rec.values(), key=np.linalg.norm)) assert_array_equal([type(e) for e in sorted(x.values(), key=np.linalg.norm)], [type(e) for e in sorted(x_rec.values(), key=np.linalg.norm)]) assert_array_equal(sorted(x.keys()), sorted(x_rec.keys())) assert_array_equal([type(e) for e in sorted(x.keys())], [type(e) for e in sorted(x_rec.keys())]) def test_numpy_array_float(self): x = np.random.rand(5).astype(np.float32) x_rec = self.encode_decode(x) assert_array_equal(x, x_rec) assert_equal(x.dtype, x_rec.dtype) def test_numpy_array_complex(self): x = (np.random.rand(5)+1j*np.random.rand(5)).astype(np.complex128) x_rec = self.encode_decode(x) assert_array_equal(x, x_rec) assert_equal(x.dtype, x_rec.dtype) def test_numpy_array_float_2d(self): x = np.random.rand(5,5).astype(np.float32) x_rec = self.encode_decode(x) assert_array_equal(x, x_rec) assert_equal(x.dtype, x_rec.dtype) def test_numpy_array_str(self): x = np.array([b'aaa', b'bbbb', b'ccccc']) x_rec = self.encode_decode(x) assert_array_equal(x, x_rec) assert_equal(x.dtype, x_rec.dtype) def test_numpy_array_mixed(self): x = np.array([(1, 2, b'a', [1.0, 2.0])], np.dtype([('arg0', np.uint32), ('arg1', np.uint32), ('arg2', 'S1'), ('arg3', np.float32, (2,))])) x_rec = self.encode_decode(x) assert_array_equal(x, x_rec) assert_equal(x.dtype, x_rec.dtype) def test_list_mixed(self): x = [1.0, np.float32(3.5), np.complex128(4.25), b'foo'] x_rec = self.encode_decode(x) assert_array_equal(x, x_rec) assert_array_equal([type(e) for e in x], [type(e) for e in x_rec]) def test_chain(self): x = ThirdParty(foo=b'test marshal/unmarshal') x_rec = self.encode_decode_thirdparty(x) self.assertEqual(x, x_rec) main() msgpack-numpy-0.4.4/setup.cfg000066400000000000000000000000341335256143400161360ustar00rootroot00000000000000[bdist_wheel] universal = 1 msgpack-numpy-0.4.4/setup.py000066400000000000000000000032401335256143400160310ustar00rootroot00000000000000#!/usr/bin/env python import os import re from setuptools import setup NAME = 'msgpack-numpy' VERSION = '0.4.4' AUTHOR = 'Lev E. Givon' AUTHOR_EMAIL = 'lev@columbia.edu' URL = 'https://github.com/lebedov/msgpack-numpy' DESCRIPTION = 'Numpy data serialization using msgpack' with open('README.rst', 'r') as f: LONG_DESCRIPTION = f.read() LONG_DESCRIPTION = re.search('.*(^Package Description.*)', LONG_DESCRIPTION, re.MULTILINE|re.DOTALL).group(1) DOWNLOAD_URL = URL LICENSE = 'BSD' CLASSIFIERS = [ 'Development Status :: 3 - Alpha', 'Intended Audience :: Developers', 'Intended Audience :: Science/Research', 'License :: OSI Approved :: BSD License', 'Operating System :: OS Independent', 'Programming Language :: Python', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Topic :: Scientific/Engineering', 'Topic :: Software Development'] if __name__ == "__main__": if os.path.exists('MANIFEST'): os.remove('MANIFEST') setup( name = NAME, version = VERSION, author = AUTHOR, author_email = AUTHOR_EMAIL, license = LICENSE, classifiers = CLASSIFIERS, description = DESCRIPTION, long_description = LONG_DESCRIPTION, url = URL, py_modules = ['msgpack_numpy'], install_requires = ['numpy>=1.9.0', 'msgpack>=0.3.0'] )