cloudpickle-0.5.2/0000775000175000017620000000000013204761670015161 5ustar ogriselogrisel00000000000000cloudpickle-0.5.2/tests/0000775000175000017620000000000013204761670016323 5ustar ogriselogrisel00000000000000cloudpickle-0.5.2/tests/testutils.py0000664000175000017620000000756313203305571020741 0ustar ogriselogrisel00000000000000import sys import os import os.path as op import tempfile from subprocess import Popen, check_output, PIPE, STDOUT, CalledProcessError from cloudpickle import dumps from pickle import loads try: from suprocess import TimeoutExpired timeout_supported = True except ImportError: # no support for timeout in Python 2 class TimeoutExpired(Exception): pass timeout_supported = False def subprocess_pickle_echo(input_data, protocol=None): """Echo function with a child Python process Pickle the input data into a buffer, send it to a subprocess via stdin, expect the subprocess to unpickle, re-pickle that data back and send it back to the parent process via stdout for final unpickling. >>> subprocess_pickle_echo([1, 'a', None]) [1, 'a', None] """ pickled_input_data = dumps(input_data, protocol=protocol) cmd = [sys.executable, __file__] # run then pickle_echo() in __main__ cloudpickle_repo_folder = op.normpath( op.join(op.dirname(__file__), '..')) cwd = cloudpickle_repo_folder pythonpath = "{src}/tests:{src}".format(src=cloudpickle_repo_folder) env = {'PYTHONPATH': pythonpath} proc = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE, cwd=cwd, env=env) try: comm_kwargs = {} if timeout_supported: comm_kwargs['timeout'] = 5 out, err = proc.communicate(pickled_input_data, **comm_kwargs) if proc.returncode != 0 or len(err): message = "Subprocess returned %d: " % proc.returncode message += err.decode('utf-8') raise RuntimeError(message) return loads(out) except TimeoutExpired: proc.kill() out, err = proc.communicate() message = u"\n".join([out.decode('utf-8'), err.decode('utf-8')]) raise RuntimeError(message) def pickle_echo(stream_in=None, stream_out=None, protocol=None): """Read a pickle from stdin and pickle it back to stdout""" if stream_in is None: stream_in = sys.stdin if stream_out is None: stream_out = sys.stdout # Force the use of bytes streams under Python 3 if hasattr(stream_in, 'buffer'): stream_in = stream_in.buffer if hasattr(stream_out, 'buffer'): stream_out = stream_out.buffer input_bytes = stream_in.read() stream_in.close() unpickled_content = loads(input_bytes) stream_out.write(dumps(unpickled_content, protocol=protocol)) stream_out.close() def assert_run_python_script(source_code, timeout=5): """Utility to help check pickleability of objects defined in __main__ The script provided in the source code should return 0 and not print anything on stderr or stdout. """ fd, source_file = tempfile.mkstemp(suffix='_src_test_cloudpickle.py') os.close(fd) try: with open(source_file, 'wb') as f: f.write(source_code.encode('utf-8')) cmd = [sys.executable, source_file] cloudpickle_repo_folder = op.normpath( op.join(op.dirname(__file__), '..')) pythonpath = "{src}/tests:{src}".format(src=cloudpickle_repo_folder) kwargs = { 'cwd': cloudpickle_repo_folder, 'stderr': STDOUT, 'env': {'PYTHONPATH': pythonpath}, } if timeout_supported: kwargs['timeout'] = timeout try: try: out = check_output(cmd, **kwargs) except CalledProcessError as e: raise RuntimeError(u"script errored with output:\n%s" % e.output.decode('utf-8')) if out != b"": raise AssertionError(out.decode('utf-8')) except TimeoutExpired as e: raise RuntimeError(u"script timeout, output so far:\n%s" % e.output.decode('utf-8')) finally: os.unlink(source_file) if __name__ == '__main__': pickle_echo() cloudpickle-0.5.2/tests/cloudpickle_file_test.py0000664000175000017620000001010313203305571023215 0ustar ogriselogrisel00000000000000import unittest import tempfile import os import shutil import pickle import sys from io import StringIO import pytest from mock import patch, mock_open import cloudpickle class CloudPickleFileTests(unittest.TestCase): """In Cloudpickle, expected behaviour when pickling an opened file is to send its contents over the wire and seek to the same position.""" def setUp(self): self.tmpdir = tempfile.mkdtemp() self.tmpfilepath = os.path.join(self.tmpdir, 'testfile') self.teststring = u'Hello world!' def tearDown(self): shutil.rmtree(self.tmpdir) def test_empty_file(self): # Empty file open(self.tmpfilepath, 'w').close() with open(self.tmpfilepath, 'r') as f: self.assertEqual('', pickle.loads(cloudpickle.dumps(f)).read()) os.remove(self.tmpfilepath) def test_closed_file(self): # Write & close with open(self.tmpfilepath, 'w') as f: f.write(self.teststring) with pytest.raises(pickle.PicklingError) as excinfo: cloudpickle.dumps(f) assert "Cannot pickle closed files" in str(excinfo.value) os.remove(self.tmpfilepath) def test_r_mode(self): # Write & close with open(self.tmpfilepath, 'w') as f: f.write(self.teststring) # Open for reading with open(self.tmpfilepath, 'r') as f: new_f = pickle.loads(cloudpickle.dumps(f)) self.assertEqual(self.teststring, new_f.read()) os.remove(self.tmpfilepath) def test_w_mode(self): with open(self.tmpfilepath, 'w') as f: f.write(self.teststring) f.seek(0) self.assertRaises(pickle.PicklingError, lambda: cloudpickle.dumps(f)) os.remove(self.tmpfilepath) def test_plus_mode(self): # Write, then seek to 0 with open(self.tmpfilepath, 'w+') as f: f.write(self.teststring) f.seek(0) new_f = pickle.loads(cloudpickle.dumps(f)) self.assertEqual(self.teststring, new_f.read()) os.remove(self.tmpfilepath) def test_seek(self): # Write, then seek to arbitrary position with open(self.tmpfilepath, 'w+') as f: f.write(self.teststring) f.seek(4) unpickled = pickle.loads(cloudpickle.dumps(f)) # unpickled StringIO is at position 4 self.assertEqual(4, unpickled.tell()) self.assertEqual(self.teststring[4:], unpickled.read()) # but unpickled StringIO also contained the start unpickled.seek(0) self.assertEqual(self.teststring, unpickled.read()) os.remove(self.tmpfilepath) @pytest.mark.skipif(sys.version_info >= (3,), reason="only works on Python 2.x") def test_temp_file(self): with tempfile.NamedTemporaryFile(mode='ab+') as fp: fp.write(self.teststring.encode('UTF-8')) fp.seek(0) f = fp.file # FIXME this doesn't work yet: cloudpickle.dumps(fp) newfile = pickle.loads(cloudpickle.dumps(f)) self.assertEqual(self.teststring, newfile.read()) def test_pickling_special_file_handles(self): # Warning: if you want to run your tests with nose, add -s option for out in sys.stdout, sys.stderr: # Regression test for SPARK-3415 self.assertEqual(out, pickle.loads(cloudpickle.dumps(out))) self.assertRaises(pickle.PicklingError, lambda: cloudpickle.dumps(sys.stdin)) def NOT_WORKING_test_tty(self): # FIXME: Mocking 'file' is not trivial... and fails for now from sys import version_info if version_info.major == 2: import __builtin__ as builtins # pylint:disable=import-error else: import builtins # pylint:disable=import-error with patch.object(builtins, 'open', mock_open(), create=True): with open('foo', 'w+') as handle: cloudpickle.dumps(handle) if __name__ == '__main__': unittest.main() cloudpickle-0.5.2/tests/__init__.py0000664000175000017620000000000013176032760020421 0ustar ogriselogrisel00000000000000cloudpickle-0.5.2/tests/cloudpickle_test.py0000664000175000017620000007177113204761500022237 0ustar ogriselogrisel00000000000000from __future__ import division import abc import collections import base64 import functools import imp from io import BytesIO import itertools import logging from operator import itemgetter, attrgetter import pickle import platform import random import subprocess import sys import textwrap import unittest import weakref try: from StringIO import StringIO except ImportError: from io import StringIO import pytest try: # try importing numpy and scipy. These are not hard dependencies and # tests should be skipped if these modules are not available import numpy as np import scipy.special as spp except ImportError: np = None spp = None try: # Ditto for Tornado import tornado except ImportError: tornado = None import cloudpickle from cloudpickle.cloudpickle import _find_module, _make_empty_cell, cell_set from .testutils import subprocess_pickle_echo from .testutils import assert_run_python_script def pickle_depickle(obj, protocol=cloudpickle.DEFAULT_PROTOCOL): """Helper function to test whether object pickled with cloudpickle can be depickled with pickle """ return pickle.loads(cloudpickle.dumps(obj, protocol=protocol)) class CloudPicklerTest(unittest.TestCase): def setUp(self): self.file_obj = StringIO() self.cloudpickler = cloudpickle.CloudPickler(self.file_obj, 2) class CloudPickleTest(unittest.TestCase): protocol = cloudpickle.DEFAULT_PROTOCOL def test_itemgetter(self): d = range(10) getter = itemgetter(1) getter2 = pickle_depickle(getter, protocol=self.protocol) self.assertEqual(getter(d), getter2(d)) getter = itemgetter(0, 3) getter2 = pickle_depickle(getter, protocol=self.protocol) self.assertEqual(getter(d), getter2(d)) def test_attrgetter(self): class C(object): def __getattr__(self, item): return item d = C() getter = attrgetter("a") getter2 = pickle_depickle(getter, protocol=self.protocol) self.assertEqual(getter(d), getter2(d)) getter = attrgetter("a", "b") getter2 = pickle_depickle(getter, protocol=self.protocol) self.assertEqual(getter(d), getter2(d)) d.e = C() getter = attrgetter("e.a") getter2 = pickle_depickle(getter, protocol=self.protocol) self.assertEqual(getter(d), getter2(d)) getter = attrgetter("e.a", "e.b") getter2 = pickle_depickle(getter, protocol=self.protocol) self.assertEqual(getter(d), getter2(d)) # Regression test for SPARK-3415 def test_pickling_file_handles(self): out1 = sys.stderr out2 = pickle.loads(cloudpickle.dumps(out1)) self.assertEqual(out1, out2) def test_func_globals(self): class Unpicklable(object): def __reduce__(self): raise Exception("not picklable") global exit exit = Unpicklable() self.assertRaises(Exception, lambda: cloudpickle.dumps(exit)) def foo(): sys.exit(0) func_code = getattr(foo, '__code__', None) if func_code is None: # PY2 backwards compatibility func_code = foo.func_code self.assertTrue("exit" in func_code.co_names) cloudpickle.dumps(foo) def test_buffer(self): try: buffer_obj = buffer("Hello") buffer_clone = pickle_depickle(buffer_obj, protocol=self.protocol) self.assertEqual(buffer_clone, str(buffer_obj)) buffer_obj = buffer("Hello", 2, 3) buffer_clone = pickle_depickle(buffer_obj, protocol=self.protocol) self.assertEqual(buffer_clone, str(buffer_obj)) except NameError: # Python 3 does no longer support buffers pass def test_memoryview(self): buffer_obj = memoryview(b"Hello") self.assertEqual(pickle_depickle(buffer_obj, protocol=self.protocol), buffer_obj.tobytes()) @pytest.mark.skipif(sys.version_info < (3, 4), reason="non-contiguous memoryview not implemented in " "old Python versions") def test_sliced_and_non_contiguous_memoryview(self): buffer_obj = memoryview(b"Hello!" * 3)[2:15:2] self.assertEqual(pickle_depickle(buffer_obj, protocol=self.protocol), buffer_obj.tobytes()) def test_large_memoryview(self): buffer_obj = memoryview(b"Hello!" * int(1e7)) self.assertEqual(pickle_depickle(buffer_obj, protocol=self.protocol), buffer_obj.tobytes()) def test_lambda(self): self.assertEqual(pickle_depickle(lambda: 1)(), 1) def test_nested_lambdas(self): a, b = 1, 2 f1 = lambda x: x + a f2 = lambda x: f1(x) // b self.assertEqual(pickle_depickle(f2, protocol=self.protocol)(1), 1) def test_recursive_closure(self): def f1(): def g(): return g return g def f2(base): def g(n): return base if n <= 1 else n * g(n - 1) return g g1 = pickle_depickle(f1()) self.assertEqual(g1(), g1) g2 = pickle_depickle(f2(2)) self.assertEqual(g2(5), 240) def test_closure_none_is_preserved(self): def f(): """a function with no closure cells """ self.assertTrue( f.__closure__ is None, msg='f actually has closure cells!', ) g = pickle_depickle(f, protocol=self.protocol) self.assertTrue( g.__closure__ is None, msg='g now has closure cells even though f does not', ) def test_empty_cell_preserved(self): def f(): if False: # pragma: no cover cell = None def g(): cell # NameError, unbound free variable return g g1 = f() with pytest.raises(NameError): g1() g2 = pickle_depickle(g1, protocol=self.protocol) with pytest.raises(NameError): g2() def test_unhashable_closure(self): def f(): s = set((1, 2)) # mutable set is unhashable def g(): return len(s) return g g = pickle_depickle(f()) self.assertEqual(g(), 2) def test_dynamically_generated_class_that_uses_super(self): class Base(object): def method(self): return 1 class Derived(Base): "Derived Docstring" def method(self): return super(Derived, self).method() + 1 self.assertEqual(Derived().method(), 2) # Pickle and unpickle the class. UnpickledDerived = pickle_depickle(Derived, protocol=self.protocol) self.assertEqual(UnpickledDerived().method(), 2) # We have special logic for handling __doc__ because it's a readonly # attribute on PyPy. self.assertEqual(UnpickledDerived.__doc__, "Derived Docstring") # Pickle and unpickle an instance. orig_d = Derived() d = pickle_depickle(orig_d, protocol=self.protocol) self.assertEqual(d.method(), 2) def test_cycle_in_classdict_globals(self): class C(object): def it_works(self): return "woohoo!" C.C_again = C C.instance_of_C = C() depickled_C = pickle_depickle(C, protocol=self.protocol) depickled_instance = pickle_depickle(C()) # Test instance of depickled class. self.assertEqual(depickled_C().it_works(), "woohoo!") self.assertEqual(depickled_C.C_again().it_works(), "woohoo!") self.assertEqual(depickled_C.instance_of_C.it_works(), "woohoo!") self.assertEqual(depickled_instance.it_works(), "woohoo!") @pytest.mark.skipif(sys.version_info >= (3, 4) and sys.version_info < (3, 4, 3), reason="subprocess has a bug in 3.4.0 to 3.4.2") def test_locally_defined_function_and_class(self): LOCAL_CONSTANT = 42 def some_function(x, y): return (x + y) / LOCAL_CONSTANT # pickle the function definition self.assertEqual(pickle_depickle(some_function, protocol=self.protocol)(41, 1), 1) self.assertEqual(pickle_depickle(some_function, protocol=self.protocol)(81, 3), 2) hidden_constant = lambda: LOCAL_CONSTANT class SomeClass(object): """Overly complicated class with nested references to symbols""" def __init__(self, value): self.value = value def one(self): return LOCAL_CONSTANT / hidden_constant() def some_method(self, x): return self.one() + some_function(x, 1) + self.value # pickle the class definition clone_class = pickle_depickle(SomeClass, protocol=self.protocol) self.assertEqual(clone_class(1).one(), 1) self.assertEqual(clone_class(5).some_method(41), 7) clone_class = subprocess_pickle_echo(SomeClass, protocol=self.protocol) self.assertEqual(clone_class(5).some_method(41), 7) # pickle the class instances self.assertEqual(pickle_depickle(SomeClass(1)).one(), 1) self.assertEqual(pickle_depickle(SomeClass(5)).some_method(41), 7) new_instance = subprocess_pickle_echo(SomeClass(5), protocol=self.protocol) self.assertEqual(new_instance.some_method(41), 7) # pickle the method instances self.assertEqual(pickle_depickle(SomeClass(1).one)(), 1) self.assertEqual(pickle_depickle(SomeClass(5).some_method)(41), 7) new_method = subprocess_pickle_echo(SomeClass(5).some_method, protocol=self.protocol) self.assertEqual(new_method(41), 7) def test_partial(self): partial_obj = functools.partial(min, 1) partial_clone = pickle_depickle(partial_obj, protocol=self.protocol) self.assertEqual(partial_clone(4), 1) @pytest.mark.skipif(platform.python_implementation() == 'PyPy', reason="Skip numpy and scipy tests on PyPy") def test_ufunc(self): # test a numpy ufunc (universal function), which is a C-based function # that is applied on a numpy array if np: # simple ufunc: np.add self.assertEqual(pickle_depickle(np.add), np.add) else: # skip if numpy is not available pass if spp: # custom ufunc: scipy.special.iv self.assertEqual(pickle_depickle(spp.iv), spp.iv) else: # skip if scipy is not available pass def test_save_unsupported(self): sio = StringIO() pickler = cloudpickle.CloudPickler(sio, 2) with pytest.raises(pickle.PicklingError) as excinfo: pickler.save_unsupported("test") assert "Cannot pickle objects of type" in str(excinfo.value) def test_loads_namespace(self): obj = 1, 2, 3, 4 returned_obj = cloudpickle.loads(cloudpickle.dumps(obj)) self.assertEqual(obj, returned_obj) def test_load_namespace(self): obj = 1, 2, 3, 4 bio = BytesIO() cloudpickle.dump(obj, bio) bio.seek(0) returned_obj = cloudpickle.load(bio) self.assertEqual(obj, returned_obj) def test_generator(self): def some_generator(cnt): for i in range(cnt): yield i gen2 = pickle_depickle(some_generator, protocol=self.protocol) assert type(gen2(3)) == type(some_generator(3)) assert list(gen2(3)) == list(range(3)) def test_classmethod(self): class A(object): @staticmethod def test_sm(): return "sm" @classmethod def test_cm(cls): return "cm" sm = A.__dict__["test_sm"] cm = A.__dict__["test_cm"] A.test_sm = pickle_depickle(sm, protocol=self.protocol) A.test_cm = pickle_depickle(cm, protocol=self.protocol) self.assertEqual(A.test_sm(), "sm") self.assertEqual(A.test_cm(), "cm") def test_method_descriptors(self): f = pickle_depickle(str.upper) self.assertEqual(f('abc'), 'ABC') def test_instancemethods_without_self(self): class F(object): def f(self, x): return x + 1 g = pickle_depickle(F.f) self.assertEqual(g.__name__, F.f.__name__) if sys.version_info[0] < 3: self.assertEqual(g.im_class.__name__, F.f.im_class.__name__) # self.assertEqual(g(F(), 1), 2) # still fails def test_module(self): pickle_clone = pickle_depickle(pickle, protocol=self.protocol) self.assertEqual(pickle, pickle_clone) def test_dynamic_module(self): mod = imp.new_module('mod') code = ''' x = 1 def f(y): return x + y class Foo: def method(self, x): return f(x) ''' exec(textwrap.dedent(code), mod.__dict__) mod2 = pickle_depickle(mod, protocol=self.protocol) self.assertEqual(mod.x, mod2.x) self.assertEqual(mod.f(5), mod2.f(5)) self.assertEqual(mod.Foo().method(5), mod2.Foo().method(5)) if platform.python_implementation() != 'PyPy': # XXX: this fails with excessive recursion on PyPy. mod3 = subprocess_pickle_echo(mod, protocol=self.protocol) self.assertEqual(mod.x, mod3.x) self.assertEqual(mod.f(5), mod3.f(5)) self.assertEqual(mod.Foo().method(5), mod3.Foo().method(5)) # Test dynamic modules when imported back are singletons mod1, mod2 = pickle_depickle([mod, mod]) self.assertEqual(id(mod1), id(mod2)) def test_find_module(self): import pickle # ensure this test is decoupled from global imports _find_module('pickle') with pytest.raises(ImportError): _find_module('invalid_module') with pytest.raises(ImportError): valid_module = imp.new_module('valid_module') _find_module('valid_module') def test_Ellipsis(self): self.assertEqual(Ellipsis, pickle_depickle(Ellipsis, protocol=self.protocol)) def test_NotImplemented(self): ExcClone = pickle_depickle(NotImplemented, protocol=self.protocol) self.assertEqual(NotImplemented, ExcClone) def test_builtin_function_without_module(self): on = object.__new__ on_depickled = pickle_depickle(on, protocol=self.protocol) self.assertEqual(type(on_depickled(object)), type(object())) fi = itertools.chain.from_iterable fi_depickled = pickle_depickle(fi, protocol=self.protocol) self.assertEqual(list(fi([[1, 2], [3, 4]])), [1, 2, 3, 4]) @pytest.mark.skipif(tornado is None, reason="test needs Tornado installed") def test_tornado_coroutine(self): # Pickling a locally defined coroutine function from tornado import gen, ioloop @gen.coroutine def f(x, y): yield gen.sleep(x) raise gen.Return(y + 1) @gen.coroutine def g(y): res = yield f(0.01, y) raise gen.Return(res + 1) data = cloudpickle.dumps([g, g]) f = g = None g2, g3 = pickle.loads(data) self.assertTrue(g2 is g3) loop = ioloop.IOLoop.current() res = loop.run_sync(functools.partial(g2, 5)) self.assertEqual(res, 7) def test_extended_arg(self): # Functions with more than 65535 global vars prefix some global # variable references with the EXTENDED_ARG opcode. nvars = 65537 + 258 names = ['g%d' % i for i in range(1, nvars)] r = random.Random(42) d = dict([(name, r.randrange(100)) for name in names]) # def f(x): # x = g1, g2, ... # return zlib.crc32(bytes(bytearray(x))) code = """ import zlib def f(): x = {tup} return zlib.crc32(bytes(bytearray(x))) """.format(tup=', '.join(names)) exec(textwrap.dedent(code), d, d) f = d['f'] res = f() data = cloudpickle.dumps([f, f]) d = f = None f2, f3 = pickle.loads(data) self.assertTrue(f2 is f3) self.assertEqual(f2(), res) def test_submodule(self): # Function that refers (by attribute) to a sub-module of a package. # Choose any module NOT imported by __init__ of its parent package # examples in standard library include: # - http.cookies, unittest.mock, curses.textpad, xml.etree.ElementTree global xml # imitate performing this import at top of file import xml.etree.ElementTree def example(): x = xml.etree.ElementTree.Comment # potential AttributeError s = cloudpickle.dumps(example) # refresh the environment, i.e., unimport the dependency del xml for item in list(sys.modules): if item.split('.')[0] == 'xml': del sys.modules[item] # deserialise f = pickle.loads(s) f() # perform test for error def test_submodule_closure(self): # Same as test_submodule except the package is not a global def scope(): import xml.etree.ElementTree def example(): x = xml.etree.ElementTree.Comment # potential AttributeError return example example = scope() s = cloudpickle.dumps(example) # refresh the environment (unimport dependency) for item in list(sys.modules): if item.split('.')[0] == 'xml': del sys.modules[item] f = cloudpickle.loads(s) f() # test def test_multiprocess(self): # running a function pickled by another process (a la dask.distributed) def scope(): import curses.textpad def example(): x = xml.etree.ElementTree.Comment x = curses.textpad.Textbox return example global xml import xml.etree.ElementTree example = scope() s = cloudpickle.dumps(example) # choose "subprocess" rather than "multiprocessing" because the latter # library uses fork to preserve the parent environment. command = ("import pickle, base64; " "pickle.loads(base64.b32decode('" + base64.b32encode(s).decode('ascii') + "'))()") assert not subprocess.call([sys.executable, '-c', command]) def test_import(self): # like test_multiprocess except subpackage modules referenced directly # (unlike test_submodule) global etree def scope(): import curses.textpad as foobar def example(): x = etree.Comment x = foobar.Textbox return example example = scope() import xml.etree.ElementTree as etree s = cloudpickle.dumps(example) command = ("import pickle, base64; " "pickle.loads(base64.b32decode('" + base64.b32encode(s).decode('ascii') + "'))()") assert not subprocess.call([sys.executable, '-c', command]) def test_cell_manipulation(self): cell = _make_empty_cell() with pytest.raises(ValueError): cell.cell_contents ob = object() cell_set(cell, ob) self.assertTrue( cell.cell_contents is ob, msg='cell contents not set correctly', ) def test_logger(self): logger = logging.getLogger('cloudpickle.dummy_test_logger') pickled = pickle_depickle(logger, protocol=self.protocol) self.assertTrue(pickled is logger, (pickled, logger)) dumped = cloudpickle.dumps(logger) code = """if 1: import cloudpickle, logging logging.basicConfig(level=logging.INFO) logger = cloudpickle.loads(%(dumped)r) logger.info('hello') """ % locals() proc = subprocess.Popen([sys.executable, "-c", code], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) out, _ = proc.communicate() self.assertEqual(proc.wait(), 0) self.assertEqual(out.strip().decode(), 'INFO:cloudpickle.dummy_test_logger:hello') def test_abc(self): @abc.abstractmethod def foo(self): raise NotImplementedError('foo') # Invoke the metaclass directly rather than using class syntax for # python 2/3 compat. AbstractClass = abc.ABCMeta('AbstractClass', (object,), {'foo': foo}) class ConcreteClass(AbstractClass): def foo(self): return 'it works!' depickled_base = pickle_depickle(AbstractClass, protocol=self.protocol) depickled_class = pickle_depickle(ConcreteClass, protocol=self.protocol) depickled_instance = pickle_depickle(ConcreteClass()) self.assertEqual(depickled_class().foo(), 'it works!') self.assertEqual(depickled_instance.foo(), 'it works!') # assertRaises doesn't return a contextmanager in python 2.6 :(. self.failUnlessRaises(TypeError, depickled_base) class DepickledBaseSubclass(depickled_base): def foo(self): return 'it works for realz!' self.assertEqual(DepickledBaseSubclass().foo(), 'it works for realz!') def test_weakset_identity_preservation(self): # Test that weaksets don't lose all their inhabitants if they're # pickled in a larger data structure that includes other references to # their inhabitants. class SomeClass(object): def __init__(self, x): self.x = x obj1, obj2, obj3 = SomeClass(1), SomeClass(2), SomeClass(3) things = [weakref.WeakSet([obj1, obj2]), obj1, obj2, obj3] result = pickle_depickle(things, protocol=self.protocol) weakset, depickled1, depickled2, depickled3 = result self.assertEqual(depickled1.x, 1) self.assertEqual(depickled2.x, 2) self.assertEqual(depickled3.x, 3) self.assertEqual(len(weakset), 2) self.assertEqual(set(weakset), set([depickled1, depickled2])) def test_faulty_module(self): for module_name in ['_faulty_module', '_missing_module', None]: class FaultyModule(object): def __getattr__(self, name): # This throws an exception while looking up within # pickle.whichmodule or getattr(module, name, None) raise Exception() class Foo(object): __module__ = module_name def foo(self): return "it works!" def foo(): return "it works!" foo.__module__ = module_name sys.modules["_faulty_module"] = FaultyModule() try: # Test whichmodule in save_global. self.assertEqual(pickle_depickle(Foo()).foo(), "it works!") # Test whichmodule in save_function. cloned = pickle_depickle(foo, protocol=self.protocol) self.assertEqual(cloned(), "it works!") finally: sys.modules.pop("_faulty_module", None) def test_dynamic_pytest_module(self): # Test case for pull request https://github.com/cloudpipe/cloudpickle/pull/116 import py def f(): s = py.builtin.set([1]) return s.pop() # some setup is required to allow pytest apimodules to be correctly # serializable. from cloudpickle import CloudPickler CloudPickler.dispatch[type(py.builtin)] = CloudPickler.save_module g = cloudpickle.loads(cloudpickle.dumps(f)) result = g() self.assertEqual(1, result) def test_function_module_name(self): func = lambda x: x cloned = pickle_depickle(func, protocol=self.protocol) self.assertEqual(cloned.__module__, func.__module__) def test_function_qualname(self): def func(x): return x # Default __qualname__ attribute (Python 3 only) if hasattr(func, '__qualname__'): cloned = pickle_depickle(func, protocol=self.protocol) self.assertEqual(cloned.__qualname__, func.__qualname__) # Mutated __qualname__ attribute func.__qualname__ = '' cloned = pickle_depickle(func, protocol=self.protocol) self.assertEqual(cloned.__qualname__, func.__qualname__) def test_namedtuple(self): MyTuple = collections.namedtuple('MyTuple', ['a', 'b', 'c']) t = MyTuple(1, 2, 3) depickled_t, depickled_MyTuple = pickle_depickle( [t, MyTuple], protocol=self.protocol) self.assertTrue(isinstance(depickled_t, depickled_MyTuple)) self.assertEqual((depickled_t.a, depickled_t.b, depickled_t.c), (1, 2, 3)) self.assertEqual((depickled_t[0], depickled_t[1], depickled_t[2]), (1, 2, 3)) self.assertEqual(depickled_MyTuple.__name__, 'MyTuple') self.assertTrue(issubclass(depickled_MyTuple, tuple)) def test_builtin_type__new__(self): # Functions occasionally take the __new__ of these types as default # parameters for factories. For example, on Python 3.3, # `tuple.__new__` is a default value for some methods of namedtuple. for t in list, tuple, set, frozenset, dict, object: cloned = pickle_depickle(t.__new__, protocol=self.protocol) self.assertTrue(cloned is t.__new__) def test_interactively_defined_function(self): # Check that callables defined in the __main__ module of a Python # script (or jupyter kernel) can be pickled / unpickled / executed. code = """\ from testutils import subprocess_pickle_echo CONSTANT = 42 class Foo(object): def method(self, x): return x foo = Foo() def f0(x): return x ** 2 def f1(): return Foo def f2(x): return Foo().method(x) def f3(): return Foo().method(CONSTANT) def f4(x): return foo.method(x) cloned = subprocess_pickle_echo(lambda x: x**2, protocol={protocol}) assert cloned(3) == 9 cloned = subprocess_pickle_echo(f0, protocol={protocol}) assert cloned(3) == 9 cloned = subprocess_pickle_echo(Foo, protocol={protocol}) assert cloned().method(2) == Foo().method(2) cloned = subprocess_pickle_echo(Foo(), protocol={protocol}) assert cloned.method(2) == Foo().method(2) cloned = subprocess_pickle_echo(f1, protocol={protocol}) assert cloned()().method('a') == f1()().method('a') cloned = subprocess_pickle_echo(f2, protocol={protocol}) assert cloned(2) == f2(2) cloned = subprocess_pickle_echo(f3, protocol={protocol}) assert cloned() == f3() cloned = subprocess_pickle_echo(f4, protocol={protocol}) assert cloned(2) == f4(2) """.format(protocol=self.protocol) assert_run_python_script(textwrap.dedent(code)) @pytest.mark.skipif(sys.version_info >= (3, 0), reason="hardcoded pickle bytes for 2.7") def test_function_pickle_compat_0_4_0(self): # The result of `cloudpickle.dumps(lambda x: x)` in cloudpickle 0.4.0, # Python 2.7 pickled = (b'\x80\x02ccloudpickle.cloudpickle\n_fill_function\nq\x00(c' b'cloudpickle.cloudpickle\n_make_skel_func\nq\x01ccloudpickle.clou' b'dpickle\n_builtin_type\nq\x02U\x08CodeTypeq\x03\x85q\x04Rq\x05(K' b'\x01K\x01K\x01KCU\x04|\x00\x00Sq\x06N\x85q\x07)U\x01xq\x08\x85q' b'\tU\x07q\nU\x08q\x0bK\x01U\x00q\x0c))tq\rRq\x0eJ' b'\xff\xff\xff\xff}q\x0f\x87q\x10Rq\x11}q\x12N}q\x13NtR.') self.assertEquals(42, cloudpickle.loads(pickled)(42)) @pytest.mark.skipif(sys.version_info >= (3, 0), reason="hardcoded pickle bytes for 2.7") def test_function_pickle_compat_0_4_1(self): # The result of `cloudpickle.dumps(lambda x: x)` in cloudpickle 0.4.1, # Python 2.7 pickled = (b'\x80\x02ccloudpickle.cloudpickle\n_fill_function\nq\x00(c' b'cloudpickle.cloudpickle\n_make_skel_func\nq\x01ccloudpickle.clou' b'dpickle\n_builtin_type\nq\x02U\x08CodeTypeq\x03\x85q\x04Rq\x05(K' b'\x01K\x01K\x01KCU\x04|\x00\x00Sq\x06N\x85q\x07)U\x01xq\x08\x85q' b'\tU\x07q\nU\x08q\x0bK\x01U\x00q\x0c))tq\rRq\x0eJ' b'\xff\xff\xff\xff}q\x0f\x87q\x10Rq\x11}q\x12N}q\x13U\x08__main__q' b'\x14NtR.') self.assertEquals(42, cloudpickle.loads(pickled)(42)) class Protocol2CloudPickleTest(CloudPickleTest): protocol = 2 if __name__ == '__main__': unittest.main() cloudpickle-0.5.2/MANIFEST.in0000664000175000017620000000040613176032760016716 0ustar ogriselogrisel00000000000000include AUTHORS.rst include CONTRIBUTING.rst include HISTORY.rst include LICENSE include README.rst include README.md recursive-include tests * recursive-exclude * __pycache__ recursive-exclude * *.py[co] recursive-include docs *.rst conf.py Makefile make.bat cloudpickle-0.5.2/LICENSE0000664000175000017620000000333213176032760016166 0ustar ogriselogrisel00000000000000This module was extracted from the `cloud` package, developed by PiCloud, Inc. Copyright (c) 2015, Cloudpickle contributors. Copyright (c) 2012, Regents of the University of California. Copyright (c) 2009 PiCloud, Inc. http://www.picloud.com. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of the University of California, Berkeley nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cloudpickle-0.5.2/setup.py0000664000175000017620000000244613204761500016671 0ustar ogriselogrisel00000000000000#!/usr/bin/env python # -*- coding: utf-8 -*- try: from setuptools import setup except ImportError: from distutils.core import setup dist = setup( name='cloudpickle', version='0.5.2', description='Extended pickling support for Python objects', author='Cloudpipe', author_email='cloudpipe@googlegroups.com', url='https://github.com/cloudpipe/cloudpickle', license='LICENSE.txt', packages=['cloudpickle'], long_description=open('README.md').read(), classifiers=[ 'Development Status :: 4 - Beta', 'Intended Audience :: Developers', 'License :: OSI Approved :: BSD License', 'Operating System :: POSIX', 'Operating System :: Microsoft :: Windows', 'Operating System :: MacOS :: MacOS X', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', 'Topic :: Software Development :: Libraries :: Python Modules', 'Topic :: Scientific/Engineering', 'Topic :: System :: Distributed Computing', ], test_suite='tests', ) cloudpickle-0.5.2/PKG-INFO0000664000175000017620000001100213204761670016250 0ustar ogriselogrisel00000000000000Metadata-Version: 1.1 Name: cloudpickle Version: 0.5.2 Summary: Extended pickling support for Python objects Home-page: https://github.com/cloudpipe/cloudpickle Author: Cloudpipe Author-email: cloudpipe@googlegroups.com License: LICENSE.txt Description-Content-Type: UNKNOWN Description: # cloudpickle [![Build Status](https://travis-ci.org/cloudpipe/cloudpickle.svg?branch=master )](https://travis-ci.org/cloudpipe/cloudpickle) [![codecov.io](https://codecov.io/github/cloudpipe/cloudpickle/coverage.svg?branch=master)](https://codecov.io/github/cloudpipe/cloudpickle?branch=master) `cloudpickle` makes it possible to serialize Python constructs not supported by the default `pickle` module from the Python standard library. `cloudpickle` is especially useful for cluster computing where Python expressions are shipped over the network to execute on remote hosts, possibly close to the data. Among other things, `cloudpickle` supports pickling for lambda expressions, functions and classes defined interactively in the `__main__` module. `cloudpickle` uses `pickle.HIGHEST_PROTOCOL` by default: it is meant to send objects between processes running the same version of Python. It is discouraged to use `cloudpickle` for long-term storage. Installation ------------ The latest release of `cloudpickle` is available from [pypi](https://pypi.python.org/pypi/cloudpickle): pip install cloudpickle Examples -------- Pickling a lambda expression: ```python >>> import cloudpickle >>> squared = lambda x: x ** 2 >>> pickled_lambda = cloudpickle.dumps(squared) >>> import pickle >>> new_squared = pickle.loads(pickled_lambda) >>> new_squared(2) 4 ``` Pickling a function interactively defined in a Python shell session (in the `__main__` module): ```python >>> CONSTANT = 42 >>> def my_function(data): ... return data + CONSTANT ... >>> pickled_function = cloudpickle.dumps(my_function) >>> pickle.loads(pickled_function)(43) 85 ``` Running the tests ----------------- - With `tox`, to test run the tests for all the supported versions of Python and PyPy: pip install tox tox or alternatively for a specific environment: tox -e py27 - With `py.test` to only run the tests for your current version of Python: pip install -r dev-requirements.txt PYTHONPATH='.:tests' py.test History ------- `cloudpickle` was initially developed by [picloud.com](http://web.archive.org/web/20140721022102/http://blog.picloud.com/2013/11/17/picloud-has-joined-dropbox/) and shipped as part of the client SDK. A copy of `cloudpickle.py` was included as part of PySpark, the Python interface to [Apache Spark](https://spark.apache.org/). Davies Liu, Josh Rosen, Thom Neale and other Apache Spark developers improved it significantly, most notably to add support for PyPy and Python 3. The aim of the `cloudpickle` project is to make that work available to a wider audience outside of the Spark ecosystem and to make it easier to improve it further notably with the help of a dedicated non-regression test suite. Platform: UNKNOWN Classifier: Development Status :: 4 - Beta Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: BSD License Classifier: Operating System :: POSIX Classifier: Operating System :: Microsoft :: Windows Classifier: Operating System :: MacOS :: MacOS X Classifier: Programming Language :: Python :: 2.7 Classifier: Programming Language :: Python :: 3.4 Classifier: Programming Language :: Python :: 3.5 Classifier: Programming Language :: Python :: 3.6 Classifier: Programming Language :: Python :: Implementation :: CPython Classifier: Programming Language :: Python :: Implementation :: PyPy Classifier: Topic :: Software Development :: Libraries :: Python Modules Classifier: Topic :: Scientific/Engineering Classifier: Topic :: System :: Distributed Computing cloudpickle-0.5.2/cloudpickle/0000775000175000017620000000000013204761670017457 5ustar ogriselogrisel00000000000000cloudpickle-0.5.2/cloudpickle/cloudpickle.py0000664000175000017620000010741513204761500022327 0ustar ogriselogrisel00000000000000""" This class is defined to override standard pickle functionality The goals of it follow: -Serialize lambdas and nested functions to compiled byte code -Deal with main module correctly -Deal with other non-serializable objects It does not include an unpickler, as standard python unpickling suffices. This module was extracted from the `cloud` package, developed by `PiCloud, Inc. `_. Copyright (c) 2012, Regents of the University of California. Copyright (c) 2009 `PiCloud, Inc. `_. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of the University of California, Berkeley nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """ from __future__ import print_function import dis from functools import partial import imp import io import itertools import logging import opcode import operator import pickle import struct import sys import traceback import types import weakref # cloudpickle is meant for inter process communication: we expect all # communicating processes to run the same Python version hence we favor # communication speed over compatibility: DEFAULT_PROTOCOL = pickle.HIGHEST_PROTOCOL if sys.version < '3': from pickle import Pickler try: from cStringIO import StringIO except ImportError: from StringIO import StringIO PY3 = False else: types.ClassType = type from pickle import _Pickler as Pickler from io import BytesIO as StringIO PY3 = True def _make_cell_set_template_code(): """Get the Python compiler to emit LOAD_FAST(arg); STORE_DEREF Notes ----- In Python 3, we could use an easier function: .. code-block:: python def f(): cell = None def _stub(value): nonlocal cell cell = value return _stub _cell_set_template_code = f() This function is _only_ a LOAD_FAST(arg); STORE_DEREF, but that is invalid syntax on Python 2. If we use this function we also don't need to do the weird freevars/cellvars swap below """ def inner(value): lambda: cell # make ``cell`` a closure so that we get a STORE_DEREF cell = value co = inner.__code__ # NOTE: we are marking the cell variable as a free variable intentionally # so that we simulate an inner function instead of the outer function. This # is what gives us the ``nonlocal`` behavior in a Python 2 compatible way. if not PY3: return types.CodeType( co.co_argcount, co.co_nlocals, co.co_stacksize, co.co_flags, co.co_code, co.co_consts, co.co_names, co.co_varnames, co.co_filename, co.co_name, co.co_firstlineno, co.co_lnotab, co.co_cellvars, # this is the trickery (), ) else: return types.CodeType( co.co_argcount, co.co_kwonlyargcount, co.co_nlocals, co.co_stacksize, co.co_flags, co.co_code, co.co_consts, co.co_names, co.co_varnames, co.co_filename, co.co_name, co.co_firstlineno, co.co_lnotab, co.co_cellvars, # this is the trickery (), ) _cell_set_template_code = _make_cell_set_template_code() def cell_set(cell, value): """Set the value of a closure cell. """ return types.FunctionType( _cell_set_template_code, {}, '_cell_set_inner', (), (cell,), )(value) #relevant opcodes STORE_GLOBAL = opcode.opmap['STORE_GLOBAL'] DELETE_GLOBAL = opcode.opmap['DELETE_GLOBAL'] LOAD_GLOBAL = opcode.opmap['LOAD_GLOBAL'] GLOBAL_OPS = (STORE_GLOBAL, DELETE_GLOBAL, LOAD_GLOBAL) HAVE_ARGUMENT = dis.HAVE_ARGUMENT EXTENDED_ARG = dis.EXTENDED_ARG def islambda(func): return getattr(func,'__name__') == '' _BUILTIN_TYPE_NAMES = {} for k, v in types.__dict__.items(): if type(v) is type: _BUILTIN_TYPE_NAMES[v] = k def _builtin_type(name): return getattr(types, name) def _make__new__factory(type_): def _factory(): return type_.__new__ return _factory # NOTE: These need to be module globals so that they're pickleable as globals. _get_dict_new = _make__new__factory(dict) _get_frozenset_new = _make__new__factory(frozenset) _get_list_new = _make__new__factory(list) _get_set_new = _make__new__factory(set) _get_tuple_new = _make__new__factory(tuple) _get_object_new = _make__new__factory(object) # Pre-defined set of builtin_function_or_method instances that can be # serialized. _BUILTIN_TYPE_CONSTRUCTORS = { dict.__new__: _get_dict_new, frozenset.__new__: _get_frozenset_new, set.__new__: _get_set_new, list.__new__: _get_list_new, tuple.__new__: _get_tuple_new, object.__new__: _get_object_new, } if sys.version_info < (3, 4): def _walk_global_ops(code): """ Yield (opcode, argument number) tuples for all global-referencing instructions in *code*. """ code = getattr(code, 'co_code', b'') if not PY3: code = map(ord, code) n = len(code) i = 0 extended_arg = 0 while i < n: op = code[i] i += 1 if op >= HAVE_ARGUMENT: oparg = code[i] + code[i + 1] * 256 + extended_arg extended_arg = 0 i += 2 if op == EXTENDED_ARG: extended_arg = oparg * 65536 if op in GLOBAL_OPS: yield op, oparg else: def _walk_global_ops(code): """ Yield (opcode, argument number) tuples for all global-referencing instructions in *code*. """ for instr in dis.get_instructions(code): op = instr.opcode if op in GLOBAL_OPS: yield op, instr.arg class CloudPickler(Pickler): dispatch = Pickler.dispatch.copy() def __init__(self, file, protocol=None): if protocol is None: protocol = DEFAULT_PROTOCOL Pickler.__init__(self, file, protocol=protocol) # set of modules to unpickle self.modules = set() # map ids to dictionary. used to ensure that functions can share global env self.globals_ref = {} def dump(self, obj): self.inject_addons() try: return Pickler.dump(self, obj) except RuntimeError as e: if 'recursion' in e.args[0]: msg = """Could not pickle object as excessively deep recursion required.""" raise pickle.PicklingError(msg) def save_memoryview(self, obj): self.save(obj.tobytes()) dispatch[memoryview] = save_memoryview if not PY3: def save_buffer(self, obj): self.save(str(obj)) dispatch[buffer] = save_buffer def save_unsupported(self, obj): raise pickle.PicklingError("Cannot pickle objects of type %s" % type(obj)) dispatch[types.GeneratorType] = save_unsupported # itertools objects do not pickle! for v in itertools.__dict__.values(): if type(v) is type: dispatch[v] = save_unsupported def save_module(self, obj): """ Save a module as an import """ mod_name = obj.__name__ # If module is successfully found then it is not a dynamically created module if hasattr(obj, '__file__'): is_dynamic = False else: try: _find_module(mod_name) is_dynamic = False except ImportError: is_dynamic = True self.modules.add(obj) if is_dynamic: self.save_reduce(dynamic_subimport, (obj.__name__, vars(obj)), obj=obj) else: self.save_reduce(subimport, (obj.__name__,), obj=obj) dispatch[types.ModuleType] = save_module def save_codeobject(self, obj): """ Save a code object """ if PY3: args = ( obj.co_argcount, obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, obj.co_varnames, obj.co_filename, obj.co_name, obj.co_firstlineno, obj.co_lnotab, obj.co_freevars, obj.co_cellvars ) else: args = ( obj.co_argcount, obj.co_nlocals, obj.co_stacksize, obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, obj.co_varnames, obj.co_filename, obj.co_name, obj.co_firstlineno, obj.co_lnotab, obj.co_freevars, obj.co_cellvars ) self.save_reduce(types.CodeType, args, obj=obj) dispatch[types.CodeType] = save_codeobject def save_function(self, obj, name=None): """ Registered with the dispatch to handle all function types. Determines what kind of function obj is (e.g. lambda, defined at interactive prompt, etc) and handles the pickling appropriately. """ if obj in _BUILTIN_TYPE_CONSTRUCTORS: # We keep a special-cased cache of built-in type constructors at # global scope, because these functions are structured very # differently in different python versions and implementations (for # example, they're instances of types.BuiltinFunctionType in # CPython, but they're ordinary types.FunctionType instances in # PyPy). # # If the function we've received is in that cache, we just # serialize it as a lookup into the cache. return self.save_reduce(_BUILTIN_TYPE_CONSTRUCTORS[obj], (), obj=obj) write = self.write if name is None: name = obj.__name__ try: # whichmodule() could fail, see # https://bitbucket.org/gutworth/six/issues/63/importing-six-breaks-pickling modname = pickle.whichmodule(obj, name) except Exception: modname = None # print('which gives %s %s %s' % (modname, obj, name)) try: themodule = sys.modules[modname] except KeyError: # eval'd items such as namedtuple give invalid items for their function __module__ modname = '__main__' if modname == '__main__': themodule = None try: lookedup_by_name = getattr(themodule, name, None) except Exception: lookedup_by_name = None if themodule: self.modules.add(themodule) if lookedup_by_name is obj: return self.save_global(obj, name) # a builtin_function_or_method which comes in as an attribute of some # object (e.g., itertools.chain.from_iterable) will end # up with modname "__main__" and so end up here. But these functions # have no __code__ attribute in CPython, so the handling for # user-defined functions below will fail. # So we pickle them here using save_reduce; have to do it differently # for different python versions. if not hasattr(obj, '__code__'): if PY3: rv = obj.__reduce_ex__(self.proto) else: if hasattr(obj, '__self__'): rv = (getattr, (obj.__self__, name)) else: raise pickle.PicklingError("Can't pickle %r" % obj) return self.save_reduce(obj=obj, *rv) # if func is lambda, def'ed at prompt, is in main, or is nested, then # we'll pickle the actual function object rather than simply saving a # reference (as is done in default pickler), via save_function_tuple. if (islambda(obj) or getattr(obj.__code__, 'co_filename', None) == '' or themodule is None): self.save_function_tuple(obj) return else: # func is nested if lookedup_by_name is None or lookedup_by_name is not obj: self.save_function_tuple(obj) return if obj.__dict__: # essentially save_reduce, but workaround needed to avoid recursion self.save(_restore_attr) write(pickle.MARK + pickle.GLOBAL + modname + '\n' + name + '\n') self.memoize(obj) self.save(obj.__dict__) write(pickle.TUPLE + pickle.REDUCE) else: write(pickle.GLOBAL + modname + '\n' + name + '\n') self.memoize(obj) dispatch[types.FunctionType] = save_function def _save_subimports(self, code, top_level_dependencies): """ Ensure de-pickler imports any package child-modules that are needed by the function """ # check if any known dependency is an imported package for x in top_level_dependencies: if isinstance(x, types.ModuleType) and hasattr(x, '__package__') and x.__package__: # check if the package has any currently loaded sub-imports prefix = x.__name__ + '.' for name, module in sys.modules.items(): # Older versions of pytest will add a "None" module to sys.modules. if name is not None and name.startswith(prefix): # check whether the function can address the sub-module tokens = set(name[len(prefix):].split('.')) if not tokens - set(code.co_names): # ensure unpickler executes this import self.save(module) # then discards the reference to it self.write(pickle.POP) def save_dynamic_class(self, obj): """ Save a class that can't be stored as module global. This method is used to serialize classes that are defined inside functions, or that otherwise can't be serialized as attribute lookups from global modules. """ clsdict = dict(obj.__dict__) # copy dict proxy to a dict clsdict.pop('__weakref__', None) # On PyPy, __doc__ is a readonly attribute, so we need to include it in # the initial skeleton class. This is safe because we know that the # doc can't participate in a cycle with the original class. type_kwargs = {'__doc__': clsdict.pop('__doc__', None)} # If type overrides __dict__ as a property, include it in the type kwargs. # In Python 2, we can't set this attribute after construction. __dict__ = clsdict.pop('__dict__', None) if isinstance(__dict__, property): type_kwargs['__dict__'] = __dict__ save = self.save write = self.write # We write pickle instructions explicitly here to handle the # possibility that the type object participates in a cycle with its own # __dict__. We first write an empty "skeleton" version of the class and # memoize it before writing the class' __dict__ itself. We then write # instructions to "rehydrate" the skeleton class by restoring the # attributes from the __dict__. # # A type can appear in a cycle with its __dict__ if an instance of the # type appears in the type's __dict__ (which happens for the stdlib # Enum class), or if the type defines methods that close over the name # of the type, (which is common for Python 2-style super() calls). # Push the rehydration function. save(_rehydrate_skeleton_class) # Mark the start of the args tuple for the rehydration function. write(pickle.MARK) # Create and memoize an skeleton class with obj's name and bases. tp = type(obj) self.save_reduce(tp, (obj.__name__, obj.__bases__, type_kwargs), obj=obj) # Now save the rest of obj's __dict__. Any references to obj # encountered while saving will point to the skeleton class. save(clsdict) # Write a tuple of (skeleton_class, clsdict). write(pickle.TUPLE) # Call _rehydrate_skeleton_class(skeleton_class, clsdict) write(pickle.REDUCE) def save_function_tuple(self, func): """ Pickles an actual func object. A func comprises: code, globals, defaults, closure, and dict. We extract and save these, injecting reducing functions at certain points to recreate the func object. Keep in mind that some of these pieces can contain a ref to the func itself. Thus, a naive save on these pieces could trigger an infinite loop of save's. To get around that, we first create a skeleton func object using just the code (this is safe, since this won't contain a ref to the func), and memoize it as soon as it's created. The other stuff can then be filled in later. """ if is_tornado_coroutine(func): self.save_reduce(_rebuild_tornado_coroutine, (func.__wrapped__,), obj=func) return save = self.save write = self.write code, f_globals, defaults, closure_values, dct, base_globals = self.extract_func_data(func) save(_fill_function) # skeleton function updater write(pickle.MARK) # beginning of tuple that _fill_function expects self._save_subimports( code, itertools.chain(f_globals.values(), closure_values or ()), ) # create a skeleton function object and memoize it save(_make_skel_func) save(( code, len(closure_values) if closure_values is not None else -1, base_globals, )) write(pickle.REDUCE) self.memoize(func) # save the rest of the func data needed by _fill_function state = { 'globals': f_globals, 'defaults': defaults, 'dict': dct, 'module': func.__module__, 'closure_values': closure_values, } if hasattr(func, '__qualname__'): state['qualname'] = func.__qualname__ save(state) write(pickle.TUPLE) write(pickle.REDUCE) # applies _fill_function on the tuple _extract_code_globals_cache = ( weakref.WeakKeyDictionary() if not hasattr(sys, "pypy_version_info") else {}) @classmethod def extract_code_globals(cls, co): """ Find all globals names read or written to by codeblock co """ out_names = cls._extract_code_globals_cache.get(co) if out_names is None: try: names = co.co_names except AttributeError: # PyPy "builtin-code" object out_names = set() else: out_names = set(names[oparg] for op, oparg in _walk_global_ops(co)) # see if nested function have any global refs if co.co_consts: for const in co.co_consts: if type(const) is types.CodeType: out_names |= cls.extract_code_globals(const) cls._extract_code_globals_cache[co] = out_names return out_names def extract_func_data(self, func): """ Turn the function into a tuple of data necessary to recreate it: code, globals, defaults, closure_values, dict """ code = func.__code__ # extract all global ref's func_global_refs = self.extract_code_globals(code) # process all variables referenced by global environment f_globals = {} for var in func_global_refs: if var in func.__globals__: f_globals[var] = func.__globals__[var] # defaults requires no processing defaults = func.__defaults__ # process closure closure = ( list(map(_get_cell_contents, func.__closure__)) if func.__closure__ is not None else None ) # save the dict dct = func.__dict__ base_globals = self.globals_ref.get(id(func.__globals__), {}) self.globals_ref[id(func.__globals__)] = base_globals return (code, f_globals, defaults, closure, dct, base_globals) def save_builtin_function(self, obj): if obj.__module__ == "__builtin__": return self.save_global(obj) return self.save_function(obj) dispatch[types.BuiltinFunctionType] = save_builtin_function def save_global(self, obj, name=None, pack=struct.pack): """ Save a "global". The name of this method is somewhat misleading: all types get dispatched here. """ if obj.__module__ == "__main__": return self.save_dynamic_class(obj) try: return Pickler.save_global(self, obj, name=name) except Exception: if obj.__module__ == "__builtin__" or obj.__module__ == "builtins": if obj in _BUILTIN_TYPE_NAMES: return self.save_reduce( _builtin_type, (_BUILTIN_TYPE_NAMES[obj],), obj=obj) typ = type(obj) if typ is not obj and isinstance(obj, (type, types.ClassType)): return self.save_dynamic_class(obj) raise dispatch[type] = save_global dispatch[types.ClassType] = save_global def save_instancemethod(self, obj): # Memoization rarely is ever useful due to python bounding if obj.__self__ is None: self.save_reduce(getattr, (obj.im_class, obj.__name__)) else: if PY3: self.save_reduce(types.MethodType, (obj.__func__, obj.__self__), obj=obj) else: self.save_reduce(types.MethodType, (obj.__func__, obj.__self__, obj.__self__.__class__), obj=obj) dispatch[types.MethodType] = save_instancemethod def save_inst(self, obj): """Inner logic to save instance. Based off pickle.save_inst""" cls = obj.__class__ # Try the dispatch table (pickle module doesn't do it) f = self.dispatch.get(cls) if f: f(self, obj) # Call unbound method with explicit self return memo = self.memo write = self.write save = self.save if hasattr(obj, '__getinitargs__'): args = obj.__getinitargs__() len(args) # XXX Assert it's a sequence pickle._keep_alive(args, memo) else: args = () write(pickle.MARK) if self.bin: save(cls) for arg in args: save(arg) write(pickle.OBJ) else: for arg in args: save(arg) write(pickle.INST + cls.__module__ + '\n' + cls.__name__ + '\n') self.memoize(obj) try: getstate = obj.__getstate__ except AttributeError: stuff = obj.__dict__ else: stuff = getstate() pickle._keep_alive(stuff, memo) save(stuff) write(pickle.BUILD) if not PY3: dispatch[types.InstanceType] = save_inst def save_property(self, obj): # properties not correctly saved in python self.save_reduce(property, (obj.fget, obj.fset, obj.fdel, obj.__doc__), obj=obj) dispatch[property] = save_property def save_classmethod(self, obj): orig_func = obj.__func__ self.save_reduce(type(obj), (orig_func,), obj=obj) dispatch[classmethod] = save_classmethod dispatch[staticmethod] = save_classmethod def save_itemgetter(self, obj): """itemgetter serializer (needed for namedtuple support)""" class Dummy: def __getitem__(self, item): return item items = obj(Dummy()) if not isinstance(items, tuple): items = (items, ) return self.save_reduce(operator.itemgetter, items) if type(operator.itemgetter) is type: dispatch[operator.itemgetter] = save_itemgetter def save_attrgetter(self, obj): """attrgetter serializer""" class Dummy(object): def __init__(self, attrs, index=None): self.attrs = attrs self.index = index def __getattribute__(self, item): attrs = object.__getattribute__(self, "attrs") index = object.__getattribute__(self, "index") if index is None: index = len(attrs) attrs.append(item) else: attrs[index] = ".".join([attrs[index], item]) return type(self)(attrs, index) attrs = [] obj(Dummy(attrs)) return self.save_reduce(operator.attrgetter, tuple(attrs)) if type(operator.attrgetter) is type: dispatch[operator.attrgetter] = save_attrgetter def save_file(self, obj): """Save a file""" try: import StringIO as pystringIO #we can't use cStringIO as it lacks the name attribute except ImportError: import io as pystringIO if not hasattr(obj, 'name') or not hasattr(obj, 'mode'): raise pickle.PicklingError("Cannot pickle files that do not map to an actual file") if obj is sys.stdout: return self.save_reduce(getattr, (sys,'stdout'), obj=obj) if obj is sys.stderr: return self.save_reduce(getattr, (sys,'stderr'), obj=obj) if obj is sys.stdin: raise pickle.PicklingError("Cannot pickle standard input") if obj.closed: raise pickle.PicklingError("Cannot pickle closed files") if hasattr(obj, 'isatty') and obj.isatty(): raise pickle.PicklingError("Cannot pickle files that map to tty objects") if 'r' not in obj.mode and '+' not in obj.mode: raise pickle.PicklingError("Cannot pickle files that are not opened for reading: %s" % obj.mode) name = obj.name retval = pystringIO.StringIO() try: # Read the whole file curloc = obj.tell() obj.seek(0) contents = obj.read() obj.seek(curloc) except IOError: raise pickle.PicklingError("Cannot pickle file %s as it cannot be read" % name) retval.write(contents) retval.seek(curloc) retval.name = name self.save(retval) self.memoize(obj) def save_ellipsis(self, obj): self.save_reduce(_gen_ellipsis, ()) def save_not_implemented(self, obj): self.save_reduce(_gen_not_implemented, ()) if PY3: dispatch[io.TextIOWrapper] = save_file else: dispatch[file] = save_file dispatch[type(Ellipsis)] = save_ellipsis dispatch[type(NotImplemented)] = save_not_implemented def save_weakset(self, obj): self.save_reduce(weakref.WeakSet, (list(obj),)) dispatch[weakref.WeakSet] = save_weakset def save_logger(self, obj): self.save_reduce(logging.getLogger, (obj.name,), obj=obj) dispatch[logging.Logger] = save_logger """Special functions for Add-on libraries""" def inject_addons(self): """Plug in system. Register additional pickling functions if modules already loaded""" pass # Tornado support def is_tornado_coroutine(func): """ Return whether *func* is a Tornado coroutine function. Running coroutines are not supported. """ if 'tornado.gen' not in sys.modules: return False gen = sys.modules['tornado.gen'] if not hasattr(gen, "is_coroutine_function"): # Tornado version is too old return False return gen.is_coroutine_function(func) def _rebuild_tornado_coroutine(func): from tornado import gen return gen.coroutine(func) # Shorthands for legacy support def dump(obj, file, protocol=None): """Serialize obj as bytes streamed into file protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to pickle.HIGHEST_PROTOCOL. This setting favors maximum communication speed between processes running the same Python version. Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure compatibility with older versions of Python. """ CloudPickler(file, protocol=protocol).dump(obj) def dumps(obj, protocol=None): """Serialize obj as a string of bytes allocated in memory protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to pickle.HIGHEST_PROTOCOL. This setting favors maximum communication speed between processes running the same Python version. Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure compatibility with older versions of Python. """ file = StringIO() try: cp = CloudPickler(file, protocol=protocol) cp.dump(obj) return file.getvalue() finally: file.close() # including pickles unloading functions in this namespace load = pickle.load loads = pickle.loads # hack for __import__ not working as desired def subimport(name): __import__(name) return sys.modules[name] def dynamic_subimport(name, vars): mod = imp.new_module(name) mod.__dict__.update(vars) sys.modules[name] = mod return mod # restores function attributes def _restore_attr(obj, attr): for key, val in attr.items(): setattr(obj, key, val) return obj def _get_module_builtins(): return pickle.__builtins__ def print_exec(stream): ei = sys.exc_info() traceback.print_exception(ei[0], ei[1], ei[2], None, stream) def _modules_to_main(modList): """Force every module in modList to be placed into main""" if not modList: return main = sys.modules['__main__'] for modname in modList: if type(modname) is str: try: mod = __import__(modname) except Exception as e: sys.stderr.write('warning: could not import %s\n. ' 'Your function may unexpectedly error due to this import failing;' 'A version mismatch is likely. Specific error was:\n' % modname) print_exec(sys.stderr) else: setattr(main, mod.__name__, mod) #object generators: def _genpartial(func, args, kwds): if not args: args = () if not kwds: kwds = {} return partial(func, *args, **kwds) def _gen_ellipsis(): return Ellipsis def _gen_not_implemented(): return NotImplemented def _get_cell_contents(cell): try: return cell.cell_contents except ValueError: # sentinel used by ``_fill_function`` which will leave the cell empty return _empty_cell_value def instance(cls): """Create a new instance of a class. Parameters ---------- cls : type The class to create an instance of. Returns ------- instance : cls A new instance of ``cls``. """ return cls() @instance class _empty_cell_value(object): """sentinel for empty closures """ @classmethod def __reduce__(cls): return cls.__name__ def _fill_function(*args): """Fills in the rest of function data into the skeleton function object The skeleton itself is create by _make_skel_func(). """ if len(args) == 2: func = args[0] state = args[1] elif len(args) == 5: # Backwards compat for cloudpickle v0.4.0, after which the `module` # argument was introduced func = args[0] keys = ['globals', 'defaults', 'dict', 'closure_values'] state = dict(zip(keys, args[1:])) elif len(args) == 6: # Backwards compat for cloudpickle v0.4.1, after which the function # state was passed as a dict to the _fill_function it-self. func = args[0] keys = ['globals', 'defaults', 'dict', 'module', 'closure_values'] state = dict(zip(keys, args[1:])) else: raise ValueError('Unexpected _fill_value arguments: %r' % (args,)) func.__globals__.update(state['globals']) func.__defaults__ = state['defaults'] func.__dict__ = state['dict'] if 'module' in state: func.__module__ = state['module'] if 'qualname' in state: func.__qualname__ = state['qualname'] cells = func.__closure__ if cells is not None: for cell, value in zip(cells, state['closure_values']): if value is not _empty_cell_value: cell_set(cell, value) return func def _make_empty_cell(): if False: # trick the compiler into creating an empty cell in our lambda cell = None raise AssertionError('this route should not be executed') return (lambda: cell).__closure__[0] def _make_skel_func(code, cell_count, base_globals=None): """ Creates a skeleton function object that contains just the provided code and the correct number of cells in func_closure. All other func attributes (e.g. func_globals) are empty. """ if base_globals is None: base_globals = {} base_globals['__builtins__'] = __builtins__ closure = ( tuple(_make_empty_cell() for _ in range(cell_count)) if cell_count >= 0 else None ) return types.FunctionType(code, base_globals, None, None, closure) def _rehydrate_skeleton_class(skeleton_class, class_dict): """Put attributes from `class_dict` back on `skeleton_class`. See CloudPickler.save_dynamic_class for more info. """ for attrname, attr in class_dict.items(): setattr(skeleton_class, attrname, attr) return skeleton_class def _find_module(mod_name): """ Iterate over each part instead of calling imp.find_module directly. This function is able to find submodules (e.g. sickit.tree) """ path = None for part in mod_name.split('.'): if path is not None: path = [path] file, path, description = imp.find_module(part, path) if file is not None: file.close() return path, description """Constructors for 3rd party libraries Note: These can never be renamed due to client compatibility issues""" def _getobject(modname, attribute): mod = __import__(modname, fromlist=[attribute]) return mod.__dict__[attribute] """ Use copy_reg to extend global pickle definitions """ if sys.version_info < (3, 4): method_descriptor = type(str.upper) def _reduce_method_descriptor(obj): return (getattr, (obj.__objclass__, obj.__name__)) try: import copy_reg as copyreg except ImportError: import copyreg copyreg.pickle(method_descriptor, _reduce_method_descriptor) cloudpickle-0.5.2/cloudpickle/__init__.py0000664000175000017620000000014513204761500021560 0ustar ogriselogrisel00000000000000from __future__ import absolute_import from cloudpickle.cloudpickle import * __version__ = '0.5.2' cloudpickle-0.5.2/README.md0000664000175000017620000000532213203305571016433 0ustar ogriselogrisel00000000000000# cloudpickle [![Build Status](https://travis-ci.org/cloudpipe/cloudpickle.svg?branch=master )](https://travis-ci.org/cloudpipe/cloudpickle) [![codecov.io](https://codecov.io/github/cloudpipe/cloudpickle/coverage.svg?branch=master)](https://codecov.io/github/cloudpipe/cloudpickle?branch=master) `cloudpickle` makes it possible to serialize Python constructs not supported by the default `pickle` module from the Python standard library. `cloudpickle` is especially useful for cluster computing where Python expressions are shipped over the network to execute on remote hosts, possibly close to the data. Among other things, `cloudpickle` supports pickling for lambda expressions, functions and classes defined interactively in the `__main__` module. `cloudpickle` uses `pickle.HIGHEST_PROTOCOL` by default: it is meant to send objects between processes running the same version of Python. It is discouraged to use `cloudpickle` for long-term storage. Installation ------------ The latest release of `cloudpickle` is available from [pypi](https://pypi.python.org/pypi/cloudpickle): pip install cloudpickle Examples -------- Pickling a lambda expression: ```python >>> import cloudpickle >>> squared = lambda x: x ** 2 >>> pickled_lambda = cloudpickle.dumps(squared) >>> import pickle >>> new_squared = pickle.loads(pickled_lambda) >>> new_squared(2) 4 ``` Pickling a function interactively defined in a Python shell session (in the `__main__` module): ```python >>> CONSTANT = 42 >>> def my_function(data): ... return data + CONSTANT ... >>> pickled_function = cloudpickle.dumps(my_function) >>> pickle.loads(pickled_function)(43) 85 ``` Running the tests ----------------- - With `tox`, to test run the tests for all the supported versions of Python and PyPy: pip install tox tox or alternatively for a specific environment: tox -e py27 - With `py.test` to only run the tests for your current version of Python: pip install -r dev-requirements.txt PYTHONPATH='.:tests' py.test History ------- `cloudpickle` was initially developed by [picloud.com](http://web.archive.org/web/20140721022102/http://blog.picloud.com/2013/11/17/picloud-has-joined-dropbox/) and shipped as part of the client SDK. A copy of `cloudpickle.py` was included as part of PySpark, the Python interface to [Apache Spark](https://spark.apache.org/). Davies Liu, Josh Rosen, Thom Neale and other Apache Spark developers improved it significantly, most notably to add support for PyPy and Python 3. The aim of the `cloudpickle` project is to make that work available to a wider audience outside of the Spark ecosystem and to make it easier to improve it further notably with the help of a dedicated non-regression test suite. cloudpickle-0.5.2/setup.cfg0000664000175000017620000000007513204761670017004 0ustar ogriselogrisel00000000000000[wheel] universal = 1 [egg_info] tag_build = tag_date = 0 cloudpickle-0.5.2/cloudpickle.egg-info/0000775000175000017620000000000013204761670021151 5ustar ogriselogrisel00000000000000cloudpickle-0.5.2/cloudpickle.egg-info/PKG-INFO0000664000175000017620000001100213204761670022240 0ustar ogriselogrisel00000000000000Metadata-Version: 1.1 Name: cloudpickle Version: 0.5.2 Summary: Extended pickling support for Python objects Home-page: https://github.com/cloudpipe/cloudpickle Author: Cloudpipe Author-email: cloudpipe@googlegroups.com License: LICENSE.txt Description-Content-Type: UNKNOWN Description: # cloudpickle [![Build Status](https://travis-ci.org/cloudpipe/cloudpickle.svg?branch=master )](https://travis-ci.org/cloudpipe/cloudpickle) [![codecov.io](https://codecov.io/github/cloudpipe/cloudpickle/coverage.svg?branch=master)](https://codecov.io/github/cloudpipe/cloudpickle?branch=master) `cloudpickle` makes it possible to serialize Python constructs not supported by the default `pickle` module from the Python standard library. `cloudpickle` is especially useful for cluster computing where Python expressions are shipped over the network to execute on remote hosts, possibly close to the data. Among other things, `cloudpickle` supports pickling for lambda expressions, functions and classes defined interactively in the `__main__` module. `cloudpickle` uses `pickle.HIGHEST_PROTOCOL` by default: it is meant to send objects between processes running the same version of Python. It is discouraged to use `cloudpickle` for long-term storage. Installation ------------ The latest release of `cloudpickle` is available from [pypi](https://pypi.python.org/pypi/cloudpickle): pip install cloudpickle Examples -------- Pickling a lambda expression: ```python >>> import cloudpickle >>> squared = lambda x: x ** 2 >>> pickled_lambda = cloudpickle.dumps(squared) >>> import pickle >>> new_squared = pickle.loads(pickled_lambda) >>> new_squared(2) 4 ``` Pickling a function interactively defined in a Python shell session (in the `__main__` module): ```python >>> CONSTANT = 42 >>> def my_function(data): ... return data + CONSTANT ... >>> pickled_function = cloudpickle.dumps(my_function) >>> pickle.loads(pickled_function)(43) 85 ``` Running the tests ----------------- - With `tox`, to test run the tests for all the supported versions of Python and PyPy: pip install tox tox or alternatively for a specific environment: tox -e py27 - With `py.test` to only run the tests for your current version of Python: pip install -r dev-requirements.txt PYTHONPATH='.:tests' py.test History ------- `cloudpickle` was initially developed by [picloud.com](http://web.archive.org/web/20140721022102/http://blog.picloud.com/2013/11/17/picloud-has-joined-dropbox/) and shipped as part of the client SDK. A copy of `cloudpickle.py` was included as part of PySpark, the Python interface to [Apache Spark](https://spark.apache.org/). Davies Liu, Josh Rosen, Thom Neale and other Apache Spark developers improved it significantly, most notably to add support for PyPy and Python 3. The aim of the `cloudpickle` project is to make that work available to a wider audience outside of the Spark ecosystem and to make it easier to improve it further notably with the help of a dedicated non-regression test suite. Platform: UNKNOWN Classifier: Development Status :: 4 - Beta Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: BSD License Classifier: Operating System :: POSIX Classifier: Operating System :: Microsoft :: Windows Classifier: Operating System :: MacOS :: MacOS X Classifier: Programming Language :: Python :: 2.7 Classifier: Programming Language :: Python :: 3.4 Classifier: Programming Language :: Python :: 3.5 Classifier: Programming Language :: Python :: 3.6 Classifier: Programming Language :: Python :: Implementation :: CPython Classifier: Programming Language :: Python :: Implementation :: PyPy Classifier: Topic :: Software Development :: Libraries :: Python Modules Classifier: Topic :: Scientific/Engineering Classifier: Topic :: System :: Distributed Computing cloudpickle-0.5.2/cloudpickle.egg-info/SOURCES.txt0000664000175000017620000000051513204761670023036 0ustar ogriselogrisel00000000000000LICENSE MANIFEST.in README.md setup.cfg setup.py cloudpickle/__init__.py cloudpickle/cloudpickle.py cloudpickle.egg-info/PKG-INFO cloudpickle.egg-info/SOURCES.txt cloudpickle.egg-info/dependency_links.txt cloudpickle.egg-info/top_level.txt tests/__init__.py tests/cloudpickle_file_test.py tests/cloudpickle_test.py tests/testutils.pycloudpickle-0.5.2/cloudpickle.egg-info/dependency_links.txt0000664000175000017620000000000113204761670025217 0ustar ogriselogrisel00000000000000 cloudpickle-0.5.2/cloudpickle.egg-info/top_level.txt0000664000175000017620000000001413204761670023676 0ustar ogriselogrisel00000000000000cloudpickle