././@PaxHeader0000000000000000000000000000003300000000000010211 xustar0027 mtime=1709281468.330042 Cerealizer-0.8.4/0000755002342100234210000000000014570310274012245 5ustar00jibajiba././@PaxHeader0000000000000000000000000000003300000000000010211 xustar0027 mtime=1709281468.330042 Cerealizer-0.8.4/Cerealizer.egg-info/0000755002342100234210000000000014570310274016024 5ustar00jibajiba././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1709281468.0 Cerealizer-0.8.4/Cerealizer.egg-info/PKG-INFO0000644002342100234210000000161214570310274017121 0ustar00jibajibaMetadata-Version: 2.1 Name: Cerealizer Version: 0.8.4 Summary: A secure pickle-like module Home-page: http://www.lesfleursdunormal.fr/static/informatique/cerealizer/index_en.html Author: Lamy Jean-Baptiste (Jiba) Author-email: jibalamy@free.fr License: Python licence Classifier: Development Status :: 5 - Production/Stable Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: Python Software Foundation License Classifier: Programming Language :: Python :: 2 Classifier: Programming Language :: Python :: 3 Classifier: Topic :: Security Classifier: Topic :: Software Development :: Libraries :: Python Modules License-File: LICENSE A secure pickle-like module. It support basic types (int, string, unicode, tuple, list, dict, set,...), old and new-style classes (you need to register the class for security), object cycles, and it can be extended to support C-defined type. ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1709281468.0 Cerealizer-0.8.4/Cerealizer.egg-info/SOURCES.txt0000644002342100234210000000050414570310274017707 0ustar00jibajibaLICENSE MANIFEST MANIFEST.in README.rst __init__.py __init__.py2 __init__.py3 datetime_handler.py datetime_handler.py2 datetime_handler.py3 setup.cfg setup.py Cerealizer.egg-info/PKG-INFO Cerealizer.egg-info/SOURCES.txt Cerealizer.egg-info/dependency_links.txt Cerealizer.egg-info/top_level.txt test/regtest.py test/test1.py././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1709281468.0 Cerealizer-0.8.4/Cerealizer.egg-info/dependency_links.txt0000644002342100234210000000000114570310274022072 0ustar00jibajiba ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1709281468.0 Cerealizer-0.8.4/Cerealizer.egg-info/top_level.txt0000644002342100234210000000001314570310274020550 0ustar00jibajibacerealizer ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1391963324.0 Cerealizer-0.8.4/LICENSE0000644002342100234210000000005512275726274013265 0ustar00jibajibaPYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1408101012.0 Cerealizer-0.8.4/MANIFEST0000644002342100234210000000030012373365224013373 0ustar00jibajiba# file GENERATED by distutils, do NOT edit __init__.py __init__.py2 __init__.py3 datetime_handler.py datetime_handler.py2 datetime_handler.py3 setup.cfg setup.py test/regtest.py test/test1.py ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1590663630.0 Cerealizer-0.8.4/MANIFEST.in0000644002342100234210000000013713663714716014017 0ustar00jibajiba include test/*.py include *.py2 include *.py3 exclude .git exclude .git/* exclude .gitignore ././@PaxHeader0000000000000000000000000000003300000000000010211 xustar0027 mtime=1709281468.330042 Cerealizer-0.8.4/PKG-INFO0000644002342100234210000000161214570310274013342 0ustar00jibajibaMetadata-Version: 2.1 Name: Cerealizer Version: 0.8.4 Summary: A secure pickle-like module Home-page: http://www.lesfleursdunormal.fr/static/informatique/cerealizer/index_en.html Author: Lamy Jean-Baptiste (Jiba) Author-email: jibalamy@free.fr License: Python licence Classifier: Development Status :: 5 - Production/Stable Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: Python Software Foundation License Classifier: Programming Language :: Python :: 2 Classifier: Programming Language :: Python :: 3 Classifier: Topic :: Security Classifier: Topic :: Software Development :: Libraries :: Python Modules License-File: LICENSE A secure pickle-like module. It support basic types (int, string, unicode, tuple, list, dict, set,...), old and new-style classes (you need to register the class for security), object cycles, and it can be extended to support C-defined type. ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1406110299.0 Cerealizer-0.8.4/README.rst0000644002342100234210000000156512363705133013743 0ustar00jibajibaCerealizer %%%%%%%%%% Cerealizer is a secure pickle-like module for Python 2 and 3. It support basic types (int, string, unicode, tuple, list, dict, set,...), old and new-style classes (you need to register the class for security), object cycles, and it can be extended to support C-defined type. Cerealizer is available under the Python licence. Requirements ============ - Python >= 2.4 (http://www.python.org) Installation ============ Do in a console:: python ./setup.py build and then as root:: python ./setup.py install Documentation ============= Use PyDoc. Contact and links ================= Jiba -- Jean-Baptiste LAMY -- Cerealizer website : http://www.lesfleursdunormal.fr/static/informatique/cerealizer/index_en.html Cerealizer on BitBucket (development repository): https://bitbucket.org/jibalamy/cerealizer ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1709280541.0 Cerealizer-0.8.4/__init__.py0000644002342100234210000000047614570306435014371 0ustar00jibajiba# -*- coding: utf-8 -*- # Cerealizer # Copyright (C) 2005-2012 Jean-Baptiste LAMY # # This program is free software. # It is available under the Python licence. import sys if sys.version[0] == "2": execfile("%s.py2" % __file__.rsplit(".", 1)[0]) else: exec(open("%s.py3" % __file__.rsplit(".", 1)[0]).read()) ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1709281328.0 Cerealizer-0.8.4/__init__.py20000644002342100234210000005632314570310060014442 0ustar00jibajiba# -*- coding: utf-8 -*- # Cerealizer # Copyright (C) 2005-2008 Jean-Baptiste LAMY # Copyright (C) 2008 Peter Eckersley # # This program is free software. # It is available under the Python licence. """Cerealizer -- A secure Pickle-like module The interface of the Cerealizer module is similar to Pickle, and it supports __getstate__, __setstate__, __getinitargs__ and __getnewargs__. Cerealizer supports int, long, float, bool, complex, string, unicode, tuple, list, set, frozenset, dict, old-style and new-style class instances. C-defined types are supported but saving the C-side data may require to write e.g. a specific Handler or a __getstate__ and __setstate__ pair. Objects with __slots__ are supported too. You have to register the class you want to serialize, by calling cerealizer.register(YourClass). Cerealizer can be considered as secure AS LONG AS the following methods of 'YourClass' are secure: - __new__ - __del__ - __getstate__ - __setstate__ - __init__ (ONLY if __getinitargs__ is used for the class) These methods are the only one Cerealizer may call. For a higher security, Cerealizer maintains its own reference to these method (exepted __del__ that can only be called indirectly). Cerealizer doesn't aim at producing Human-readable files. About performances, Cerealizer is really fast and, when powered by Psyco, it may even beat cPickle! Although Cerealizer is implemented in less than 500 lines of pure-Python code (which is another reason for Cerealizer to be secure, since less code means less bugs :-). Compared to Pickle (cPickle): - Cerealizer is secure - Cerealizer achieves similar performances (using Psyco) - Cerealizer requires you to declare the serializable classes Compared to Jelly (from TwistedMatrix): - Cerealizer is faster - Cerealizer does a better job with object cycles, C-defined types and tuples (*) - Cerealizer files are not Human readable (*) Jelly handles them, but tuples and objects in a cycle are first created as _Tuple or _Dereference objects; this works for Python classes, but not with C-defined types which expects a precise type (e.g. tuple and not _Tuple). IMPLEMENTATION DETAILS GENERAL FILE FORMAT STRUCTURE Cerealizer format is simple but quite surprising. It uses a "double flat list" format. It looks like that : \\n \\n \\n \\n [...] [...] As you can see, the information for a given object is splitted in two parts, the first one for object's class, and the second one for the object's data. To avoid problems, the order of the objects is the following: Objects are put after basic types (list,...), since object's __setstate__ might rely on a list, and thus the list must be fully loaded BEFORE calling the object's __setstate__. DATA ( above) The part saves the data of object #n. It may contains reference to other data (see below, in Cerealizer references include reference to other objects but also raw data like int). - an object is saved by : e.g. 'r7\\n' (object #7 being e.g. the __dict__). - a list or a set is saved by : \\n [...] e.g. '3\\ni0\\ni1\\ni2\\n' for [0, 1, 2] - a dict is saved by : \\n [...] REFERENCES ( above) In Cerealizer a reference can be either a reference to another object being serialized in the same file, or a raw value (e.g. an integer). - an int is saved by e.g. 'i187\\n' - a long is saved by e.g. 'l10000000000\\n' - a float is saved by e.g. 'f1.07\\n' - a bool is saved by 'b0' or 'b1' - a string is saved by e.g. 's5\\nascii' (where 5 is the number of characters) - an unicode is saved by e.g. 'u4\\nutf8' (where 4 is the number of characters) - an object reference is saved by e.g. 'r3\\n' (where 3 means reference to object #3) - None is saved by 'n' """ __all__ = ["load", "dump", "loads", "dumps", "freeze_configuration", "register", "register_class", "register_alias", "unregister"] VERSION = "0.8.3" import logging logger = logging.getLogger("cerealizer") #logging.basicConfig(level=logging.INFO) from cStringIO import StringIO from new import instance class EndOfFile(StandardError): pass class NotCerealizerFileError(StandardError): pass class NonCerealizableObjectError(StandardError): pass def _priority_sorter(a, b): return cmp(a[0], b[0]) class Dumper(object): def __init__(self): self.init() def init(self): self.objs = [] self.objs_id = set() self.priorities_objs = [] # [(priority1, obj1), (priority2, obj2),...] self.obj2state = {} self.obj2newargs = {} self.id2id = {} self.id2obj = None def dump(self, root_obj, s): self.collect(root_obj) self.priorities_objs.sort(_priority_sorter) self.objs.extend([o for (priority, o) in self.priorities_objs]) s.write("cereal1\n%s\n" % len(self.objs)) i = 0 for obj in self.objs: self.id2id[id(obj)] = i i += 1 for obj in self.objs: _HANDLERS_[obj.__class__].dump_obj (obj, self, s) for obj in self.objs: _HANDLERS_[obj.__class__].dump_data(obj, self, s) _HANDLERS_[root_obj.__class__].dump_ref(root_obj, self, s) self.init() def undump(self, s): txt = s.read(8) if txt != "cereal1\n": if txt == "": raise EndOfFile("") raise NotCerealizerFileError('Not a cerealizer file:\n"%s"' % txt) nb = int(s.readline()) self.id2obj = [ None ] * nb # DO NOT DO self.id2obj = [comprehension list], since undump_ref may access id2obj during its construction for i in range(nb): classname = s.readline() handler = _HANDLERS.get(classname) if not handler: raise NonCerealizableObjectError("Object of class/type '%s' cannot be de-cerealized! Use cerealizer.register to extend Cerealizer support to other classes." % classname[:-1]) self.id2obj[i] = handler.undump_obj(self, s) for obj in self.id2obj: _HANDLERS_[obj.__class__].undump_data(obj, self, s) r = self.undump_ref(s) self.init() return r def collect(self, obj): """Dumper.collect(OBJ) -> bool Collects OBJ for serialization. Returns false is OBJ is already collected; else returns true.""" handler = _HANDLERS_.get(obj.__class__) if not handler: raise NonCerealizableObjectError("Object of class/type '%s' cannot be cerealized! Use cerealizer.register to extend Cerealizer support to other classes." % obj.__class__) handler.collect(obj, self) def dump_ref (self, obj, s): """Dumper.dump_ref(OBJ, S) Writes a reference to OBJ in file S.""" _HANDLERS_[obj.__class__].dump_ref(obj, self, s) def undump_ref(self, s): """Dumper.undump_ref(S) -> obj Reads a reference from file S.""" c = s.read(1) if c == "i": return int (s.readline()) elif c == "f": return float(s.readline()) elif c == "s": return s.read(int(s.readline())) # str in Python 2 elif c == "u": return s.read(int(s.readline())).decode("utf8") # str in Python 3 or unicode in Python2 elif c == "y": return s.read(int(s.readline())) # bytes in Python 3 => str in Python 2 elif c == "r": return self.id2obj[int(s.readline())] elif c == "n": return None elif c == "b": return bool(int(s.read(1))) elif c == "l": return long(s.readline()) elif c == "c": return complex(s.readline()) raise ValueError("Unknown ref code '%s'!" % c) def immutable_depth(self, t): depth = 0 for i in t: i2 = self.obj2newargs.get(id(i)) if not i2 is None: i = i2 if isinstance(i, tuple) or isinstance(i, frozenset): x = self.immutable_depth(i) if x > depth: depth = x return depth + 1 class Handler(object): """Handler A customized handler for serialization and deserialization. You can subclass it to extend cerealization support to new object. See also ObjHandler.""" def collect(self, obj, dumper): """Handler.collect(obj, dumper) -> bool Collects all the objects referenced by OBJ. For each objects ROBJ referenced by OBJ, calls collect method of the Handler for ROBJ's class, i.e._HANDLERS_[ROBJ.__class__].collect(ROBJ, dumper). Returns false if OBJ is already referenced (and thus no collection should occur); else returns true. """ i = id(obj) if not i in dumper.objs_id: dumper.objs.append(obj) dumper.objs_id.add(i) return 1 def dump_obj (self, obj, dumper, s): """Handler.dump_obj(obj, dumper, s) Dumps OBJ classname in file S.""" s.write(self.classname) def dump_data(self, obj, dumper, s): """Handler.dump_data(obj, dumper, s) Dumps OBJ data in file S.""" def dump_ref (self, obj, dumper, s): """Handler.dump_ref(obj, dumper, s) Write a reference to OBJ in file S. You should not override dump_ref, since they is no corresponding 'undump_ref' that you can override.""" s.write("r%s\n" % dumper.id2id[id(obj)]) def undump_obj(self, dumper, s): """Handler.undump_obj(dumper, s) Returns a new uninitialized (=no __init__'ed) instance of the class. If you override undump_obj, DUMPER and file S can be used to read additional data saved by Handler.dump_obj().""" def undump_data(self, obj, dumper, s): """Handler.undump_data(obj, dumper, s) Reads the data for OBJ, from DUMPER and file S. If you override undump_data, you should use DUMPER.undump_ref(S) to read a reference or a basic type (=a string, an int,...).""" class RefHandler(object): def collect (self, obj, dumper) : pass def dump_obj (self, obj, dumper, s): pass def dump_data(self, obj, dumper, s): pass class NoneHandler(RefHandler): def dump_ref (self, obj, dumper, s): s.write("n") class StrHandler(RefHandler): def dump_ref (self, obj, dumper, s): s.write("s%s\n%s" % (len(obj), obj)) class UnicodeHandler(RefHandler): def dump_ref (self, obj, dumper, s): obj = obj.encode("utf8") s.write("u%s\n%s" % (len(obj), obj)) class BoolHandler(RefHandler): def dump_ref (self, obj, dumper, s): s.write("b%r" % int(obj)) class IntHandler(RefHandler): def dump_ref (self, obj, dumper, s): s.write("i%r\n" % obj) class LongHandler(RefHandler): def dump_ref (self, obj, dumper, s): s.write("l%r\n" % obj) class FloatHandler(RefHandler): def dump_ref (self, obj, dumper, s): s.write("f%r\n" % obj) class ComplexHandler(RefHandler): def dump_ref (self, obj, dumper, s): c = repr(obj) if c.startswith("("): c = c[1:-1] # complex("(1+2j)") doesn't work s.write("c%s\n" % c) class TupleHandler(Handler): classname = "tuple\n" def collect(self, obj, dumper): if not id(obj) in dumper.objs_id: dumper.priorities_objs.append((dumper.immutable_depth(obj), obj)) dumper.objs_id.add(id(obj)) for i in obj: dumper.collect(i) return 1 def dump_obj(self, obj, dumper, s): s.write("%s%s\n" % (self.classname, len(obj))) for i in obj: _HANDLERS_[i.__class__].dump_ref(i, dumper, s) def undump_obj(self, dumper, s): return tuple([dumper.undump_ref(s) for i in range(int(s.readline()))]) class FrozensetHandler(TupleHandler): classname = "frozenset\n" def undump_obj(self, dumper, s): return frozenset([dumper.undump_ref(s) for i in range(int(s.readline()))]) class ListHandler(Handler): classname = "list\n" def collect(self, obj, dumper): if Handler.collect(self, obj, dumper): for i in obj: dumper.collect(i) return 1 def dump_data(self, obj, dumper, s): s.write("%s\n" % len(obj)) for i in obj: _HANDLERS_[i.__class__].dump_ref(i, dumper, s) def undump_obj(self, dumper, s): return [] def undump_data(self, obj, dumper, s): for i in range(int(s.readline())): obj.append(dumper.undump_ref(s)) class SetHandler(ListHandler): classname = "set\n" def undump_obj(self, dumper, s): return set() def undump_data(self, obj, dumper, s): for i in range(int(s.readline())): obj.add(dumper.undump_ref(s)) class DictHandler(Handler): classname = "dict\n" def collect(self, obj, dumper): if Handler.collect(self, obj, dumper): for i in obj.iterkeys (): dumper.collect(i) # Collect is not ordered for i in obj.itervalues(): dumper.collect(i) return 1 def dump_data(self, obj, dumper, s): s.write("%s\n" % len(obj)) for k, v in obj.iteritems(): _HANDLERS_[v.__class__].dump_ref(v, dumper, s) # Value is saved fist _HANDLERS_[k.__class__].dump_ref(k, dumper, s) def undump_obj(self, dumper, s): return {} def undump_data(self, obj, dumper, s): for i in range(int(s.readline())): obj[dumper.undump_ref(s)] = dumper.undump_ref(s) # Value is read fist class ObjHandler(Handler): """ObjHandler A Cerealizer Handler that can support any new-style class instances, old-style class instances as well as C-defined types (although it may not save the C-side data).""" def __init__(self, Class, classname = ""): self.Class = Class self.Class_new = getattr(Class, "__new__" , instance) self.Class_getstate = getattr(Class, "__getstate__", None) # Check for and store __getstate__ and __setstate__ now self.Class_setstate = getattr(Class, "__setstate__", None) # so we are are they are not modified in the class or the object if classname: self.classname = "%s\n" % classname else: self.classname = "%s.%s\n" % (Class.__module__, Class.__name__) def collect(self, obj, dumper): i = id(obj) if not i in dumper.objs_id: dumper.priorities_objs.append((-1, obj)) dumper.objs_id.add(i) if self.Class_getstate: state = self.Class_getstate(obj) else: state = obj.__dict__ dumper.obj2state[i] = state dumper.collect(state) return 1 def dump_data(self, obj, dumper, s): i = dumper.obj2state[id(obj)] _HANDLERS_[i.__class__].dump_ref(i, dumper, s) def undump_obj(self, dumper, s): return self.Class_new(self.Class) def undump_data(self, obj, dumper, s): if self.Class_setstate: self.Class_setstate(obj, dumper.undump_ref(s)) else: obj.__dict__ = dumper.undump_ref(s) class SlotedObjHandler(ObjHandler): """SlotedObjHandler A Cerealizer Handler that can support new-style class instances with __slot__.""" def __init__(self, Class, classname = ""): ObjHandler.__init__(self, Class, classname) self.Class_slots = Class.__slots__ def collect(self, obj, dumper): i = id(obj) if not i in dumper.objs_id: dumper.priorities_objs.append((-1, obj)) dumper.objs_id.add(i) if self.Class_getstate: state = self.Class_getstate(obj) else: state = dict([(slot, getattr(obj, slot, None)) for slot in self.Class_slots]) dumper.obj2state[i] = state dumper.collect(state) return 1 def undump_data(self, obj, dumper, s): if self.Class_setstate: self.Class_setstate(obj, dumper.undump_ref(s)) else: state = dumper.undump_ref(s) for slot in self.Class_slots: setattr(obj, slot, state[slot]) class InitArgsObjHandler(ObjHandler): """InitArgsObjHandler A Cerealizer Handler that can support class instances with __getinitargs__.""" def __init__(self, Class, classname = ""): ObjHandler.__init__(self, Class, classname) self.Class_getinitargs = Class.__getinitargs__ self.Class_init = Class.__init__ def collect(self, obj, dumper): i = id(obj) if not i in dumper.objs_id: dumper.priorities_objs.append((-1, obj)) dumper.objs_id.add(i) dumper.obj2state[i] = state = self.Class_getinitargs(obj) dumper.collect(state) return 1 def undump_data(self, obj, dumper, s): self.Class_init(obj, *dumper.undump_ref(s)) class NewArgsObjHandler(ObjHandler): """NewArgsObjHandler A Cerealizer Handler that can support class instances with __getnewargs__.""" def __init__(self, Class, classname = ""): ObjHandler.__init__(self, Class, classname) self.Class_getnewargs = Class.__getnewargs__ def collect(self, obj, dumper): i = id(obj) if not i in dumper.objs_id: dumper.obj2newargs[i] = newargs = self.Class_getnewargs(obj) dumper.collect(newargs) dumper.priorities_objs.append((dumper.immutable_depth(newargs), obj)) dumper.objs_id.add(i) if self.Class_getstate: state = self.Class_getstate(obj) else: state = obj.__dict__ dumper.obj2state[i] = state dumper.collect(state) return 1 def dump_obj (self, obj, dumper, s): s.write(self.classname) newargs = dumper.obj2newargs[id(obj)] _HANDLERS_[newargs.__class__].dump_ref(newargs, dumper, s) def undump_obj(self, dumper, s): return self.Class_new(self.Class, *dumper.undump_ref(s)) _configurable = 1 _HANDLERS = {} _HANDLERS_ = {} def register(Class, handler = None, classname = ""): """register(Class, handler = None, classname = "") Registers CLASS as a serializable and secure class. By calling register, YOU HAVE TO ASSUME THAT THE FOLLOWING METHODS ARE SECURE: - CLASS.__new__ - CLASS.__del__ - CLASS.__getstate__ - CLASS.__setstate__ - CLASS.__getinitargs__ - CLASS.__init__ (only if CLASS.__getinitargs__ exists) HANDLER is the Cerealizer Handler object that handles serialization and deserialization for Class. If not given, Cerealizer create an instance of ObjHandler, which is suitable for old-style and new_style Python class, and also C-defined types (although if it has some C-side data, you may have to write a custom Handler or a __getstate__ and __setstate__ pair). CLASSNAME is the classname used in Cerealizer files. It defaults to the full classname (module.class) but you may choose something shorter -- as long as there is no risk of name clash.""" if not _configurable: raise StandardError("Cannot register new classes after freeze_configuration has been called!") if "\n" in classname: raise ValueError("CLASSNAME cannot have \\n (Cerealizer automatically add a trailing \\n for performance reason)!") if not handler: if hasattr(Class, "__getnewargs__" ): handler = NewArgsObjHandler (Class, classname) elif hasattr(Class, "__getinitargs__"): handler = InitArgsObjHandler(Class, classname) elif hasattr(Class, "__slots__" ): handler = SlotedObjHandler (Class, classname) else: handler = ObjHandler (Class, classname) if _HANDLERS_.has_key(Class): raise ValueError("Class %s has already been registred!" % Class) if not isinstance(handler, RefHandler): if _HANDLERS .has_key(handler.classname): raise ValueError("A class has already been registred under the name %s!" % handler.classname[:-1]) _HANDLERS [handler.classname] = handler if handler.__class__ is ObjHandler: logger.info("Registring class %s as '%s'" % (Class, handler.classname[:-1])) else: logger.info("Registring class %s as '%s' (using %s)" % (Class, handler.classname[:-1], handler.__class__.__name__)) else: logger.info("Registring reference '%s'" % Class) _HANDLERS_[Class] = handler register_class = register # For backward compatibility def unregister(Class_or_alias): """unregister(Class_or_alias) Unregister the given CLASS or ALIAS.""" if isinstance(Class_or_alias, str): Class_or_alias = Class_or_alias + "\n" del _HANDLERS_[Class_or_alias] def register_alias(Class, alias): """register_alias(Class, alias) Registers ALIAS as an alias classname for CLASS. Usefull for keeping backward compatibility in files: e.g. if you have renamed OldClass to NewClass, just do: cerealizer.register_alias(NewClass, "OldClass") and you'll be able to open old files containing OldClass serialized.""" handler = _HANDLERS_.get(Class) if not handler: raise ValueError("Cannot register alias '%s' to Class %s: the class is not yet registred!" % (alias, Class)) if _HANDLERS.has_key(alias): raise ValueError("Cannot register alias '%s' to Class %s: another class is already registred under the alias name!" % (alias, Class)) logger.info("Registring alias '%s' for %s" % (alias, Class)) _HANDLERS[alias + "\n"] = handler def freeze_configuration(): """freeze_configuration() Ends Cerealizer configuration. When freeze_configuration() is called, it is no longer possible to register classes, using register(). Calling freeze_configuration() is not mandatory, but it may enforce security, by forbidding unexpected calls to register().""" global _configurable _configurable = 0 logger.info("Configuration frozen") register(type(None), NoneHandler ()) register(str , StrHandler ()) register(unicode , UnicodeHandler ()) register(bool , BoolHandler ()) register(int , IntHandler ()) register(long , LongHandler ()) register(float , FloatHandler ()) register(complex , ComplexHandler ()) register(dict , DictHandler ()) register(list , ListHandler ()) register(set , SetHandler ()) register(tuple , TupleHandler ()) register(frozenset , FrozensetHandler()) def dump(obj, file, protocol = 0): """dump(obj, file, protocol = 0) Serializes object OBJ in FILE. FILE should be an opened file in *** binary *** mode. PROTOCOL is unused, it exists only for compatibility with Pickle.""" Dumper().dump(obj, file) def load(file): """load(file) -> obj De-serializes an object from FILE. FILE should be an opened file in *** binary *** mode.""" return Dumper().undump(file) def dumps(obj, protocol = 0): """dumps(obj, protocol = 0) -> str Serializes object OBJ and returns the serialized string. PROTOCOL is unused, it exists only for compatibility with Pickle.""" s = StringIO() Dumper().dump(obj, s) return s.getvalue() def loads(string): """loads(file) -> obj De-serializes an object from STRING.""" return Dumper().undump(StringIO(string)) def dump_class_of_module(*modules): """dump_class_of_module(*modules) Utility function; for each classes found in the given module, print the needed call to register.""" class D: pass class O(object): pass s = set([c for module in modules for c in module.__dict__.values() if isinstance(c, type(D)) or isinstance(c, type(O))]) l = ['cerealizer.register(%s.%s)' % (c.__module__, c.__name__) for c in s] l.sort() for i in l: print i ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1709281391.0 Cerealizer-0.8.4/__init__.py30000644002342100234210000005622714570310157014455 0ustar00jibajiba# -*- coding: utf-8 -*- # Cerealizer # Copyright (C) 2005-2010 Jean-Baptiste LAMY # Copyright (C) 2008 Peter Eckersley # # This program is free software. # It is available under the Python licence. """Cerealizer -- A secure Pickle-like module The interface of the Cerealizer module is similar to Pickle, and it supports __getstate__, __setstate__, __getinitargs__ and __getnewargs__. Cerealizer supports int, float, bool, complex, string, unicode, tuple, list, set, frozenset, dict, old-style and new-style class instances. C-defined types are supported but saving the C-side data may require to write e.g. a specific Handler or a __getstate__ and __setstate__ pair. Objects with __slots__ are supported too. You have to register the class you want to serialize, by calling cerealizer.register(YourClass). Cerealizer can be considered as secure AS LONG AS the following methods of 'YourClass' are secure: - __new__ - __del__ - __getstate__ - __setstate__ - __init__ (ONLY if __getinitargs__ is used for the class) These methods are the only one Cerealizer may call. For a higher security, Cerealizer maintains its own reference to these method (exepted __del__ that can only be called indirectly). Cerealizer doesn't aim at producing Human-readable files. About performances, Cerealizer is really fast and, when powered by Psyco, it may even beat cPickle! Although Cerealizer is implemented in less than 500 lines of pure-Python code (which is another reason for Cerealizer to be secure, since less code means less bugs :-). Compared to Pickle (cPickle): - Cerealizer is secure - Cerealizer achieves similar performances (using Psyco) - Cerealizer requires you to declare the serializable classes Compared to Jelly (from TwistedMatrix): - Cerealizer is faster - Cerealizer does a better job with object cycles, C-defined types and tuples (*) - Cerealizer files are not Human readable (*) Jelly handles them, but tuples and objects in a cycle are first created as _Tuple or _Dereference objects; this works for Python classes, but not with C-defined types which expects a precise type (e.g. tuple and not _Tuple). IMPLEMENTATION DETAILS GENERAL FILE FORMAT STRUCTURE Cerealizer format is simple but quite surprising. It uses a "double flat list" format. It looks like that : \\n \\n \\n \\n [...] [...] As you can see, the information for a given object is splitted in two parts, the first one for object's class, and the second one for the object's data. To avoid problems, the order of the objects is the following: Objects are put after basic types (list,...), since object's __setstate__ might rely on a list, and thus the list must be fully loaded BEFORE calling the object's __setstate__. DATA ( above) The part saves the data of object #n. It may contains reference to other data (see below, in Cerealizer references include reference to other objects but also raw data like int). - an object is saved by : e.g. 'r7\\n' (object #7 being e.g. the __dict__). - a list or a set is saved by : \\n [...] e.g. '3\\ni0\\ni1\\ni2\\n' for [0, 1, 2] - a dict is saved by : \\n [...] REFERENCES ( above) In Cerealizer a reference can be either a reference to another object being serialized in the same file, or a raw value (e.g. an integer). - an int is saved by e.g. 'i187\\n' - a float is saved by e.g. 'f1.07\\n' - a bool is saved by 'b0' or 'b1' - a string is saved by e.g. 's5\\nascii' (where 5 is the number of characters) - an unicode is saved by e.g. 'u4\\nutf8' (where 4 is the number of characters) - an object reference is saved by e.g. 'r3\\n' (where 3 means reference to object #3) - None is saved by 'n' """ __all__ = ["load", "dump", "loads", "dumps", "freeze_configuration", "register", "register_class", "register_alias", "unregister"] VERSION = "0.8.3" import logging logger = logging.getLogger("cerealizer") from io import BytesIO class EndOfFile (Exception): pass class NotCerealizerFileError (Exception): pass class NonCerealizableObjectError(Exception): pass def _priority_sorter_key(a): return a[0] class Dumper(object): def __init__(self): self.init() def init(self): self.objs = [] self.objs_id = set() self.priorities_objs = [] # [(priority1, obj1), (priority2, obj2),...] self.obj2state = {} self.obj2newargs = {} self.id2id = {} self.id2obj = None def dump(self, root_obj, s): self.collect(root_obj) self.priorities_objs.sort(key = _priority_sorter_key) self.objs.extend([o for (priority, o) in self.priorities_objs]) s.write(("cereal1\n%s\n" % len(self.objs)).encode("ascii")) i = 0 for obj in self.objs: self.id2id[id(obj)] = i i += 1 for obj in self.objs: _HANDLERS_[obj.__class__].dump_obj (obj, self, s) for obj in self.objs: _HANDLERS_[obj.__class__].dump_data(obj, self, s) _HANDLERS_[root_obj.__class__].dump_ref(root_obj, self, s) self.init() def undump(self, s): txt = s.read(8) if txt != b"cereal1\n": if txt == b"": raise EndOfFile("") raise NotCerealizerFileError('Not a cerealizer file:\n"%s"' % txt) nb = int(s.readline()) self.id2obj = [ None ] * nb # DO NOT DO self.id2obj = [comprehension list], since undump_ref may access id2obj during its construction for i in range(nb): classname = s.readline().decode("utf8") handler = _HANDLERS.get(classname) if not handler: raise NonCerealizableObjectError("Object of class/type '%s' cannot be de-cerealized! Use cerealizer.register to extend Cerealizer support to other classes." % classname[:-1]) self.id2obj[i] = handler.undump_obj(self, s) for obj in self.id2obj: _HANDLERS_[obj.__class__].undump_data(obj, self, s) r = self.undump_ref(s) self.init() return r def collect(self, obj): """Dumper.collect(OBJ) -> bool Collects OBJ for serialization. Returns false is OBJ is already collected; else returns true.""" handler = _HANDLERS_.get(obj.__class__) if not handler: raise NonCerealizableObjectError("Object of class/type '%s' cannot be cerealized! Use cerealizer.register to extend Cerealizer support to other classes." % obj.__class__) handler.collect(obj, self) def dump_ref (self, obj, s): """Dumper.dump_ref(OBJ, S) Writes a reference to OBJ in file S.""" _HANDLERS_[obj.__class__].dump_ref(obj, self, s) def undump_ref(self, s): """Dumper.undump_ref(S) -> obj Reads a reference from file S.""" c = s.read(1) if c == b"i": return int (s.readline()) elif c == b"f": return float(s.readline()) elif c == b"s": return s.read(int(s.readline())).decode("latin") # str in Python 2 => str in Python 3 elif c == b"u": return s.read(int(s.readline())).decode("utf8") # str in Python 3 or unicode in Python2 elif c == b"y": return s.read(int(s.readline())) # bytes in Python 3 elif c == b"r": return self.id2obj[int(s.readline())] elif c == b"n": return None elif c == b"b": return bool(int(s.read(1))) elif c == b"l": return int(s.readline()) elif c == b"c": return complex(s.readline().decode("ascii")) raise ValueError("Unknown ref code '%s'!" % c) def immutable_depth(self, t): depth = 0 for i in t: i2 = self.obj2newargs.get(id(i)) if not i2 is None: i = i2 if isinstance(i, tuple) or isinstance(i, frozenset): x = self.immutable_depth(i) if x > depth: depth = x return depth + 1 class Handler(object): """Handler A customized handler for serialization and deserialization. You can subclass it to extend cerealization support to new object. See also ObjHandler.""" def collect(self, obj, dumper): """Handler.collect(obj, dumper) -> bool Collects all the objects referenced by OBJ. For each objects ROBJ referenced by OBJ, calls collect method of the Handler for ROBJ's class, i.e._HANDLERS_[ROBJ.__class__].collect(ROBJ, dumper). Returns false if OBJ is already referenced (and thus no collection should occur); else returns true. """ i = id(obj) if not i in dumper.objs_id: dumper.objs.append(obj) dumper.objs_id.add(i) return 1 def dump_obj (self, obj, dumper, s): """Handler.dump_obj(obj, dumper, s) Dumps OBJ classname in file S.""" s.write(self.classname.encode("utf8")) def dump_data(self, obj, dumper, s): """Handler.dump_data(obj, dumper, s) Dumps OBJ data in file S.""" def dump_ref (self, obj, dumper, s): """Handler.dump_ref(obj, dumper, s) Write a reference to OBJ in file S. You should not override dump_ref, since they is no corresponding 'undump_ref' that you can override.""" s.write(("r%s\n" % dumper.id2id[id(obj)]).encode("ascii")) def undump_obj(self, dumper, s): """Handler.undump_obj(dumper, s) Returns a new uninitialized (=no __init__'ed) instance of the class. If you override undump_obj, DUMPER and file S can be used to read additional data saved by Handler.dump_obj().""" def undump_data(self, obj, dumper, s): """Handler.undump_data(obj, dumper, s) Reads the data for OBJ, from DUMPER and file S. If you override undump_data, you should use DUMPER.undump_ref(S) to read a reference or a basic type (=a string, an int,...).""" class RefHandler(object): def collect (self, obj, dumper) : pass def dump_obj (self, obj, dumper, s): pass def dump_data(self, obj, dumper, s): pass class NoneHandler(RefHandler): def dump_ref (self, obj, dumper, s): s.write(b"n") class BytesHandler(RefHandler): def dump_ref (self, obj, dumper, s): s.write(("y%s\n" % len(obj)).encode("ascii")) s.write(obj) class StrHandler(RefHandler): def dump_ref (self, obj, dumper, s): obj = obj.encode("utf8") s.write(("u%s\n" % len(obj)).encode("ascii")) s.write(obj) class BoolHandler(RefHandler): def dump_ref (self, obj, dumper, s): if obj: s.write(b"b1") else: s.write(b"b0") class IntHandler(RefHandler): def dump_ref (self, obj, dumper, s): s.write(("i%r\n" % obj).encode("ascii")) class FloatHandler(RefHandler): def dump_ref (self, obj, dumper, s): s.write(("f%r\n" % obj).encode("ascii")) class ComplexHandler(RefHandler): def dump_ref (self, obj, dumper, s): s.write(("c%s\n" % obj).encode("ascii")) class TupleHandler(Handler): classname = "tuple\n" def collect(self, obj, dumper): if not id(obj) in dumper.objs_id: dumper.priorities_objs.append((dumper.immutable_depth(obj), obj)) dumper.objs_id.add(id(obj)) for i in obj: dumper.collect(i) return 1 def dump_obj(self, obj, dumper, s): s.write(("%s%s\n" % (self.classname, len(obj))).encode("ascii")) for i in obj: _HANDLERS_[i.__class__].dump_ref(i, dumper, s) def undump_obj(self, dumper, s): return tuple([dumper.undump_ref(s) for i in range(int(s.readline()))]) class FrozensetHandler(TupleHandler): classname = "frozenset\n" def undump_obj(self, dumper, s): return frozenset([dumper.undump_ref(s) for i in range(int(s.readline()))]) class ListHandler(Handler): classname = "list\n" def collect(self, obj, dumper): if Handler.collect(self, obj, dumper): for i in obj: dumper.collect(i) return 1 def dump_data(self, obj, dumper, s): s.write(("%s\n" % len(obj)).encode("ascii")) for i in obj: _HANDLERS_[i.__class__].dump_ref(i, dumper, s) def undump_obj(self, dumper, s): return [] def undump_data(self, obj, dumper, s): for i in range(int(s.readline())): obj.append(dumper.undump_ref(s)) class SetHandler(ListHandler): classname = "set\n" def undump_obj(self, dumper, s): return set() def undump_data(self, obj, dumper, s): for i in range(int(s.readline())): obj.add(dumper.undump_ref(s)) class DictHandler(Handler): classname = "dict\n" def collect(self, obj, dumper): if Handler.collect(self, obj, dumper): for i in obj.keys (): dumper.collect(i) # Collect is not ordered for i in obj.values(): dumper.collect(i) return 1 def dump_data(self, obj, dumper, s): s.write(("%s\n" % len(obj)).encode("ascii")) for k, v in obj.items(): _HANDLERS_[v.__class__].dump_ref(v, dumper, s) # Value is saved fist _HANDLERS_[k.__class__].dump_ref(k, dumper, s) def undump_obj(self, dumper, s): return {} def undump_data(self, obj, dumper, s): for i in range(int(s.readline())): obj[dumper.undump_ref(s)] = dumper.undump_ref(s) # Value is read fist class ObjHandler(Handler): """ObjHandler A Cerealizer Handler that can support any new-style class instances, old-style class instances as well as C-defined types (although it may not save the C-side data).""" def __init__(self, Class, classname = ""): self.Class = Class self.Class_new = getattr(Class, "__new__") self.Class_getstate = getattr(Class, "__getstate__", None) # Check for and store __getstate__ and __setstate__ now self.Class_setstate = getattr(Class, "__setstate__", None) # so we are are they are not modified in the class or the object if classname: self.classname = "%s\n" % classname else: self.classname = "%s.%s\n" % (Class.__module__, Class.__name__) def collect(self, obj, dumper): i = id(obj) if not i in dumper.objs_id: dumper.priorities_objs.append((-1, obj)) dumper.objs_id.add(i) if self.Class_getstate: state = self.Class_getstate(obj) else: state = obj.__dict__ dumper.obj2state[i] = state dumper.collect(state) return 1 def dump_data(self, obj, dumper, s): i = dumper.obj2state[id(obj)] _HANDLERS_[i.__class__].dump_ref(i, dumper, s) def undump_obj(self, dumper, s): return self.Class_new(self.Class) def undump_data(self, obj, dumper, s): if self.Class_setstate: self.Class_setstate(obj, dumper.undump_ref(s)) else: obj.__dict__ = dumper.undump_ref(s) or {} class SlotedObjHandler(ObjHandler): """SlotedObjHandler A Cerealizer Handler that can support new-style class instances with __slot__.""" def __init__(self, Class, classname = ""): ObjHandler.__init__(self, Class, classname) self.Class_slots = Class.__slots__ def collect(self, obj, dumper): i = id(obj) if not i in dumper.objs_id: dumper.priorities_objs.append((-1, obj)) dumper.objs_id.add(i) if self.Class_getstate: state = self.Class_getstate(obj); print(self.Class_getstate) else: state = dict([(slot, getattr(obj, slot, None)) for slot in self.Class_slots]) dumper.obj2state[i] = state dumper.collect(state) return 1 def undump_data(self, obj, dumper, s): if self.Class_setstate: self.Class_setstate(obj, dumper.undump_ref(s)) else: state = dumper.undump_ref(s) for slot in self.Class_slots: setattr(obj, slot, state[1][slot]) class InitArgsObjHandler(ObjHandler): """InitArgsObjHandler A Cerealizer Handler that can support class instances with __getinitargs__.""" def __init__(self, Class, classname = ""): ObjHandler.__init__(self, Class, classname) self.Class_getinitargs = Class.__getinitargs__ self.Class_init = Class.__init__ def collect(self, obj, dumper): i = id(obj) if not i in dumper.objs_id: dumper.priorities_objs.append((-1, obj)) dumper.objs_id.add(i) dumper.obj2state[i] = state = self.Class_getinitargs(obj) dumper.collect(state) return 1 def undump_data(self, obj, dumper, s): self.Class_init(obj, *dumper.undump_ref(s)) class NewArgsObjHandler(ObjHandler): """NewArgsObjHandler A Cerealizer Handler that can support class instances with __getnewargs__.""" def __init__(self, Class, classname = ""): ObjHandler.__init__(self, Class, classname) self.Class_getnewargs = Class.__getnewargs__ def collect(self, obj, dumper): i = id(obj) if not i in dumper.objs_id: dumper.obj2newargs[i] = newargs = self.Class_getnewargs(obj) dumper.collect(newargs) dumper.priorities_objs.append((dumper.immutable_depth(newargs), obj)) dumper.objs_id.add(i) if self.Class_getstate: state = self.Class_getstate(obj) else: state = obj.__dict__ dumper.obj2state[i] = state dumper.collect(state) return 1 def dump_obj (self, obj, dumper, s): s.write(self.classname.encode("utf8")) newargs = dumper.obj2newargs[id(obj)] _HANDLERS_[newargs.__class__].dump_ref(newargs, dumper, s) def undump_obj(self, dumper, s): return self.Class_new(self.Class, *dumper.undump_ref(s)) _configurable = 1 _HANDLERS = {} _HANDLERS_ = {} def register(Class, handler = None, classname = ""): """register(Class, handler = None, classname = "") Registers CLASS as a serializable and secure class. By calling register, YOU HAVE TO ASSUME THAT THE FOLLOWING METHODS ARE SECURE: - CLASS.__new__ - CLASS.__del__ - CLASS.__getstate__ - CLASS.__setstate__ - CLASS.__getinitargs__ - CLASS.__init__ (only if CLASS.__getinitargs__ exists) HANDLER is the Cerealizer Handler object that handles serialization and deserialization for Class. If not given, Cerealizer create an instance of ObjHandler, which is suitable for old-style and new_style Python class, and also C-defined types (although if it has some C-side data, you may have to write a custom Handler or a __getstate__ and __setstate__ pair). CLASSNAME is the classname used in Cerealizer files. It defaults to the full classname (module.class) but you may choose something shorter -- as long as there is no risk of name clash.""" if not _configurable: raise Exception("Cannot register new classes after freeze_configuration has been called!") if "\n" in classname: raise ValueError("CLASSNAME cannot have \\n (Cerealizer automatically add a trailing \\n for performance reason)!") if not handler: if hasattr(Class, "__getnewargs__" ): handler = NewArgsObjHandler (Class, classname) elif hasattr(Class, "__getinitargs__"): handler = InitArgsObjHandler(Class, classname) elif hasattr(Class, "__slots__" ): handler = SlotedObjHandler (Class, classname) else: handler = ObjHandler (Class, classname) if Class in _HANDLERS_: raise ValueError("Class %s has already been registred!" % Class) if not isinstance(handler, RefHandler): if handler.classname in _HANDLERS: raise ValueError("A class has already been registred under the name %s!" % handler.classname[:-1]) _HANDLERS [handler.classname] = handler if handler.__class__ is ObjHandler: logger.info("Registring class %s as '%s'" % (Class, handler.classname[:-1])) else: logger.info("Registring class %s as '%s' (using %s)" % (Class, handler.classname[:-1], handler.__class__.__name__)) else: logger.info("Registring reference '%s'" % Class) _HANDLERS_[Class] = handler register_class = register # For backward compatibility def unregister(Class_or_alias): """unregister(Class_or_alias) Unregister the given CLASS or ALIAS.""" if isinstance(Class_or_alias, str): Class_or_alias = Class_or_alias + "\n" del _HANDLERS_[Class_or_alias] def register_alias(Class, alias): """register_alias(Class, alias) Registers ALIAS as an alias classname for CLASS. Usefull for keeping backward compatibility in files: e.g. if you have renamed OldClass to NewClass, just do: cerealizer.register_alias(NewClass, "OldClass") and you'll be able to open old files containing OldClass serialized.""" handler = _HANDLERS_.get(Class) if not handler: raise ValueError("Cannot register alias '%s' to Class %s: the class is not yet registred!" % (alias, Class)) if alias in _HANDLERS: raise ValueError("Cannot register alias '%s' to Class %s: another class is already registred under the alias name!" % (alias, Class)) logger.info("Registring alias '%s' for %s" % (alias, Class)) _HANDLERS[alias + "\n"] = handler def freeze_configuration(): """freeze_configuration() Ends Cerealizer configuration. When freeze_configuration() is called, it is no longer possible to register classes, using register(). Calling freeze_configuration() is not mandatory, but it may enforce security, by forbidding unexpected calls to register().""" global _configurable _configurable = 0 logger.info("Configuration frozen") register(type(None), NoneHandler ()) register(bytes , BytesHandler ()) register(str , StrHandler ()) register(bool , BoolHandler ()) register(int , IntHandler ()) register(float , FloatHandler ()) register(complex , ComplexHandler ()) register(dict , DictHandler ()) register(list , ListHandler ()) register(set , SetHandler ()) register(tuple , TupleHandler ()) register(frozenset , FrozensetHandler()) def dump(obj, file, protocol = 0): """dump(obj, file, protocol = 0) Serializes object OBJ in FILE. FILE should be an opened file in *** binary *** mode. PROTOCOL is unused, it exists only for compatibility with Pickle.""" Dumper().dump(obj, file) def load(file): """load(file) -> obj De-serializes an object from FILE. FILE should be an opened file in *** binary *** mode.""" return Dumper().undump(file) def dumps(obj, protocol = 0): """dumps(obj, protocol = 0) -> str Serializes object OBJ and returns the serialized string. PROTOCOL is unused, it exists only for compatibility with Pickle.""" s = BytesIO() Dumper().dump(obj, s) return s.getvalue() def loads(string): """loads(file) -> obj De-serializes an object from STRING.""" return Dumper().undump(BytesIO(string)) def dump_class_of_module(*modules): """dump_class_of_module(*modules) Utility function; for each classes found in the given module, print the needed call to register.""" class D: pass class O(object): pass s = {c for module in modules for c in list(module.__dict__.values()) if isinstance(c, type(D)) or isinstance(c, type(O))} l = ['cerealizer.register(%s.%s)' % (c.__module__, c.__name__) for c in s] l.sort() for i in l: print(i) ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1391963324.0 Cerealizer-0.8.4/datetime_handler.py0000644002342100234210000000053212275726274016123 0ustar00jibajiba# -*- coding: utf-8 -*- # Cerealizer # Copyright (C) 2005-2012 Jean-Baptiste LAMY # Copyright (C) 2008 Peter Eckersley # # This program is free software. # It is available under the Python licence. import sys if sys.version[0] == "2": execfile("%s.py2" % __file__.rsplit(".")[0]) else: exec(open("%s.py3" % __file__.rsplit(".")[0]).read()) ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1391963324.0 Cerealizer-0.8.4/datetime_handler.py20000644002342100234210000000533312275726274016211 0ustar00jibajiba# Cerealizer # Copyright (C) 2008 Steve Benson, Lann Martin # Copyright (C) 2012 Jean-Baptiste LAMY # # This program is free software. # It is available under the Python licence. import datetime import cerealizer class DatetimeHandler(cerealizer.Handler): classname = 'datetime\n' def dump_obj(self, obj, dumper, s): assert issubclass(obj.__class__, datetime.datetime) # This works based on datetime.__reduce__(): # datetime.__reduce__()[1] is a string of bytes that is an internal # representation of a datetime object (the data field # PyDateTime_DateTime struct, an unsigned char # _PyDateTime_DATETIME_DATASIZE bytes long). This should be platform # independent and is validated upon reconstruction of a datetime. # Pickle uses it and guarantees backwards compatibility, so # presumably we can use it here too. # I haven't verified that tzinfo objects have the same properties, # which is why they're unsupported right now. # This string returned by __reduce__ is encoded because it can # return data that interferes with the cerealizer protocol. if obj.tzinfo != None: raise ValueError("DatetimeHandler doesn't yet know how to handle datetime objects with tzinfo.") s.write('%s%s\n' % ( self.classname, obj.__reduce__()[1][0].encode('string_escape') )) def undump_obj(self, dumper, s): line = s.readline() if len(line)>0 and line[-1]=='\n': line = line[:-1] return datetime.datetime(line.decode('string_escape')) class DateHandler(cerealizer.Handler): classname = 'date\n' def dump_obj(self, obj, dumper, s): assert issubclass(obj.__class__, datetime.date) s.write('%s%s\n' % ( self.classname, obj.__reduce__()[1][0].encode('string_escape') )) def undump_obj(self, dumper, s): line = s.readline() if len(line) > 0 and line[-1] == '\n': line = line[:-1] return datetime.date(line.decode('string_escape')) class TimeHandler(cerealizer.Handler): classname = 'time\n' def dump_obj(self, obj, dumper, s): assert issubclass(obj.__class__, datetime.time) if obj.tzinfo != None: raise ValueError("DatetimeHandler doesn't yet know how to handle datetime objects with tzinfo.") s.write('%s%s\n' % ( self.classname, obj.__reduce__()[1][0].encode('string_escape') )) def undump_obj(self, dumper, s): line = s.readline() if len(line)>0 and line[-1]=='\n': line = line[:-1] return datetime.time(line.decode('string_escape')) cerealizer.register(datetime.datetime, DatetimeHandler()) cerealizer.register(datetime.date , DateHandler ()) cerealizer.register(datetime.time , TimeHandler ()) ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1391963324.0 Cerealizer-0.8.4/datetime_handler.py30000644002342100234210000000551412275726274016213 0ustar00jibajiba# Cerealizer # Copyright (C) 2008 Steve Benson, Lann Martin # Copyright (C) 2012 Jean-Baptiste LAMY # # This program is free software. # It is available under the Python licence. import datetime import cerealizer class DatetimeHandler(cerealizer.Handler): classname = 'datetime\n' def dump_obj(self, obj, dumper, s): assert issubclass(obj.__class__, datetime.datetime) # This works based on datetime.__reduce__(): # datetime.__reduce__()[1] is a string of bytes that is an internal # representation of a datetime object (the data field # PyDateTime_DateTime struct, an unsigned char # _PyDateTime_DATETIME_DATASIZE bytes long). This should be platform # independent and is validated upon reconstruction of a datetime. # Pickle uses it and guarantees backwards compatibility, so # presumably we can use it here too. # I haven't verified that tzinfo objects have the same properties, # which is why they're unsupported right now. # This string returned by __reduce__ is encoded because it can # return data that interferes with the cerealizer protocol. if obj.tzinfo != None: raise ValueError("DatetimeHandler doesn't yet know how to handle datetime objects with tzinfo.") s.write(self.classname.encode("utf8")) s.write(obj.__reduce__()[1][0].decode("latin").encode('unicode_escape')) s.write(b'\n') def undump_obj(self, dumper, s): line = s.readline().decode('unicode_escape') if line and (line[-1] == '\n'): line = line[:-1] return datetime.datetime(line.encode("latin")) class DateHandler(cerealizer.Handler): classname = 'date\n' def dump_obj(self, obj, dumper, s): assert issubclass(obj.__class__, datetime.date) s.write(self.classname.encode("utf8")) s.write(obj.__reduce__()[1][0].decode("latin").encode('unicode_escape')) s.write(b'\n') def undump_obj(self, dumper, s): line = s.readline().decode('unicode_escape') if line and (line[-1] == '\n'): line = line[:-1] return datetime.date(line.encode("latin")) class TimeHandler(cerealizer.Handler): classname = 'time\n' def dump_obj(self, obj, dumper, s): assert issubclass(obj.__class__, datetime.time) if obj.tzinfo != None: raise ValueError("DatetimeHandler doesn't yet know how to handle datetime objects with tzinfo.") s.write(self.classname.encode("utf8")) s.write(obj.__reduce__()[1][0].decode("latin").encode('unicode_escape')) s.write(b'\n') def undump_obj(self, dumper, s): line = s.readline().decode('unicode_escape') if line and (line[-1] == '\n'): line = line[:-1] return datetime.time(line.encode("latin")) cerealizer.register(datetime.datetime, DatetimeHandler()) cerealizer.register(datetime.date , DateHandler ()) cerealizer.register(datetime.time , TimeHandler ()) ././@PaxHeader0000000000000000000000000000003300000000000010211 xustar0027 mtime=1709281468.330042 Cerealizer-0.8.4/setup.cfg0000644002342100234210000000020514570310274014063 0ustar00jibajiba[install_lib] compile = 1 optimize = 1 [sdist] force-manifest = 1 dist-dir = /home/jiba/dist [egg_info] tag_build = tag_date = 0 ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1709281418.0 Cerealizer-0.8.4/setup.py0000644002342100234210000000414714570310212013755 0ustar00jibajiba#! /usr/bin/env python # Cerealizer # Copyright (C) 2005-2012 Jean-Baptiste LAMY # # This program is free software. # It is available under the Python licence. import sys, os.path import setuptools if ("build" in sys.argv) or ("install" in sys.argv) or ("bdist_wheel" in sys.argv) or ("bdist_egg" in sys.argv): HERE = os.path.dirname(sys.argv[0]) or "." for filename in os.listdir(HERE): filename = os.path.join(HERE, filename) if filename.endswith(".py%s" % sys.version[0]): open(filename[:-1], "w").write(open(filename).read()) include_package_data = False else: include_package_data = True setuptools.setup (name = "Cerealizer", version = "0.8.4", license = "Python licence", description = "A secure pickle-like module", long_description = """A secure pickle-like module. It support basic types (int, string, unicode, tuple, list, dict, set,...), old and new-style classes (you need to register the class for security), object cycles, and it can be extended to support C-defined type.""", author = "Lamy Jean-Baptiste (Jiba)", author_email = "jibalamy@free.fr", url = "http://www.lesfleursdunormal.fr/static/informatique/cerealizer/index_en.html", classifiers = [ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "License :: OSI Approved :: Python Software Foundation License", "Programming Language :: Python :: 2", "Programming Language :: Python :: 3", "Topic :: Security", "Topic :: Software Development :: Libraries :: Python Modules", ], package_dir = {"cerealizer" : ""}, packages = ["cerealizer"], #include_package_data = include_package_data, ) ././@PaxHeader0000000000000000000000000000003300000000000010211 xustar0027 mtime=1709281468.330042 Cerealizer-0.8.4/test/0000755002342100234210000000000014570310274013224 5ustar00jibajiba././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1709281340.0 Cerealizer-0.8.4/test/regtest.py0000644002342100234210000002240114570310074015250 0ustar00jibajiba# -*- coding: latin-1 -*- # Cerealizer # Copyright (C) 2005-2006 Jean-Baptiste LAMY # # This program is free software. # It is available under the Python licence. # Unit tests for Cerealizer import cerealizer import unittest class TestBasicType(unittest.TestCase): def setUp(self): self.obj = [1, 2, "jiba"] def loads_dumps_and_compare(self, obj1): obj2 = cerealizer.loads(cerealizer.dumps(obj1)) assert obj1 == obj2 def test_int1 (self): self.loads_dumps_and_compare(7828) def test_int2 (self): self.loads_dumps_and_compare(-579) def test_float1 (self): self.loads_dumps_and_compare(4.9) def test_float2 (self): self.loads_dumps_and_compare(-0.0043) def test_float3 (self): self.loads_dumps_and_compare(4.0) def test_complex (self): self.loads_dumps_and_compare(1+2j) def test_string1 (self): self.loads_dumps_and_compare( "jiba") def test_string2 (self): self.loads_dumps_and_compare( "jibé") def test_unicode1(self): self.loads_dumps_and_compare(u"jiba") def test_unicode2(self): self.loads_dumps_and_compare(u"jibé") def test_tuple1 (self): self.loads_dumps_and_compare(()) def test_tuple2 (self): self.loads_dumps_and_compare((1, 2.2, "jiba")) def test_tuple3 (self): self.loads_dumps_and_compare((1, (2.2, "jiba"))) def test_frozenset(self): self.loads_dumps_and_compare(frozenset([1, (2.2, "jiba")])) def test_list1 (self): self.loads_dumps_and_compare([]) def test_list2 (self): self.loads_dumps_and_compare([1, 2.2, "jiba"]) def test_list3 (self): self.loads_dumps_and_compare([1, [2.2, "jiba"]]) def test_set1 (self): self.loads_dumps_and_compare(set()) def test_set2 (self): self.loads_dumps_and_compare(set([1, 2.2, "jiba"])) def test_dict1 (self): self.loads_dumps_and_compare({}) def test_dict2 (self): self.loads_dumps_and_compare({ "jiba" : 100, "other" : 0 }) def test_dict3 (self): self.loads_dumps_and_compare({ "jiba" : list(range(100)), "other" : { 1:2 } }) def test_None (self): self.loads_dumps_and_compare(None) def test_obj_oldstyle(self): class Obj1: def __init__(self): self.x = 1 self.name = "jiba" def __eq__(a, b): return (a.__class__ is b.__class__) and (a.__dict__ == b.__dict__) cerealizer.register(Obj1) self.loads_dumps_and_compare(Obj1()) def test_obj_newstyle(self): class Obj2(object): def __init__(self): self.x = 1 self.name = "jiba" def __eq__(a, b): return (a.__class__ is b.__class__) and (a.__dict__ == b.__dict__) cerealizer.register(Obj2) self.loads_dumps_and_compare(Obj2()) def test_obj_setstate_priority1(self): LIST = [1, 2, 3] class Obj3(object): def __init__(self): self.x = 1 self.name = "jiba" self.list = LIST[:] def __eq__(a, b): return (a.__class__ is b.__class__) and (a.__dict__ == b.__dict__) def __setstate__(self, state): assert state["list"] == LIST # Test that list is initialized BEFORE the object self.__dict__ = state cerealizer.register(Obj3) self.loads_dumps_and_compare(Obj3()) def test_obj_setstate_priority2(self): TUPLE = (1, 2, 3, (4, (5, 6, (7,)))) class Obj6(object): def __init__(self): self.x = 1 self.name = "jiba" self.list = TUPLE[:] def __eq__(a, b): return (a.__class__ is b.__class__) and (a.__dict__ == b.__dict__) def __setstate__(self, state): assert state["list"] == TUPLE # Test that list is initialized BEFORE the object self.__dict__ = state cerealizer.register(Obj6) self.loads_dumps_and_compare(Obj6()) def test_obj_getstate_setstate(self): STATE = (1, 2, "jiba") class Obj4(object): def __init__(self): self.x = 1 self.name = "jiba" def __eq__(a, b): return (a.__class__ is b.__class__) and (a.__dict__ == b.__dict__) def __getstate__(self): return STATE def __setstate__(self, state): assert state == STATE self.x = 1 self.name = "jiba" cerealizer.register(Obj4) self.loads_dumps_and_compare(Obj4()) def test_obj_new_and_init(self): nbs = [0, 0] class Obj5(object): def __new__(Class): nbs[0] += 1 return object.__new__(Class) def __init__(self): nbs[1] += 1 self.x = 1 self.name = "jiba" def __eq__(a, b): return (a.__class__ is b.__class__) and (a.__dict__ == b.__dict__) cerealizer.register(Obj5) o = Obj5() self.loads_dumps_and_compare(o) assert nbs == [2, 1] def test_identity(self): o = {} l1 = [o, o] l2 = cerealizer.loads(cerealizer.dumps(l1)) assert l2[0] is l2[1] def test_cycle1(self): obj1 = [1, [2.2, "jiba"]] obj1[1].append(obj1) obj2 = cerealizer.loads(cerealizer.dumps(obj1)) assert repr(obj1) == repr(obj2) # Cannot use == on cyclic list! def test_cycle2(self): class Obj11(object): pass cerealizer.register(Obj11) o = Obj11() o.o = o o2 = cerealizer.loads(cerealizer.dumps(o)) assert o2.o is o2 def test_cycle3(self): class Parent: pass class Child: def __init__(self, parent): self.parent = parent def __getstate__(self): return (self.parent,) def __setstate__(self, state): self.parent = state[0] cerealizer.register(Parent) cerealizer.register(Child) p = Parent() p.c = Child(p) p2 = cerealizer.loads(cerealizer.dumps(p)) assert not p2.c.parent is None def test_obj_slot(self): class Obj7(object): __slots__ = ["x", "name"] def __init__(self): self.x = 11.1 self.name = "jiba" def __eq__(a, b): return (a.__class__ is b.__class__) and (a.x == b.x) and (a.name == b.name) cerealizer.register(Obj7) o = Obj7() self.loads_dumps_and_compare(o) def test_obj_initargs1(self): class Obj8: def __init__(self, x, name): self.x = x self.name = name def __getinitargs__(self): return self.x, self.name def __eq__(a, b): return (a.__class__ is b.__class__) and (a.x == b.x) and (a.name == b.name) cerealizer.register(Obj8) o = Obj8(45, u"uioef") self.loads_dumps_and_compare(o) def test_obj_initargs2(self): class Obj9(object): def __init__(self, x, name): self.x = x self.name = name def __getinitargs__(self): return self.x, self.name def __eq__(a, b): return (a.__class__ is b.__class__) and (a.x == b.x) and (a.name == b.name) cerealizer.register(Obj9) o = Obj9(45, u"uioef") self.loads_dumps_and_compare(o) def test_obj_newargs1(self): class Obj10(object): def __new__(Class, x, name): self = object.__new__(Class) self.x = x self.name = name return self def __getnewargs__(self): return self.x, self.name def __eq__(a, b): return (a.__class__ is b.__class__) and (a.x == b.x) and (a.name == b.name) cerealizer.register(Obj10) o = Obj10(45, u"uioef") self.loads_dumps_and_compare(o) def test_obj_newargs2(self): class Obj12(object): def __new__(Class, x): self = object.__new__(Class) self.x = x return self def __getnewargs__(self): return (self.x,) def __eq__(a, b): return (a.__class__ is b.__class__) and (a.x == b.x) cerealizer.register(Obj12) o = Obj12(45) o2 = Obj12(o) self.loads_dumps_and_compare(o2) def test_alias1(self): class OldObj(object): def __init__(self): self.x = 1 self.name = "jiba" cerealizer.register(OldObj) o = OldObj() s = cerealizer.dumps(o) cerealizer.unregister(OldObj) class NewObj(object): def __init__(self): self.x = 2 self.name = "jiba2" cerealizer.register(NewObj) cerealizer.register_alias(NewObj, "__main__.OldObj") o = cerealizer.loads(s) assert o.__class__ is NewObj assert o.x == 1 assert o.name == "jiba" class TestSecurity(unittest.TestCase): def test_register1(self): class Sec1: pass self.assertRaises(cerealizer.NonCerealizableObjectError, lambda : cerealizer.dumps(Sec1())) def test_register2(self): class Sec2: pass cerealizer.register(Sec2) self.assertRaises(ValueError, lambda : cerealizer.register(Sec2)) def test_register3(self): class Sec3: pass cerealizer.register(Sec3) class Sec3: pass self.assertRaises(ValueError, lambda : cerealizer.register(Sec3)) def test_setstate_hacked(self): class Sec4: pass cerealizer.register(Sec4) o = Sec4() Sec4.__setstate__ = lambda obj, state: self.fail() cerealizer.loads(cerealizer.dumps(o)) def test_getstate_hacked(self): class Sec5: pass cerealizer.register(Sec5) o = Sec5() Sec5.__getstate__ = lambda obj: self.fail() cerealizer.loads(cerealizer.dumps(o)) def test_new_hacked(self): class Sec6: pass cerealizer.register(Sec6) o = Sec6() Sec6.__new__ = lambda Class: self.fail() cerealizer.loads(cerealizer.dumps(o)) def test_craked_file1(self): craked_file = "cereal1\n2\n__builtin__.dict\nfile\n0\nr0\nr1\n" self.assertRaises(Exception, lambda : cerealizer.loads(craked_file)) if __name__ == '__main__': unittest.main() ././@PaxHeader0000000000000000000000000000002600000000000010213 xustar0022 mtime=1391963324.0 Cerealizer-0.8.4/test/test1.py0000644002342100234210000000471512275726274014660 0ustar00jibajiba# Cerealizer # Copyright (C) 2005 Jean-Baptiste LAMY # # This program is free software. # It is available under the Python licence. # Small benchmark from __future__ import print_function import cerealizer import time #import psyco #psyco.full() class O(object): def __init__(self): self.x = 1 self.s = "jiba" self.o = None self.l = [1, 2, 3, 4] cerealizer.register(O) cerealizer.freeze_configuration() l = [] for i in range(20000): o = O() if l: o.o = l[-1] l.append(o) print("cerealizer") t = time.time() s = cerealizer.dumps(l) print("dumps in", time.time() - t, "s,", len(s), "bytes length") t = time.time() l2 = cerealizer.loads(s) print("loads in", time.time() - t, "s") import pickle print() print("pickle") t = time.time() s = pickle.dumps(l) print("dumps in", time.time() - t, "s,", len(s), "bytes length") t = time.time() l2 = pickle.loads(s) print("loads in", time.time() - t, "s") import cPickle print() print("cPickle") t = time.time() s = cPickle.dumps(l) print("dumps in", time.time() - t, "s,", len(s), "bytes length") t = time.time() l2 = cPickle.loads(s) print("loads in", time.time() - t, "s") import twisted.spread.jelly, twisted.spread.banana class O(object, twisted.spread.jelly.Jellyable, twisted.spread.jelly.Unjellyable): def __init__(self): self.x = 1 self.s = "jiba" self.o = None self.l = [1, 2, 3, 4] cerealizer.register(O) cerealizer.freeze_configuration() l = [] for i in range(20000): o = O() if l: o.o = l[-1] l.append(o) print() print("jelly + banana") t = time.time() s = twisted.spread.banana.encode(twisted.spread.jelly.jelly(l)) print("dumps in", time.time() - t, "s,", len(s), "bytes length") t = time.time() l2 = twisted.spread.jelly.unjelly(twisted.spread.banana.decode(s)) print("loads in", time.time() - t, "s") import twisted.spread.cBanana twisted.spread.banana.cBanana = twisted.spread.cBanana twisted.spread.cBanana.pyb1282int=twisted.spread.banana.b1282int twisted.spread.cBanana.pyint2b128=twisted.spread.banana.int2b128 twisted.spread.banana._i = twisted.spread.banana.Canana() twisted.spread.banana._i.connectionMade() twisted.spread.banana._i._selectDialect("none") print() print("jelly + cBanana") t = time.time() s = twisted.spread.banana.encode(twisted.spread.jelly.jelly(l)) print("dumps in", time.time() - t, "s,", len(s), "bytes length") t = time.time() l2 = twisted.spread.jelly.unjelly(twisted.spread.banana.decode(s)) print("loads in", time.time() - t, "s")