pax_global_header00006660000000000000000000000064125605556460014530gustar00rootroot0000000000000052 comment=e00238867875df0258cf4f83f528d846e7c1afc4 picklable-itertools-0.1.1/000077500000000000000000000000001256055564600154775ustar00rootroot00000000000000picklable-itertools-0.1.1/.gitignore000066400000000000000000000012521256055564600174670ustar00rootroot00000000000000# Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] # C extensions *.so # Distribution / packaging .Python env/ build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ *.egg-info/ .installed.cfg *.egg # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .coverage .cache nosetests.xml coverage.xml # Translations *.mo *.pot # Django stuff: *.log # Sphinx documentation docs/_build/ # PyBuilder target/ picklable-itertools-0.1.1/.scrutinizer.yml000066400000000000000000000141701256055564600206640ustar00rootroot00000000000000build: dependencies: override: - pip install -q flake8 tests: override: - flake8 picklable_itertools tests checks: python: code_rating: true duplicate_code: true format_bad_indentation: indentation: '4 spaces' format_mixed_indentation: true format_line_too_long: max_length: '79' imports_relative_import: true imports_wildcard_import: true format_bad_whitespace: true format_multiple_statements: true basic_invalid_name: functions: '[a-z_][a-z0-9_]{0,30}$' variables: '(([a-z_][a-z0-9_]{0,30})|(_?[A-Z]))$' whitelisted_names: '_,floatX,logger,config' constants: '(([A-Z_][A-Z0-9_]*)|(__.*__))$' attributes: '(([a-z_][a-z0-9_]{0,30})|(_?[A-Z]))$' arguments: '(([a-z_][a-z0-9_]{0,30})|(_?[A-Z]))$' class_attributes: '([A-Za-z_][A-Za-z0-9_]{0,30}|(__.*__))$' inline_vars: '[A-Za-z_][A-Za-z0-9_]*$' classes: '[A-Z_][a-zA-Z0-9]+$' modules: '(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$' methods: '[a-z_][a-z0-9_]{0,30}$' classes_no_self_argument: true classes_bad_mcs_method_argument: true classes_bad_classmethod_argument: true variables_unused_variable: true variables_unused_import: true variables_used_before_assignment: true variables_undefined_variable: true variables_undefined_loop_variable: true variables_redefined_outer_name: true variables_redefined_builtin: true variables_redefine_in_handler: true variables_no_name_in_module: true variables_global_variable_undefined: true variables_global_variable_not_assigned: true variables_global_statement: true typecheck_unexpected_keyword_arg: true variables_global_at_module_level: true variables_unused_wildcard_import: true variables_unused_argument: true variables_unpacking_non_sequence: true variables_undefined_all_variable: true variables_unbalanced_tuple_unpacking: true variables_invalid_all_object: true typecheck_too_many_function_args: true typecheck_redundant_keyword_arg: true typecheck_not_callable: true typecheck_no_member: true typecheck_missing_kwoa: true typecheck_maybe_no_member: true typecheck_duplicate_keyword_arg: true typecheck_assignment_from_none: true typecheck_assignment_from_no_return: true string_unused_format_string_key: true string_truncated_format_string: true string_too_many_format_args: true string_too_few_format_args: true string_mixed_format_string: true string_missing_format_string_key: true string_format_needs_mapping: true string_constant_anomalous_unicode_escape_in_string: true string_constant_anomalous_backslash_in_string: true string_bad_str_strip_call: true string_bad_format_string_key: true string_bad_format_character: true open_mode_bad_open_mode: true newstyle_bad_super_call: true logging_unsupported_format: true logging_too_many_args: true logging_too_few_args: true logging_not_lazy: true logging_format_truncated: true imports_reimported: true imports_import_self: true imports_deprecated_module: true imports_cyclic_import: true format_unnecessary_semicolon: true format_trailing_whitespace: true format_superfluous_parens: true format_old_ne_operator: true format_missing_final_newline: true format_lowercase_l_suffix: true format_backtick: true exceptions_raising_string: true exceptions_raising_non_exception: true exceptions_raising_bad_type: true exceptions_pointless_except: true exceptions_notimplemented_raised: true exceptions_catching_non_exception: true exceptions_broad_except: true exceptions_binary_op_exception: true exceptions_bare_except: true exceptions_bad_except_order: true design_interface_not_implemented: true design_abstract_class_not_used: true design_abstract_class_little_used: true classes_valid_slots: true classes_super_init_not_called: true classes_signature_differs: true classes_protected_access: true classes_non_parent_init_called: true classes_non_iterator_returned: true classes_no_method_argument: true classes_no_method_argument: true classes_no_init: true classes_missing_interface_method: true classes_method_hidden: true classes_interface_is_not_class: true classes_bad_staticmethod_argument: true classes_bad_mcs_classmethod_argument: true classes_bad_context_manager: true classes_arguments_differ: true classes_access_member_before_definition: true basic_yield_outside_function: true basic_useless_else_on_loop: true basic_unreachable: true basic_unnecessary_pass: true basic_unnecessary_lambda: true basic_return_outside_function: true basic_return_in_init: true basic_return_arg_in_generator: true basic_pointless_string_statement: true basic_pointless_statement: true basic_old_raise_syntax: true basic_not_in_loop: true basic_nonexistent_operator: true basic_missing_reversed_argument: true basic_missing_module_attribute: true basic_lost_exception: true basic_init_is_generator: true basic_function_redefined: true basic_expression_not_assigned: true basic_exec_used: true basic_eval_used: true basic_empty_docstring: true basic_duplicate_key: true basic_duplicate_argument_name: true basic_dangerous_default_value: true basic_bad_reversed_sequence: true basic_assert_on_tuple: true basic_abstract_class_instantiated: true filter: paths: - picklable_itertools/* picklable-itertools-0.1.1/.travis.yml000066400000000000000000000014171256055564600176130ustar00rootroot00000000000000sudo: false cache: directories: - $TRAVIS_BUILD_DIR/mnist - $TRAVIS_BUILD_DIR/cifar10 - $TRAVIS_BUILD_DIR/binarized_mnist branches: only: - master language: python python: - "2.7" - "3.4" before_install: - wget -q http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh - chmod +x miniconda.sh - ./miniconda.sh -b - export PATH=$HOME/miniconda/bin:$PATH - conda update -q --yes conda install: # Install all Python dependencies - conda install -q --yes python=$TRAVIS_PYTHON_VERSION pip coverage six toolz nose numpy - pip install -q nose2[coverage-plugin] coveralls script: - rm -f .coverage - nosetests - coverage run --source=picklable_itertools -m nose.__main__ tests after_script: - coveralls --verbose picklable-itertools-0.1.1/LICENSE000066400000000000000000000021021256055564600164770ustar00rootroot00000000000000The MIT License (MIT) Copyright (c) 2015 Universite de Montreal Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. picklable-itertools-0.1.1/README.rst000066400000000000000000000066171256055564600172000ustar00rootroot00000000000000.. image:: https://img.shields.io/coveralls/mila-udem/picklable-itertools.svg :target: https://coveralls.io/r/mila-udem/picklable-itertools .. image:: https://travis-ci.org/mila-udem/picklable-itertools.svg?branch=master :target: https://travis-ci.org/mila-udem/picklable-itertools .. image:: https://img.shields.io/scrutinizer/g/mila-udem/picklable-itertools.svg :target: https://scrutinizer-ci.com/g/mila-udem/picklable-itertools/ .. image:: https://img.shields.io/badge/license-MIT-blue.svg :target: https://github.com/mila-udem/picklable-itertools/blob/master/LICENSE picklable-itertools =================== A reimplementation of the Python standard library's ``itertools``, in Python, using picklable iterator objects. Intended to be Python 2.7 and 3.4+ compatible. Also includes picklable, Python {2, 3}-compatible implementations of some related utilities, including some functions from the Toolz_ library, in ``picklable_itertools.extras``. .. _Toolz: http://toolz.readthedocs.org/en/latest/ Why? ---- * Because the standard library pickle module (nor the excellent dill_ package) can't serialize all of the ``itertools`` iterators, at least on Python 2 (at least some appear to be serializable on Python 3). * Because there are lots of instances where these things in ``itertools`` would simplify code, but can't be used because serializability must be maintained across both Python 2 and Python 3. The in-development framework Blocks_ is our first consumer. We'd like to be able to serialize the entire state of a long-running program for later resumption. We can't do this with non-picklable objects. .. _dill: https://github.com/uqfoundation/dill .. _blocks: https://github.com/bartvm/blocks Philosophy ---------- * *This should be a drop-in replacement.* Pretty self-explanatory. Test against the standard library ``itertools`` or builtin implementation to verify behaviour matches. Where Python 2 and Python 3 differ in their naming, (``filterfalse`` vs ``ifilterfalse``, ``zip_longest`` vs. ``izip_longest``) we provide both. We also provide names that were only available in the Python 2 incarnation of ``itertools`` (``ifilter``, ``izip``), also available under their built-in names in Python 3 (``filter``, ``zip``), for convenience. As new objects are added to the Python 3 ``itertools`` module, we intend to add them (``accumulate``, for example, appears only in Python 3, and a picklable implementation is contained in this package.) * *Handle built-in types gracefully if possible.* List iterators, etc. are not picklable on Python 2.x, so we provide an alternative implementation. File iterators are handled transparently as well. dict iterators and set iterators are currently *not* supported. ``picklable_itertools.xrange`` can be used as a drop-in replacement for Python 2 ``xrange``/Python 3 ``range``, with the benefit that the iterators produced by it will be picklable on both Python 2 and 3. * *Premature optimization is the root of all evil.* These things are implemented in Python, so speed is obviously not our primary concern. Several of the more advanced iterators are constructed by chaining simpler iterators together, which is not the most efficient thing to do but simplifies the code a lot. If it turns out that speed (or a shallower object graph) is necessary or desirable, these can always be reimplemented. Pull requests to this effect are welcome. picklable-itertools-0.1.1/picklable_itertools/000077500000000000000000000000001256055564600215315ustar00rootroot00000000000000picklable-itertools-0.1.1/picklable_itertools/__init__.py000066400000000000000000000030671256055564600236500ustar00rootroot00000000000000import os.path from pkg_resources import get_distribution, DistributionNotFound from .filter import ifilter, ifilterfalse, takewhile, dropwhile from .grouping import groupby from .iter_dispatch import ( iter_, ordered_sequence_iterator, file_iterator, range_iterator ) from .map_zip import imap, starmap, izip, izip_longest from .permutations import ( permutations, combinations, combinations_with_replacement ) from .product import product # noqa from .range import xrange # noqa from .simple import accumulate, chain, compress, count, cycle, repeat from .slicing import islice from .tee import tee # Python 3 equivalents. filter = ifilter filterfalse = ifilterfalse zip = izip zip_longest = izip_longest # Remove after bartvm/fuel has been updated to use this version. _iter = iter_ try: DIST = get_distribution('picklable_itertools') DIST_LOC = os.path.normcase(DIST.location) HERE = os.path.normcase(__file__) if not HERE.startswith(os.path.join(DIST_LOC, 'picklable_itertools')): raise DistributionNotFound except DistributionNotFound: __version__ = 'not installed' else: __version__ = DIST.version __all__ = ['ifilter', 'ifilterfalse', 'takewhile', 'dropwhile', 'groupby', '_iter', 'ordered_sequence_iterator', 'file_iterator', 'range_iterator', 'imap', 'starmap', 'izip', 'izip_longest', 'permutations', 'combinations', 'combinations_with_replacement', 'accumulate', 'chain', 'compress', 'count', 'cycle', 'repeat', 'islice', 'tee', 'filter', 'filterfalse', 'zip', 'zip_longest'] picklable-itertools-0.1.1/picklable_itertools/base.py000066400000000000000000000003311256055564600230120ustar00rootroot00000000000000from abc import ABCMeta, abstractmethod import six @six.add_metaclass(ABCMeta) class BaseItertool(six.Iterator): def __iter__(self): return self @abstractmethod def __next__(self): pass picklable-itertools-0.1.1/picklable_itertools/extras.py000066400000000000000000000102121256055564600234050ustar00rootroot00000000000000"""Related things that aren't part of the standard library's `itertools`. Currently home to picklable reimplementations of a few of the generators from Matthew Rocklin's Toolz package. Docstrings for `partition`, `partition_all`, and 'interleave' are lifted wholesale from the Toolz documentation . """ import six from .base import BaseItertool from .map_zip import imap, izip_longest from .iter_dispatch import iter_ class partition(BaseItertool): """Partition sequence into tuples of length n >>> list(partition(2, [1, 2, 3, 4])) [(1, 2), (3, 4)] If length of `seq` is not evenly divisible by `n`, the final tuple is dropped if `pad` is not specified, or filled to length `n` by `pad`: >>> list(partition(2, [1, 2, 3, 4, 5])) [(1, 2), (3, 4)] >>> list(partition(2, [1, 2, 3, 4, 5], pad=None)) [(1, 2), (3, 4), (5, None)] See Also: partition_all """ _NO_PAD = '__no_pad__' def __init__(self, n, seq, pad=_NO_PAD): self._n = n self._partition_all = partition_all(n, seq) self._pad = pad def __next__(self): items = next(self._partition_all) if len(items) < self._n: if self._pad != self._NO_PAD: items += (self._pad,) * (self._n - len(items)) else: raise StopIteration return items class partition_all(BaseItertool): """Partition all elements of sequence into tuples of length at most n The final tuple may be shorter to accommodate extra elements. >>> list(partition_all(2, [1, 2, 3, 4])) [(1, 2), (3, 4)] >>> list(partition_all(2, [1, 2, 3, 4, 5])) [(1, 2), (3, 4), (5,)] See Also: partition """ def __init__(self, n, seq): self._n = n self._seq = iter_(seq) def __next__(self): items = [] try: for _ in six.moves.xrange(self._n): items.append(next(self._seq)) except StopIteration: pass if len(items) == 0: raise StopIteration return tuple(items) class NoMoreItems(object): """Sentinel value for `equizip`. Do not use for any other purpose.""" pass class IterableLengthMismatch(ValueError): """Raised if an iterator passed to `equizip` is shorter than others.""" pass class equizip(izip_longest): """Like `izip_longest` but ensures the sequences are the same length. Raises :class:`IterableLengthMismatch` if one of the iterators terminates prematurely. """ def __init__(self, *args): super(equizip, self).__init__(*args, fillvalue=NoMoreItems) def __next__(self): next_item = super(equizip, self).__next__() if any(value is NoMoreItems for value in next_item): raise IterableLengthMismatch return next_item class interleave(BaseItertool): """Interleave a sequence of sequences >>> list(interleave([[1, 2], [3, 4]])) [1, 3, 2, 4] >>> ''.join(interleave(('ABC', 'XY'))) 'AXBYC' Both the individual sequences and the sequence of sequences may be infinite Returns a lazy iterator """ def __init__(self, iterables, pass_exceptions=()): self._iters = imap(iter, iterables) self._more = [] self._pass_exceptions = pass_exceptions def __next__(self): try: it = next(self._iters) except StopIteration: if len(self._more) == 0: raise else: self._iters = imap(iter, self._more) self._more = [] return next(self) try: result = next(it) self._more.append(it) return result except (StopIteration,) + tuple(self._pass_exceptions): return next(self) def roundrobin(*iterables): """Grab items from a collection of iterators in a round robin fashion until all are exhausted. >>> list(roundrobin('ABC', 'DEF', 'GH')) ['A', 'D', 'G', 'B', 'E', 'H', 'C', 'F'] >>> list(roundrobin(xrange(2), xrange(5, 10), xrange(10, 12))) [0, 5, 10, 1, 6, 11, 7, 8, 9] """ return interleave(iterables) picklable-itertools-0.1.1/picklable_itertools/filter.py000066400000000000000000000037221256055564600233740ustar00rootroot00000000000000from abc import ABCMeta, abstractmethod import six from .iter_dispatch import iter_ from .base import BaseItertool @six.add_metaclass(ABCMeta) class BaseFilter(BaseItertool): def __init__(self, pred, seq): self._predicate = pred self._iter = iter_(seq) @abstractmethod def __next__(self): pass class ifilter(BaseFilter): """ifilter(function or None, iterable) --> ifilter object Return an iterator yielding those items of iterable for which function(item) is true. If function is None, return the items that are true. """ def _keep(self, value): predicate = bool if self._predicate is None else self._predicate return predicate(value) def __next__(self): val = next(self._iter) while not self._keep(val): val = next(self._iter) return val class ifilterfalse(ifilter): """ifilterfalse(function or None, sequence) --> ifilterfalse object Return those items of sequence for which function(item) is false. If function is None, return the items that are false. """ def _keep(self, value): return not super(ifilterfalse, self)._keep(value) class takewhile(BaseFilter): """takewhile(predicate, iterable) --> takewhile object Return successive entries from an iterable as long as the predicate evaluates to true for each entry. """ def __next__(self): value = next(self._iter) if not self._predicate(value): raise StopIteration return value class dropwhile(takewhile): """dropwhile(predicate, iterable) --> dropwhile object Drop items from the iterable while predicate(item) is true. Afterwards, return every element until the iterable is exhausted. """ def __next__(self): value = next(self._iter) while not getattr(self, '_started', False) and self._predicate(value): value = next(self._iter) self._started = True return value picklable-itertools-0.1.1/picklable_itertools/grouping.py000066400000000000000000000041611256055564600237370ustar00rootroot00000000000000from .base import BaseItertool from .iter_dispatch import iter_ class _grouper(BaseItertool): def __init__(self, value, iterator, groupby_obj): self._value = value self._groupby = groupby_obj self._key = self._groupby.key(self._value) self._initialized = False self._iterator = iterator self.stream_ended = False self._done = False def __next__(self): if not self._initialized: self._initialized = True return self._value else: if self._done: raise StopIteration try: value = next(self._iterator) except StopIteration: self.stream_ended = True self._done = True raise if self._groupby.key(value) != self._key: self.terminal_value = value self._done = True raise StopIteration return value class groupby(BaseItertool): """groupby(iterable[, keyfunc]) -> create an iterator which returns (key, sub-iterator) grouped by each value of key(value). """ def __init__(self, iterable, key=None): self._keyfunc = key self._iterator = iter_(iterable) self._current_key = self._initial_key = object() def key(self, value): if self._keyfunc is None: return value else: return self._keyfunc(value) def __next__(self): if not hasattr(self, '_current_grouper'): value = next(self._iterator) self._current_grouper = _grouper(value, self._iterator, self) return self.key(value), self._current_grouper else: while True: try: next(self._current_grouper) except StopIteration: break if self._current_grouper.stream_ended: raise StopIteration value = self._current_grouper.terminal_value self._current_grouper = _grouper(value, self._iterator, self) return self.key(value), self._current_grouper picklable-itertools-0.1.1/picklable_itertools/iter_dispatch.py000066400000000000000000000050651256055564600247330ustar00rootroot00000000000000import io import six try: import numpy NUMPY_AVAILABLE = True except ImportError: numpy = None NUMPY_AVAILABLE = False from .base import BaseItertool def iter_(obj): """A custom replacement for iter(), dispatching a few custom picklable iterators for known types. """ if six.PY2: file_types = file, # noqa if six.PY3: file_types = io.IOBase, dict_items = {}.items().__class__ dict_values = {}.values().__class__ dict_keys = {}.keys().__class__ dict_view = (dict_items, dict_values, dict_keys) if isinstance(obj, dict): return ordered_sequence_iterator(list(obj.keys())) if isinstance(obj, file_types): return file_iterator(obj) if six.PY2: if isinstance(obj, (list, tuple)): return ordered_sequence_iterator(obj) if isinstance(obj, xrange): # noqa return range_iterator(obj) if NUMPY_AVAILABLE and isinstance(obj, numpy.ndarray): return ordered_sequence_iterator(obj) if six.PY3 and isinstance(obj, dict_view): return ordered_sequence_iterator(list(obj)) return iter(obj) class range_iterator(BaseItertool): """A picklable range iterator for Python 2.""" def __init__(self, xrange_): self._start, self._stop, self._step = xrange_.__reduce__()[1] self._n = self._start def __next__(self): if (self._step > 0 and self._n < self._stop or self._step < 0 and self._n > self._stop): value = self._n self._n += self._step return value else: raise StopIteration class file_iterator(BaseItertool): """A picklable file iterator.""" def __init__(self, f): self._f = f def __next__(self): line = self._f.readline() if not line: raise StopIteration return line def __getstate__(self): name, pos, mode = self._f.name, self._f.tell(), self._f.mode return name, pos, mode def __setstate__(self, state): name, pos, mode = state self._f = open(name, mode=mode) self._f.seek(pos) class ordered_sequence_iterator(BaseItertool): """A picklable replacement for list and tuple iterators.""" def __init__(self, sequence): self._sequence = sequence self._position = 0 def __next__(self): if self._position < len(self._sequence): value = self._sequence[self._position] self._position += 1 return value else: raise StopIteration picklable-itertools-0.1.1/picklable_itertools/map_zip.py000066400000000000000000000054621256055564600235510ustar00rootroot00000000000000from .base import BaseItertool from .iter_dispatch import iter_ class imap(BaseItertool): """imap(func, *iterables) --> imap object Make an iterator that computes the function using arguments from each of the iterables. Stops when the shortest iterable is exhausted. """ def __init__(self, function, *iterables): self._function = function self._iterables = tuple(iter_(it) for it in iterables) def _run(self, args): return self._function(*args) def __next__(self): args = tuple([next(it) for it in self._iterables]) if self._function is None: return args else: return self._run(args) class starmap(imap): """starmap(function, sequence) --> starmap object Return an iterator whose values are returned from the function evaluated with a argument tuple taken from the given sequence. """ def __init__(self, function, iterable): self._iterables = (iter_(iterable),) self._function = function def _run(self, args): return self._function(*args[0]) def izip(*iterables): """zip(iter1 [,iter2 [...]]) --> zip object Return a zip object whose .__next__() method returns a tuple where the i-th element comes from the i-th iterable argument. The .__next__() method continues until the shortest iterable in the argument sequence is exhausted and then it raises StopIteration. """ return imap(None, *iterables) class izip_longest(BaseItertool): """zip_longest(iter1 [,iter2 [...]], [fillvalue=None]) --> zip_longest object Return an zip_longest object whose .__next__() method returns a tuple where the i-th element comes from the i-th iterable argument. The .__next__() method continues until the longest iterable in the argument sequence is exhausted and then it raises StopIteration. When the shorter iterables are exhausted, the fillvalue is substituted in their place. The fillvalue defaults to None or can be specified by a keyword argument. """ def __init__(self, *iterables, **kwargs): if 'fillvalue' in kwargs: self._fillvalue = kwargs['fillvalue'] del kwargs['fillvalue'] else: self._fillvalue = None if len(kwargs) > 0: raise ValueError("Unrecognized keyword arguments: {}".format( ", ".join(kwargs))) self._iterables = tuple(iter_(it) for it in iterables) def __next__(self): found_any = False result = [] for it in self._iterables: try: result.append(next(it)) found_any = True except StopIteration: result.append(self._fillvalue) if found_any: return tuple(result) else: raise StopIteration picklable-itertools-0.1.1/picklable_itertools/permutations.py000066400000000000000000000037611256055564600246440ustar00rootroot00000000000000from abc import ABCMeta, abstractmethod import six from .product import product from .base import BaseItertool @six.add_metaclass(ABCMeta) class IndexBased(BaseItertool): def __init__(self, iterable, r=None): self._pool = tuple(iterable) self._r = r if r is not None else len(self._pool) self._iter = self._construct_iter() def _construct_iter(self): return product(range(len(self._pool)), repeat=self._r) @abstractmethod def _valid_indices(self, indices): pass def __next__(self): indices = next(self._iter) while not self._valid_indices(indices): indices = next(self._iter) return tuple(self._pool[i] for i in indices) class permutations(IndexBased): """permutations(iterable[, r]) --> permutations object Return successive r-length permutations of elements in the iterable. permutations(range(3), 2) --> (0,1), (0,2), (1,0), (1,2), (2,0), (2,1) """ def _valid_indices(self, indices): return len(set(indices)) == self._r @six.add_metaclass(ABCMeta) class AbstractCombinations(IndexBased): def __init__(self, iterable, r): super(AbstractCombinations, self).__init__(iterable, r) def _valid_indices(self, indices): return sorted(indices) == list(indices) class combinations_with_replacement(AbstractCombinations): """combinations_with_replacement(iterable, r) --> combinations_with_replacement object Return successive r-length combinations of elements in the iterable allowing individual elements to have successive repeats. combinations_with_replacement('ABC', 2) --> AA AB AC BB BC CC """ pass class combinations(AbstractCombinations): """combinations(iterable, r) --> combinations object Return successive r-length combinations of elements in the iterable. combinations(range(4), 3) --> (0,1,2), (0,1,3), (0,2,3), (1,2,3) """ def _construct_iter(self): return permutations(range(len(self._pool)), self._r) picklable-itertools-0.1.1/picklable_itertools/product.py000066400000000000000000000101201256055564600235550ustar00rootroot00000000000000import collections from .base import BaseItertool from .iter_dispatch import iter_ from .tee import tee _zip = zip class product(BaseItertool): """product(*iterables, repeat=1) --> product object Cartesian product of input iterables. Equivalent to nested for-loops. For example, product(A, B) returns the same as: ((x,y) for x in A for y in B). The leftmost iterators are in the outermost for-loop, so the output tuples cycle in a manner similar to an odometer (with the rightmost element changing on every iteration). To compute the product of an iterable with itself, specify the number of repetitions with the optional repeat keyword argument. For example, product(A, repeat=4) means the same as product(A, A, A, A). product('ab', range(3)) --> ('a',0) ('a',1) ('a',2) ('b',0) ('b',1) ('b',2) product((0,1), (0,1), (0,1)) --> (0,0,0) (0,0,1) (0,1,0) (0,1,1) (1,0,0) ... """ def __init__(self, *args, **kwargs): if 'repeat' in kwargs: repeat = kwargs['repeat'] del kwargs['repeat'] else: repeat = None if len(kwargs) > 0: raise ValueError("Unrecognized keyword arguments: {}".format( ", ".join(kwargs))) if repeat is not None: self._iterables = sum(_zip(*(tee(it, repeat) for it in args)), ()) else: self._iterables = tuple(iter_(it) for it in args) self._contents = tuple([] for it in self._iterables) self._exhausted = [False for it in self._iterables] self._position = [-1 for it in self._iterables] self._initialized = False def __next__(self): # Welcome to the spaghetti factory. def _next(i): # Advance the i'th iterator, wrapping if necessary. # Returns the next value as well as a "carry bit" that # tells us we've wrapped, i.e. to advance iterator i - 1 as well. flip = False # Handle the case where iteratior i still has stuff in it. if not self._exhausted[i]: try: value = next(self._iterables[i]) self._contents[i].append(value) self._position[i] += 1 except StopIteration: # If contents is empty, the iterator was empty. if len(self._contents[i]) == 0: raise StopIteration self._exhausted[i] = True self._position[i] = -1 # The recursive call increments. return _next(i) else: self._position[i] += 1 self._position[i] %= len(self._contents[i]) value = self._contents[i][self._position[i]] # If we've wrapped, return True for the carry bit. if self._position[i] == 0: flip = True return value, flip # deque is already imported, could append to a list and reverse it. result = collections.deque() i = len(self._iterables) - 1 if not self._initialized: # On the very first draw we need to draw one from every iterator. while i >= 0: result.appendleft(_next(i)[0]) i -= 1 self._initialized = True else: # Always advance the least-significant position iterator. flip = True # Keep drawing from lower-index iterators until the carry bit # is unset. while flip and i >= 0: value, flip = _next(i) i -= 1 result.appendleft(value) # If the carry bit is still set after breaking out of the loop # above, it means the most-significant iterator has wrapped, # and we're done. if flip: raise StopIteration # Read off any other unchanged values from lower-index iterators. while i >= 0: result.appendleft(self._contents[i][self._position[i]]) i -= 1 return tuple(result) picklable-itertools-0.1.1/picklable_itertools/range.py000066400000000000000000000053051256055564600232020ustar00rootroot00000000000000import six.moves from numbers import Integral from .iter_dispatch import range_iterator __all__ = ['xrange'] def _check_integral(value): if not isinstance(value, Integral): raise TypeError("'{}' object cannot be interpreted " "as an integer".format(type(value).__name__)) class xrange(object): """A replacement for Python 3 `range()` (and Python 2 `xrange()`) that yields picklable iterators when iterated upon. """ __slots__ = ['_start', '_stop', '_step'] def __init__(self, *args): self._start = 0 self._step = 1 if len(args) == 0: raise TypeError("{} expected 1 arguments, got 0".format( self.__class__.__name__)) elif len(args) == 1: self._stop = args[0] self._start = 0 elif len(args) >= 2: self._start = args[0] self._stop = args[1] if len(args) == 3: self._step = args[2] if len(args) > 3: raise TypeError("{} expected at most 3 arguments, got {}".format( self.__class__.__name__, len(args))) _check_integral(self._start) _check_integral(self._stop) _check_integral(self._step) @property def start(self): return self._start @property def stop(self): return self._stop @property def step(self): return self._step def count(self, i): """rangeobject.count(value) -> integer -- return number of occurrences of value """ if self._stop > self._start and self._step > 0: return int(self._start <= i < self._stop and (i - self._start) % self._step == 0) elif self._stop < self._start and self._step < 0: return int(self._start >= i > self._stop and (i - self._start) % self._step == 0) else: return False def index(self, i): """xrangeobject.index(value, [start, [stop]]) -> integer -- return index of value. Raise ValueError if the value is not present. """ if self.count(i) == 0: raise ValueError("{} is not in range".format(i)) return (i - self._start) // self._step def __len__(self): return len(six.moves.xrange(self._start, self._stop, self._step)) def __reduce__(self): return (self.__class__, (self.start, self.stop, self.step)) def __iter__(self): return range_iterator(self) def __repr__(self): return (__name__.split('.')[0] + '.' + self.__class__.__name__ + (str((self.start, self.stop)) if self.step == 1 else str((self.start, self.stop, self.step)))) picklable-itertools-0.1.1/picklable_itertools/simple.py000066400000000000000000000100151256055564600233710ustar00rootroot00000000000000import collections from .base import BaseItertool from .iter_dispatch import iter_ class repeat(BaseItertool): """ repeat(object [,times]) -> create an iterator which returns the object for the specified number of times. If not specified, returns the object endlessly. """ def __init__(self, obj, times=None): self._obj = obj self._times = times self._times_called = 0 def __next__(self): if self._times is None: return self._obj else: if self._times > self._times_called: self._times_called += 1 return self._obj else: raise StopIteration class chain(BaseItertool): """ chain(*iterables) --> chain object Return a chain object whose .__next__() method returns elements from the first iterable until it is exhausted, then elements from the next iterable, until all of the iterables are exhausted. """ def __init__(self, *iterables): self._iterables = iter_(iterables) self._current = repeat(None, 0) def __next__(self): try: return next(self._current) except StopIteration: self._current = iter_(next(self._iterables)) return next(self) @classmethod def from_iterable(cls, iterable): obj = cls() obj._iterables = iter_(iterable) return obj class compress(BaseItertool): """compress(data, selectors) --> iterator over selected data Return data elements corresponding to true selector elements. Forms a shorter iterator from selected data elements using the selectors to choose the data elements. """ def __init__(self, data, selectors): self._data = iter_(data) self._selectors = iter_(selectors) def __next__(self): # We terminate on the shortest input sequence, so leave # StopIteration uncaught here. data = next(self._data) selector = next(self._selectors) while not bool(selector): data = next(self._data) selector = next(self._selectors) return data class count(BaseItertool): """count(start=0, step=1) --> count object Return a count object whose .__next__() method returns consecutive values. """ def __init__(self, start=0, step=1): self._n = start self._step = step def __next__(self): n = self._n self._n += self._step return n class cycle(BaseItertool): """cycle(iterable) --> cycle object Return elements from the iterable until it is exhausted. Then repeat the sequence indefinitely. """ def __init__(self, iterable): self._iterable = iter_(iterable) self._exhausted = False self._elements = collections.deque() def __next__(self): if not self._exhausted: try: value = next(self._iterable) except StopIteration: self._exhausted = True return next(self) self._elements.append(value) else: if len(self._elements) == 0: raise StopIteration value = self._elements.popleft() self._elements.append(value) return value class accumulate(BaseItertool): """accumulate(iterable[, func]) --> accumulate object Return series of accumulated sums (or other binary function results). """ def __init__(self, iterable, func=None): self._iter = iter_(iterable) self._func = func self._initialized = False self._accumulated = None def _combine(self, value): if self._func is not None: return self._func(self._accumulated, value) else: return self._accumulated + value def __next__(self): value = next(self._iter) if not self._initialized: self._accumulated = value self._initialized = True else: self._accumulated = self._combine(value) return self._accumulated picklable-itertools-0.1.1/picklable_itertools/slicing.py000066400000000000000000000030521256055564600235330ustar00rootroot00000000000000import sys from .base import BaseItertool from .iter_dispatch import iter_ class islice(BaseItertool): """islice(iterable, stop) --> islice object islice(iterable, start, stop[, step]) --> islice object Return an iterator whose next() method returns selected values from an iterable. If start is specified, will skip all preceding elements; otherwise, start defaults to zero. Step defaults to one. If specified as another value, step determines how many values are skipped between successive calls. Works like a slice() on a list but returns an iterator. """ def __init__(self, iterable, start, stop=None, step=1): if stop is None: start, stop = 0, start if (not 0 <= start <= sys.maxsize or not 0 <= stop <= sys.maxsize or not 0 <= step <= sys.maxsize): raise ValueError("Indices for islice() must be None or an " "integer: 0 <= x <= maxint.") self._iterable = iter_(iterable) i = 0 while i < start: try: next(self._iterable) i += 1 except StopIteration: break self._stop = stop - start self._step = step self._n = 0 def __next__(self): while self._n % self._step and self._n < self._stop: next(self._iterable) self._n += 1 if self._n == self._stop: raise StopIteration value = next(self._iterable) self._n += 1 return value picklable-itertools-0.1.1/picklable_itertools/tee.py000066400000000000000000000024601256055564600226620ustar00rootroot00000000000000"""Support code for implementing `tee`.""" import collections import six from picklable_itertools import iter_ class tee_iterator(six.Iterator): """An iterator that works in conjunction with a `tee_manager`.""" def __init__(self, deque, manager): self._deque = deque self._manager = manager def __iter__(self): return self def __next__(self): if len(self._deque) > 0: return self._deque.popleft() else: self._manager.advance() assert len(self._deque) > 0 return next(self) class tee_manager(object): """An object that manages a base iterator and publishes results to one or more client `tee_iterators`. """ def __init__(self, iterable, n=2): self._iterable = iter_(iterable) self._deques = tuple(collections.deque() for _ in range(n)) def iterators(self): return tuple(tee_iterator(deque, self) for deque in self._deques) def advance(self): """Advance the base iterator, publish to constituent iterators.""" elem = next(self._iterable) for deque in self._deques: deque.append(elem) def tee(iterable, n=2): """tee(iterable, n=2) --> tuple of n independent iterators.""" return tee_manager(iter_(iterable), n=n).iterators() picklable-itertools-0.1.1/setup.py000066400000000000000000000022361256055564600172140ustar00rootroot00000000000000"""Installation script.""" from os import path from setuptools import find_packages, setup HERE = path.abspath(path.dirname(__file__)) with open(path.join(HERE, 'README.rst')) as f: LONG_DESCRIPTION = f.read().strip() setup( name='picklable-itertools', version='0.1.1', # PEP 440 compliant description='itertools. But picklable. Even on Python 2.', long_description=LONG_DESCRIPTION, url='https://github.com/mila-udem/picklable-itertools', author='David Warde-Farley', author_email='d.warde.farley@gmail.com', license='MIT', # See https://pypi.python.org/pypi?%3Aaction=list_classifiers classifiers=[ 'Development Status :: 3 - Alpha', 'Intended Audience :: Developers', 'Topic :: Utilities', 'License :: OSI Approved :: MIT License', 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.4', ], keywords='pickle serialize pickling itertools iterable iteration', packages=find_packages(exclude=['tests']), install_requires=['six'], zip_safe=True ) picklable-itertools-0.1.1/tests/000077500000000000000000000000001256055564600166415ustar00rootroot00000000000000picklable-itertools-0.1.1/tests/__init__.py000066400000000000000000000624661256055564600207700ustar00rootroot00000000000000from functools import partial import itertools from operator import add, sub, pos, gt, lt, le import random import tempfile import six from six.moves import cPickle from six.moves import xrange from nose.tools import assert_raises, assert_equal from unittest import SkipTest from picklable_itertools import ( repeat, chain, compress, count, cycle, ifilter, ifilterfalse, imap, izip, file_iterator, ordered_sequence_iterator, izip_longest, iter_, islice, range_iterator, product, tee, accumulate, takewhile, dropwhile, starmap, groupby, permutations, combinations, combinations_with_replacement, xrange as _xrange ) from picklable_itertools.iter_dispatch import numpy, NUMPY_AVAILABLE _map = map if six.PY3 else itertools.imap _zip = zip if six.PY3 else itertools.izip _zip_longest = itertools.zip_longest if six.PY3 else itertools.izip_longest _filter = filter if six.PY3 else itertools.ifilter _filterfalse = itertools.filterfalse if six.PY3 else itertools.ifilterfalse _islice = itertools.islice def _identity(x): return x def safe_assert_equal(expected_val, actual_val): if NUMPY_AVAILABLE and (isinstance(expected_val, numpy.ndarray) or isinstance(actual_val, numpy.ndarray)): assert (expected_val == actual_val).all() else: assert expected_val == actual_val def verify_same(picklable_version, reference_version, n, *args, **kwargs): """Take a reference version from itertools, verify the same operation in our version. """ try: expected = reference_version(*args, **kwargs) except Exception as e: assert_raises(e.__class__, picklable_version, *args, **kwargs) return actual = picklable_version(*args, **kwargs) done = 0 assert n is None or isinstance(n, int) while done != n: try: expected_val = next(expected) except StopIteration: check_stops(actual) break try: actual_val = next(actual) except StopIteration: assert False, "prematurely exhausted; expected {}".format( str(expected_val)) safe_assert_equal(expected_val, actual_val) done += 1 def verify_pickle(picklable_version, reference_version, n, m, *args, **kwargs): """Take n steps, pickle at m < n, and make sure it continues the same.""" expected = reference_version(*args, **kwargs) actual = picklable_version(*args, **kwargs) done = 0 if not m < n: raise ValueError("Test only makes sense with m < n") while done != n: expected_val = next(expected) actual_val = next(actual) safe_assert_equal(expected_val, actual_val) if done == m: actual = cPickle.loads(cPickle.dumps(actual)) done += 1 def conditional_run(condition, f, *args, **kwargs): if condition: f(*args, **kwargs) else: raise SkipTest def check_stops(it): """Verify that an exhausted iterator yields StopIteration.""" try: val = next(it) except StopIteration: return assert False, "expected exhausted iterator; got {}".format(str(val)) def test_ordered_sequence_iterator(): yield verify_same, ordered_sequence_iterator, iter, None, [] yield verify_same, ordered_sequence_iterator, iter, None, () yield verify_same, ordered_sequence_iterator, iter, None, [5, 2] yield verify_same, ordered_sequence_iterator, iter, None, ("D", "X", "J") yield verify_pickle, ordered_sequence_iterator, iter, 4, 3, [2, 9, 3, 4] yield verify_pickle, ordered_sequence_iterator, iter, 3, 2, ['a', 'c', 'b'] array = numpy.array if NUMPY_AVAILABLE else list numpy_pickle_test = partial(conditional_run, NUMPY_AVAILABLE, verify_pickle) numpy_same_test = partial(conditional_run, NUMPY_AVAILABLE, verify_same) yield (numpy_same_test, ordered_sequence_iterator, iter, None, array([4, 3, 9])) yield (numpy_same_test, ordered_sequence_iterator, iter, None, array([[4, 3, 9], [2, 9, 6]])) yield (numpy_pickle_test, ordered_sequence_iterator, iter, 4, 3, array([2, 9, 3, 4])) yield (numpy_pickle_test, ordered_sequence_iterator, iter, 3, 2, array([[2, 1], [2, 9], [9, 4], [3, 9]])) # Make sure the range iterator is actually getting dispatched by iter_. yield (numpy_pickle_test, iter_, iter, 4, 3, array([2, 9, 3, 4])) yield (numpy_pickle_test, iter_, iter, 3, 2, array([[2, 1], [2, 9], [9, 4], [3, 9]])) def test_dict_iterator(): d = {'a': 'b', 1: 2} assert list(iter_(d)) == list(iter(d)) assert list(iter_(d.items())) == list(iter(d.items())) assert list(iter_(d.keys())) == list(iter(d.keys())) assert list(iter_(d.values())) == list(iter(d.values())) yield verify_pickle, iter_, iter, 2, 1, d yield verify_pickle, iter_, iter, 2, 1, d.items() yield verify_pickle, iter_, iter, 2, 1, d.values() yield verify_pickle, iter_, iter, 2, 1, d.keys() def test_range_iterator(): yield verify_same, range_iterator, iter, None, xrange(5) yield verify_same, range_iterator, iter, None, xrange(2, 5) yield verify_same, range_iterator, iter, None, xrange(5, 2) yield verify_same, range_iterator, iter, None, xrange(0) yield verify_same, range_iterator, iter, None, xrange(-3) yield verify_same, range_iterator, iter, None, xrange(-5, -3) yield verify_same, range_iterator, iter, None, xrange(-5, -1, -2) yield verify_same, range_iterator, iter, None, xrange(-5, -1, 2) yield verify_same, range_iterator, iter, None, xrange(2, 5, 7) yield verify_same, range_iterator, iter, None, xrange(2, 7, -1) yield verify_same, range_iterator, iter, None, xrange(5, 3, -1) yield verify_same, range_iterator, iter, None, xrange(5, 4, -2) yield verify_pickle, range_iterator, iter, 5, 2, xrange(5) yield verify_pickle, range_iterator, iter, 3, 1, xrange(2, 5) yield verify_pickle, range_iterator, iter, 2, 1, xrange(-5, -1, 2) yield verify_pickle, range_iterator, iter, 2, 1, xrange(5, 3, -1) def _create_test_file(): f = tempfile.NamedTemporaryFile(mode='w') f.write("\n".join(map(str, range(4)))) f.flush() return f def test_file_iterator(): f = _create_test_file() assert list(file_iterator(open(f.name))) == list(iter(open(f.name))) def test_file_iterator_pickling(): f = _create_test_file() it = iter_(open(f.name)) last = [next(it) for _ in range(2)][-1] first = next(cPickle.loads(cPickle.dumps(it))) assert int(first) == int(last) + 1 def test_repeat(): yield verify_same, repeat, itertools.repeat, None, 5, 0 yield verify_same, repeat, itertools.repeat, None, 'abc', 5 yield verify_pickle, repeat, itertools.repeat, 5, 0, 'abc', 5 yield verify_pickle, repeat, itertools.repeat, 5, 3, 'abc', 5 yield verify_same, repeat, itertools.repeat, None, 'def', 3 yield verify_pickle, repeat, itertools.repeat, 3, 0, 'def', 3 yield verify_pickle, repeat, itertools.repeat, 3, 1, 'def', 3 def test_chain(): yield verify_same, chain, itertools.chain, None, [5, 4], [3], [9, 10] yield verify_pickle, chain, itertools.chain, 5, 2, [5, 4], [3], [9, 10] yield verify_pickle, chain, itertools.chain, 5, 4, [5, 4], [3], [9, 10] yield verify_pickle, chain, itertools.chain, 5, 0, [5, 4], [3], [9, 10] yield verify_same, chain, itertools.chain, None, [3, 1], [], ['x', 'y'] yield verify_pickle, chain, itertools.chain, 3, 1, [3, 1], [], ['x', 'y'] yield verify_pickle, chain, itertools.chain, 3, 0, [3, 1], [], ['x', 'y'] yield verify_same, chain, itertools.chain, None, [], [], [] yield verify_same, chain, itertools.chain, None def test_chain_from_iterable(): yield (verify_same, chain.from_iterable, itertools.chain.from_iterable, None, [xrange(i) for i in xrange(3)]) yield (verify_pickle, chain.from_iterable, itertools.chain.from_iterable, 5, 2, [xrange(i) for i in xrange(4)]) yield (verify_same, chain.from_iterable, itertools.chain.from_iterable, None, [[1], [], [2]]) yield (verify_same, chain.from_iterable, itertools.chain.from_iterable, None, [[], [], []]) yield (verify_same, chain.from_iterable, itertools.chain.from_iterable, None, [[], [None], []]) def test_compress(): yield verify_same, compress, itertools.compress, None, [1, 2, 3], [1, 2, 3] yield verify_same, compress, itertools.compress, None, [1, 2, 3], [1, 0, 0] yield verify_same, compress, itertools.compress, None, [1, 2, 3], [1, 0] yield verify_same, compress, itertools.compress, None, [1, 2], [1, 0, 1] yield verify_same, compress, itertools.compress, None, [1, 2], [0, 0] yield verify_same, compress, itertools.compress, None, [1, 2], [0] yield verify_same, compress, itertools.compress, None, [1, 2], [0, 0, 0] yield (verify_pickle, compress, itertools.compress, 3, 1, [1, 2, 3], [1, 2, 3]) yield (verify_pickle, compress, itertools.compress, 3, 0, [1, 2, 3], [1, 2, 3]) yield (verify_pickle, compress, itertools.compress, 1, 0, [1, 2, 3], [1, 0, 0]) yield (verify_pickle, compress, itertools.compress, 1, 0, [1, 2, 3], [1, 0]) yield (verify_pickle, compress, itertools.compress, 1, 0, [1, 2], [1, 0, 1]) def test_count(): yield verify_same, count, itertools.count, 6 yield verify_same, count, itertools.count, 20, 2 yield verify_same, count, itertools.count, 10, 5, 9 yield verify_same, count, itertools.count, 30, 3, 10 yield verify_pickle, count, itertools.count, 6, 1 yield verify_pickle, count, itertools.count, 20, 5, 2 yield verify_pickle, count, itertools.count, 20, 0, 2 yield verify_pickle, count, itertools.count, 10, 9, 5, 9 yield verify_pickle, count, itertools.count, 10, 6, 5, 9 yield verify_pickle, count, itertools.count, 30, 7, 3, 10 def test_cycle(): yield verify_same, cycle, itertools.cycle, 40, [4, 9, 10] yield verify_same, cycle, itertools.cycle, 10, [4, 9, 20, 10] yield verify_same, cycle, itertools.cycle, 20, [4, 9, 30, 10, 9] yield verify_same, cycle, itertools.cycle, 60, [8, 4, 5, 4, 9, 10] yield verify_pickle, cycle, itertools.cycle, 40, 20, [4, 9, 10] yield verify_pickle, cycle, itertools.cycle, 10, 9, [4, 9, 20, 10] yield verify_pickle, cycle, itertools.cycle, 20, 1, [4, 9, 30, 10, 9] yield verify_pickle, cycle, itertools.cycle, 60, 55, [8, 4, 5, 4, 9, 10] yield verify_pickle, cycle, itertools.cycle, 60, 0, [8, 4, 5, 4, 9, 10] yield verify_same, cycle, itertools.cycle, None, [] def test_imap(): yield verify_same, imap, _map, None, partial(add, 2), [3, 4, 5] yield verify_same, imap, _map, None, add, [3, 4], [9, 2] yield verify_same, imap, _map, None, add, [3], [9, 2] yield verify_same, imap, _map, None, add, [3], [9, 2], [] yield verify_pickle, imap, _map, 3, 1, partial(add, 2), [3, 4, 5] yield verify_pickle, imap, _map, 3, 0, partial(add, 2), [3, 4, 5] yield verify_pickle, imap, _map, 2, 1, add, [3, 4], [9, 2] yield verify_pickle, imap, _map, 2, 0, add, [3, 4], [9, 2] yield verify_pickle, imap, _map, 1, 0, add, [3], [9, 2] def test_izip(): yield verify_same, izip, _zip, None, [3, 4, 5] yield verify_same, izip, _zip, None, [3, 4], [9, 2] yield verify_same, izip, _zip, None, [3], [9, 2] yield verify_same, izip, _zip, None, [3], [9, 2], [] yield verify_pickle, izip, _zip, 3, 2, [3, 4, 5] yield verify_pickle, izip, _zip, 3, 1, [3, 4, 5] yield verify_pickle, izip, _zip, 2, 1, [3, 4], [9, 2] yield verify_pickle, izip, _zip, 2, 0, [3, 4], [9, 2] yield verify_pickle, izip, _zip, 1, 0, [3], [9, 2] def test_ifilter(): yield verify_same, ifilter, _filter, None, partial(le, 4), [3, 4, 5] yield verify_same, ifilter, _filter, None, partial(gt, 3), [] yield verify_same, ifilter, _filter, None, partial(le, 6), [3, 4, 5] yield verify_same, ifilter, _filter, None, None, [0, 3, 0, 0, 1] yield verify_pickle, ifilter, _filter, 2, 0, partial(le, 4), [3, 4, 5] yield verify_pickle, ifilter, _filter, 2, 1, partial(le, 4), [3, 4, 5] yield verify_pickle, ifilter, _filter, 2, 1, None, [0, 3, 0, 0, 1] yield verify_pickle, ifilter, _filter, 2, 0, None, [0, 3, 0, 0, 1] def test_ifilterfalse(): yield (verify_same, ifilterfalse, _filterfalse, None, partial(le, 4), [3, 4, 5]) yield (verify_same, ifilterfalse, _filterfalse, None, partial(le, 6), [3, 4, 5]) yield (verify_same, ifilterfalse, _filterfalse, None, partial(gt, 3), []) yield (verify_same, ifilterfalse, _filterfalse, None, None, [0, 3, 0, 0, 1]) yield (verify_pickle, ifilterfalse, _filterfalse, 1, 0, partial(le, 4), [3, 4, 5]) yield (verify_pickle, ifilterfalse, _filterfalse, 3, 2, partial(le, 6), [3, 4, 5]) yield (verify_pickle, ifilterfalse, _filterfalse, 3, 0, partial(le, 6), [3, 4, 5]) yield (verify_pickle, ifilterfalse, _filterfalse, 3, 0, None, [0, 3, 0, 0, 1]) yield (verify_pickle, ifilterfalse, _filterfalse, 3, 2, None, [0, 3, 0, 0, 1]) def test_product(): yield verify_same, product, itertools.product, None yield verify_same, product, itertools.product, None, [] yield verify_same, product, itertools.product, None, [], [] yield verify_same, product, itertools.product, None, [], [], [] yield verify_same, product, itertools.product, None, [5] yield verify_same, product, itertools.product, None, [5], [] yield verify_same, product, itertools.product, None, [], [5], [] yield verify_same, product, itertools.product, None, [2, 5], [3, 5, 9] yield verify_same, product, itertools.product, None, [2, 5], [1], [3, 5, 9] yield (verify_same, partial(product, repeat=3), partial(itertools.product, repeat=3), None, [1, 2, 3]) yield (verify_same, partial(product, repeat=4), partial(itertools.product, repeat=4), None, [1], [1, 2]) yield (verify_same, partial(product, repeat=2), partial(itertools.product, repeat=2), None, [3, 1], [1]) yield (verify_same, partial(product, repeat=3), partial(itertools.product, repeat=3), None, []) yield (verify_same, partial(product, repeat=3), partial(itertools.product, repeat=3), None, [], [3]) yield (verify_same, partial(product, repeat=3), partial(itertools.product, repeat=3), None, [1], []) yield (verify_pickle, product, itertools.product, 8, 3, [1, 2], [2, 3], [5, 6]) yield (verify_pickle, partial(product, repeat=3), partial(itertools.product, repeat=3), 50, 45, [1, 2], [3, 4]) def test_izip_longest(): yield (verify_same, izip_longest, _zip_longest, None, [], []) yield (verify_same, izip_longest, _zip_longest, None, [], [5, 4]) yield (verify_same, izip_longest, _zip_longest, None, [2], [5, 4]) yield (verify_same, izip_longest, _zip_longest, None, [7, 9], [5, 4]) yield (verify_same, izip_longest, _zip_longest, None, [7, 9], [4], [2, 9, 3]) yield (verify_same, izip_longest, _zip_longest, None, [7, 9], [4], []) yield (verify_same, izip_longest, _zip_longest, None, [7], [4], [], [5, 9]) yield (verify_same, partial(izip_longest, fillvalue=-1), partial(_zip_longest, fillvalue=-1), None, [7], [4], [], [5, 9]) yield (verify_pickle, izip_longest, _zip_longest, 3, 2, [7, 9, 8], [1, 2]) yield (verify_pickle, izip_longest, _zip_longest, 3, 1, [7, 9, 8], [1, 2]) def test_islice(): yield (verify_same, islice, _islice, None, [], 0) yield (verify_same, islice, _islice, None, [1], 0) yield (verify_same, islice, _islice, None, [1], 1) yield (verify_same, islice, _islice, None, [1], 3) yield (verify_same, islice, _islice, None, [1, 2, 3], 5) yield (verify_same, islice, _islice, None, [1, 2, 3], 1, 2) yield (verify_same, islice, _islice, None, [1, 2, 3], 1, 5, 3) yield (verify_same, islice, _islice, None, [1, 2, 3], 0, 3, 2) yield (verify_same, islice, _islice, None, [1, 2, 3, 4, 5], 1, 4, 3) yield (verify_same, islice, _islice, None, [1, 2, 3, 4, 5], -2, 9, 4) yield (verify_same, islice, _islice, None, [1, 2, 3], 4, 9) yield (verify_same, islice, _islice, None, [1, 2, 3], 0, 9, 5) yield (verify_pickle, islice, _islice, 3, 2, [1, 2, 3], 5) def verify_tee(n, original, seed): try: state = random.getstate() iterators = list(tee(original, n=n)) results = [[] for _ in range(n)] exhausted = [False] * n while not all(exhausted): # Upper argument of random.randint is inclusive. Argh. i = random.randint(0, n - 1) if not exhausted[i]: if len(results[i]) == len(original): assert_raises(StopIteration, next, iterators[i]) assert results[i] == original exhausted[i] = True else: if random.randint(0, 1): iterators[i] = cPickle.loads( cPickle.dumps(iterators[i])) elem = next(iterators[i]) results[i].append(elem) finally: random.setstate(state) def test_tee(): yield verify_tee, 2, [5, 2, 4], 1 yield verify_tee, 3, [5, 2, 4, 6, 9], 2 yield verify_tee, 5, [5, 2, 4, 6, 9], 3 yield verify_tee, 6, [], 3 def test_accumulate(): if not six.PY3: raise SkipTest() yield verify_same, accumulate, itertools.accumulate, None, [5, 4, 9] yield verify_same, accumulate, itertools.accumulate, None, ['a', 'b', 'c'] yield (verify_same, accumulate, itertools.accumulate, None, [[1], [2], [3, 4]]) yield (verify_same, accumulate, itertools.accumulate, None, [9, 1, 2], sub) yield verify_pickle, accumulate, itertools.accumulate, 3, 1, [5, 4, 9] yield verify_pickle, accumulate, itertools.accumulate, 3, 0, [5, 4, 9] yield (verify_pickle, accumulate, itertools.accumulate, 3, 2, ['a', 'b', 'c']) yield (verify_pickle, accumulate, itertools.accumulate, 2, 1, ['a', 'b', 'c']) yield (verify_pickle, accumulate, itertools.accumulate, 3, 1, [[1], [2], [3, 4]]) yield (verify_pickle, accumulate, itertools.accumulate, 2, 1, [9, 1, 2], sub) def test_takewhile(): base = (verify_same, takewhile, itertools.takewhile, None) yield base + (bool,) yield base + (bool, []) yield base + (bool, [0, 0, 5]) yield base + (bool, [1, 2, 0, 4, 0]) yield base + (partial(lt, 3), range(5, 0, -1)) base = (verify_pickle, takewhile, itertools.takewhile) yield base + (2, 0, bool, [1, 2, 0, 4, 0]) yield base + (2, 1, bool, [1, 2, 0, 4, 0]) yield base + (1, 0, partial(lt, 3), range(5, 0, -1)) def test_dropwhile(): base = (verify_same, dropwhile, itertools.dropwhile, None) yield base + (bool,) yield base + (bool, []) yield base + (bool, [5, 5, 2, 0, 0]) yield base + (bool, [1, 2, 0, 4, 0]) yield base + (partial(lt, 3), range(5, 0, -1)) base = (verify_pickle, dropwhile, itertools.dropwhile) yield base + (2, 1, bool, [5, 5, 2, 0, 0]) yield base + (2, 0, bool, [5, 5, 2, 0, 0]) yield base + (3, 0, bool, [1, 2, 0, 4, 0]) yield base + (3, 2, bool, [1, 2, 0, 4, 0]) def test_starmap(): yield verify_same, starmap, itertools.starmap, None, pos yield verify_same, starmap, itertools.starmap, None, pos, [] yield (verify_same, starmap, itertools.starmap, None, add, [(5, 9), [4, 2]]) yield (verify_pickle, starmap, itertools.starmap, 2, 0, add, [(5, 9), [4, 2]]) yield (verify_pickle, starmap, itertools.starmap, 2, 1, add, [(5, 9), [4, 2]]) def verify_groupby(*args, **kwargs): if 'n' in kwargs: if 'm' not in kwargs: raise ValueError('got n without m') pickle = True n = kwargs.pop('n') m = kwargs.pop('m') elif 'm' in kwargs: raise ValueError('got m without n') else: pickle = False n = m = None if 'pickle_outer' in kwargs: pickle_outer = kwargs.pop('pickle_outer') else: pickle_outer = None reference = itertools.groupby(*args, **kwargs) actual = groupby(*args, **kwargs) outer_iters = 0 while True: if outer_iters == pickle_outer: actual = cPickle.loads(cPickle.dumps(actual)) try: ref_key, ref_grouper = next(reference) except StopIteration: check_stops(actual) break try: actual_key, actual_grouper = next(actual) except StopIteration: assert False, "prematurely exhausted; expected {}".format(ref_key) if pickle: this_n = n[0] n = n[1:] this_m = m[0] m = m[1:] verify_pickle(partial(_identity, actual_grouper), partial(_identity, ref_grouper), this_n, this_m) else: verify_same(partial(_identity, actual_grouper), partial(_identity, ref_grouper), None) outer_iters += 1 def _mod(x, divisor): return x % divisor def test_groupby(): yield verify_groupby, [] yield verify_groupby, [1, 1, 2, 3, 3, 3, 4, 5, 7, 7] yield (verify_groupby, [1, 1, 3, 3, 4, 4, 2, 3, 3, 5], partial(_mod, divisor=2)) yield (partial(verify_groupby, n=[4, 3, 3], m=[0, 1, 2]), [1, 1, 3, 3, 4, 4, 2, 3, 3, 5], partial(_mod, divisor=2)) yield (partial(verify_groupby, n=[4, 3, 3], m=[1, 0, 1]), [1, 1, 3, 3, 4, 4, 2, 3, 3, 5], partial(_mod, divisor=2)) yield (partial(verify_groupby, n=[4, 3, 3], m=[1, 0, 1], pickle_outer=1), [1, 1, 3, 3, 4, 4, 2, 3, 3, 5], partial(_mod, divisor=2)) yield (partial(verify_groupby, n=[4, 3, 3], m=[2, 1, 0], pickle_outer=2), [1, 1, 3, 3, 4, 4, 2, 3, 3, 5], partial(_mod, divisor=2)) yield (partial(verify_groupby, n=[4, 3, 3], m=[1, 2, 0], pickle_outer=0), [1, 1, 3, 3, 4, 4, 2, 3, 3, 5], partial(_mod, divisor=2)) def test_permutations(): yield verify_same, permutations, itertools.permutations, None, _identity, yield (verify_same, permutations, itertools.permutations, None, []) yield (verify_same, permutations, itertools.permutations, None, [5, 4, 3, 2, 1]) yield (verify_same, permutations, itertools.permutations, None, [5, 4, 3, 2, 1], 2) yield (verify_pickle, permutations, itertools.permutations, 5 * 4 * 3 * 2, 0, [5, 4, 3, 2, 1]) yield (verify_pickle, permutations, itertools.permutations, 5 * 4 * 3 * 2, 5 * 4 * 3, [5, 4, 3, 2, 1]) yield (verify_pickle, permutations, itertools.permutations, 5 * 4, 10, [5, 4, 3, 2, 1], 2) yield (verify_pickle, permutations, itertools.permutations, 5 * 4, 5 * 4 - 1, [5, 4, 3, 2, 1], 2) def test_combinations(): yield verify_same, combinations, itertools.combinations, None yield (verify_same, combinations, itertools.combinations, None, []) yield (verify_same, combinations, itertools.combinations, 5 * 4 * 3, 0, [5, 4, 3, 2, 1], 2) yield (verify_pickle, combinations, itertools.combinations, 10, 5, [5, 4, 3, 2, 1], 2) def test_combinations_with_replacement(): yield (verify_same, combinations_with_replacement, itertools.combinations_with_replacement, None, _identity) yield (verify_same, combinations_with_replacement, itertools.combinations_with_replacement, None, _identity, []) yield (verify_same, combinations_with_replacement, itertools.combinations_with_replacement, None) yield (verify_same, combinations_with_replacement, itertools.combinations_with_replacement, None, [5, 4, 3, 2, 1], 2) yield (verify_pickle, combinations_with_replacement, itertools.combinations_with_replacement, 15, 3, [5, 4, 3, 2, 1], 2) yield (verify_pickle, combinations_with_replacement, itertools.combinations_with_replacement, 15, 0, [5, 4, 3, 2, 1], 2) def test_xrange(): yield assert_equal, list(xrange(10)), list(_xrange(10)) yield assert_equal, list(xrange(10, 15)), list(_xrange(10, 15)) yield assert_equal, list(xrange(10, 20, 2)), list(_xrange(10, 20, 2)) yield assert_equal, list(xrange(5, 1, -1)), list(_xrange(5, 1, -1)) yield (assert_equal, list(xrange(5, 55, 3)), list(cPickle.loads(cPickle.dumps(_xrange(5, 55, 3))))) yield assert_equal, _xrange(5).index(4), 4 yield assert_equal, _xrange(5, 9).index(6), 1 yield assert_equal, _xrange(8, 24, 3).index(11), 1 yield assert_equal, _xrange(25, 4, -5).index(25), 0 yield assert_equal, _xrange(28, 7, -7).index(14), 2 yield assert_raises, ValueError, _xrange(2, 9, 2).index, 3 yield assert_raises, ValueError, _xrange(2, 20, 2).index, 9 yield assert_equal, _xrange(5).count(5), 0 yield assert_equal, _xrange(5).count(4), 1 yield assert_equal, _xrange(4, 9).count(4), 1 yield assert_equal, _xrange(3, 9, 2).count(4), 0 yield assert_equal, _xrange(3, 9, 2).count(5), 1 yield assert_equal, _xrange(3, 9, 2).count(20), 0 yield assert_equal, _xrange(9, 3).count(5), 0 yield assert_equal, _xrange(3, 10, -1).count(5), 0 yield assert_equal, _xrange(10, 3, -1).count(5), 1 yield assert_equal, _xrange(10, 0, -2).count(6), 1 yield assert_equal, _xrange(10, -1, -3).count(7), 1 picklable-itertools-0.1.1/tests/test_extras.py000066400000000000000000000035061256055564600215640ustar00rootroot00000000000000from unittest import SkipTest from nose.tools import assert_raises from six.moves import zip from picklable_itertools.extras import (partition, partition_all, IterableLengthMismatch, equizip, interleave, roundrobin) from . import verify_same, verify_pickle def test_partition(): try: from toolz import itertoolz except ImportError: raise SkipTest() for obj, ref in zip([partition, partition_all], [itertoolz.partition, itertoolz.partition_all]): yield verify_same, obj, ref, None, 2, [5, 9, 2, 6] yield verify_same, obj, ref, None, 2, [5, 9, 2], 3 yield verify_same, obj, ref, None, 3, [5], 'a' yield verify_same, obj, ref, None, 3, [5, 9, 2, 9, 2] yield verify_same, obj, ref, None, 3, [5, 9, 2, 9, 2] yield verify_pickle, obj, ref, 2, 1, 3, [5, 9, 2, 9, 2, 4, 3] def test_equizip(): yield verify_same, equizip, zip, None, [3, 4], [9, 2], [9, 9] yield verify_same, equizip, zip, None, [3, 4, 8, 4, 2] assert_raises(IterableLengthMismatch, list, equizip([5, 4, 3], [2, 1])) assert_raises(IterableLengthMismatch, list, equizip([5, 4, 3], [])) def test_roundrobin(): assert list(roundrobin('ABC', 'D', 'EF')) == list('ADEBFC') assert (list(roundrobin('ABCDEF', 'JK', 'GHI', 'L')) == list('AJGLBKHCIDEF')) def test_interleave(): assert list(interleave(['ABC', 'D', 'EF'])) == list('ADEBFC') assert (list(interleave(['ABCDEF', 'JK', 'GHI', 'L'])) == list('AJGLBKHCIDEF')) class StupidException(Exception): pass def stupid_gen(): yield 'A' yield 'B' raise StupidException assert (list(interleave(['ABCDEF', stupid_gen()], [StupidException])) == list('AABBCDEF'))