pax_global_header00006660000000000000000000000064145512631010014510gustar00rootroot0000000000000052 comment=21efff8850e5a625cb86abd007105248da74d377 plyvel-1.5.1/000077500000000000000000000000001455126310100130275ustar00rootroot00000000000000plyvel-1.5.1/.dockerignore000066400000000000000000000000171455126310100155010ustar00rootroot00000000000000.direnv/ .tox/ plyvel-1.5.1/.gitignore000066400000000000000000000001741455126310100150210ustar00rootroot00000000000000*.py[co] __pycache__/ build/ dist/ *.egg-info/ .coverage .tox/ plyvel/_plyvel.cpp plyvel/_plyvel.html plyvel/_plyvel*.so plyvel-1.5.1/.travis.yml000066400000000000000000000004771455126310100151500ustar00rootroot00000000000000dist: xenial language: python branches: only: - master services: - docker python: # - "3.9" - "3.8" - "3.7" - "3.6" before_install: - scripts/install-snappy.sh - scripts/install-leveldb.sh install: - pip install -r requirements-dev.txt - make cython - pip install . script: - make test plyvel-1.5.1/Dockerfile000066400000000000000000000005141455126310100150210ustar00rootroot00000000000000FROM quay.io/pypa/manylinux2014_x86_64 # Remove sudo executable, since it does not work at all. # The installation scripts will not to invoke it. RUN rm "$(which sudo)" COPY scripts/ . RUN ./install-snappy.sh RUN ./install-leveldb.sh ENV PATH="/opt/python/cp39-cp39/bin:${PATH}" RUN pip install --upgrade pip setuptools cython plyvel-1.5.1/LICENSE.rst000066400000000000000000000030371455126310100146460ustar00rootroot00000000000000License ======= Copyright © 2012‒2017, Wouter Bolsterlee All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of the author nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. *(This is the OSI approved 3-clause "New BSD License".)* plyvel-1.5.1/MANIFEST.in000066400000000000000000000002541455126310100145660ustar00rootroot00000000000000include Makefile include requirements*.txt include *.rst include test/*.py include doc/conf.py doc/*.rst include plyvel/*.pyx plyvel/*.pxd plyvel/*.pxi plyvel/comparator.h plyvel-1.5.1/Makefile000066400000000000000000000015761455126310100145000ustar00rootroot00000000000000.PHONY: all cython ext doc clean test docker-build-env release all: cython ext cython: cython --version cython --cplus --fast-fail --annotate plyvel/_plyvel.pyx ext: cython python setup.py build_ext --inplace --force doc: python setup.py build_sphinx @echo @echo Generated documentation: "file://"$$(readlink -f doc/build/html/index.html) @echo clean: python setup.py clean $(RM) plyvel/_plyvel.cpp plyvel/_plyvel*.so $(RM) -r testdb/ $(RM) -r doc/build/ $(RM) -r plyvel.egg-info/ find . -name '*.py[co]' -delete find . -name __pycache__ -delete test: ext pytest docker-build-env: docker build -t plyvel-build . release: docker-build-env CIBW_BUILD='cp3*-manylinux_x86_64' \ CIBW_SKIP='cp36-manylinux_x86_64' \ CIBW_MANYLINUX_X86_64_IMAGE=plyvel-build \ CIBW_BEFORE_BUILD=scripts/cibuildwheel-before-build.sh \ CIBW_PLATFORM=linux \ cibuildwheel --output-dir dist plyvel-1.5.1/NEWS.rst000066400000000000000000000165421455126310100143450ustar00rootroot00000000000000=============== Version history =============== Plyvel 1.5.1 ============ Release date: 2024-01-15 * Add Python 3.12 support * Rebuild Linux wheels, including Python 3.12 wheels. * Allow using plyvel.DB as a context manager (`pr #151 `_) Plyvel 1.5.0 ============ Release date: 2022-10-26 * Rebuild Linux wheels, with ``manylinux_2_17`` and ``manylinux2014`` compatibility (``x86_64`` only). Also produce Python 3.11 wheels. Still using Snappy 1.1.9 and LevelDB 1.22. (`issue #148 `_) Plyvel 1.4.0 ============ Release date: 2021-12-29 * Build Linux wheels against Snappy 1.1.9, LevelDB 1.22, and produce Python 3.10 wheels (`issue #138 `_) * The minimum LevelDB version is now 1.21 (`pr #121 `_) * Add support for :py:meth:`WriteBatch.append()` (`pr #121 `_) * Add support for :py:meth:`WriteBatch.approximate_size()` (`pr #121 `_) Plyvel 1.3.0 ============ Release date: 2020-10-10 * Use manylinux2010 instead of manylinux1 to build wheels (`pr #103 `_) * Add Python 3.9 support * Drop Python 3.5 support * Completely drop Python 2 support Plyvel 1.2.0 ============ Release date: 2020-01-22 * Add Python 3.8 support (`pr #109 `_) * Drop Python 3.4 support (`pr #109 `_) * Build Linux wheels against Snappy 1.1.8, LevelDB 1.22, and produce Python 3.8 wheels (`issue #108 `_, `pr #111 `_, ) * Improve compilation flags for Darwin (OSX) builds (`pr #107 `_) Plyvel 1.1.0 ============ Release date: 2019-05-02 * Expose :py:attr:`~DB.name` attribute to Python code (`pr #90 `_) * Fix building sources on OSX. (`issue #95 `_, `pr #97 `_) * Build Linux wheels against LevelDB 1.21 Plyvel 1.0.5 ============ Release date: 2018-07-17 * Rebuild wheels: build against Snappy 1.1.7, and produce Python 3.7 wheels (`issue #78 `_, `pr #79 `_) Plyvel 1.0.4 ============ Release date: 2018-01-17 * Build Python wheels with Snappy compression support. (`issue #68 `_) Plyvel 1.0.3 ============ Release date: 2018-01-16 * Fix building sources on OSX. (`issue #66 `_, `pr #67 `_) Plyvel 1.0.2 ============ Release date: 2018-01-12 * Correctly build wide unicode Python 2.7 wheels (cp27-cp27mu, UCS4). (`issue #65 `_) Plyvel 1.0.1 ============ Release date: 2018-01-05 * Provide binary packages (manylinux1 wheels) for Linux. These wheel packages have the LevelDB library embedded. This should make installation on many Linux systems easier since these packages do not depend on a recent LevelDB version being installed system-wide: running ``pip install`` will simply download and install the extension, instead of compiling it. (`pr #64 `_, `issue #62 `_, `issue #63 `_) Plyvel 1.0.0 ============ Release date: 2018-01-03 * First 1.x release. This library is quite mature, so there is no reason to keep using 0.x version numbers. While at it, switch to semantic versioning. * Drop support for older Python versions. Minimum versions are now Python 3.4+ for modern Python and Python 2.7+ for legacy Python. * The mimimum LevelDB version is now 1.20, which added an option for the maximum file size, which is now exposed in Plyvel. (`pr #61 `_) * The various ``.put()`` methods are no longer restricted to just `bytes` (`str` in Python 2), but will accept any type implementing Python's buffer protocol, such as `bytes`, `bytearray`, and `memoryview`. Note that this only applies to values; keys must still be `bytes`. (`issue #52 `_) Plyvel 0.9 ========== Release date: 2014-08-27 * Ensure that the Python GIL is initialized when a custom comparator is used, since the background thread LevelDB uses for compaction calls back into Python code in that case. This makes single-threaded programs using a custom comparator work as intended. (`issue #35 `_) Plyvel 0.8 ========== Release date: 2013-11-29 * Allow snapshots to be closed explicitly using either :py:meth:`Snapshot.close()` or a ``with`` block (`issue #21 `_) Plyvel 0.7 ========== Release date: 2013-11-15 * New raw iterator API that mimics the LevelDB C++ interface. See :py:meth:`DB.raw_iterator()` and :py:class:`RawIterator`. (`issue #17 `_) * Migrate to `pytest` and `tox` for testing (`issue #24 `_) * Performance improvements in iterator and write batch construction. The internal calls within Plyvel are now a bit faster, and the `weakref` handling required for iterators is now a lot faster due to replacing :py:class:`weakref.WeakValueDictionary` with manual `weakref` handling. * The `fill_cache`, `verify_checksums`, and `sync` arguments to various methods are now correctly taken into account everywhere, and their default values are now booleans reflecting the the LevelDB defaults. Plyvel 0.6 ========== Release date: 2013-10-18 * Allow iterators to be closed explicitly using either :py:meth:`Iterator.close()` or a ``with`` block (`issue #19 `_) * Add useful ``__repr__()`` for :py:class:`DB` and :py:class:`PrefixedDB` instances (`issue #16 `_) Plyvel 0.5 ========== Release date: 2013-09-17 * Fix :py:meth:`Iterator.seek()` for :py:class:`PrefixedDB` iterators (`issue #15 `_) * Make some argument type checking a bit stricter (mostly ``None`` checks) * Support LRU caches larger than 2GB by using the right integer type for the ``lru_cache_size`` :py:class:`DB` constructor argument. * Documentation improvements Plyvel 0.4 ========== Release date: 2013-06-17 * Add optional 'default' argument for all ``.get()`` methods (`issue #11 `_) Plyvel 0.3 ========== Release date: 2013-06-03 * Fix iterator behaviour for reverse iterators using a prefix (`issue #9 `_) * Documentation improvements Plyvel 0.2 ========== Release date: 2013-03-15 * Fix iterator behaviour for iterators using non-existing start or stop keys (`issue #4 `_) Plyvel 0.1 ========== Release date: 2012-11-26 * Initial release plyvel-1.5.1/README.rst000066400000000000000000000013431455126310100145170ustar00rootroot00000000000000====== Plyvel ====== .. image:: https://travis-ci.org/wbolster/plyvel.svg?branch=master :target: https://travis-ci.org/wbolster/plyvel **Plyvel** is a fast and feature-rich Python interface to LevelDB_. Plyvel has a rich feature set, high performance, and a friendly Pythonic API. See the documentation and project page for more information: * Documentation_ * `Project page`_ * `PyPI page`_ .. _Project page: https://github.com/wbolster/plyvel .. _Documentation: https://plyvel.readthedocs.io/ .. _PyPI page: http://pypi.python.org/pypi/plyvel/ .. _LevelDB: https://github.com/google/leveldb Note that using a released version is recommended over a checkout from version control. See the installation docs for more information. plyvel-1.5.1/TODO.rst000066400000000000000000000001301455126310100143200ustar00rootroot00000000000000TODO ==== See the `issue list on Github `_. plyvel-1.5.1/doc/000077500000000000000000000000001455126310100135745ustar00rootroot00000000000000plyvel-1.5.1/doc/api.rst000066400000000000000000000475001455126310100151050ustar00rootroot00000000000000============= API reference ============= This document is the API reference for Plyvel. It describes all classes, methods, functions, and attributes that are part of the public API. Most of the terminology in the Plyvel API comes straight from the LevelDB API. See the LevelDB documentation and the LevelDB header files (``$prefix/include/leveldb/*.h``) for more detailed explanations of all flags and options. Database ======== Plyvel exposes the :py:class:`DB` class as the main interface to LevelDB. Application code should create a :py:class:`DB` and use the appropriate methods on this instance to create write batches, snapshots, and iterators for that LevelDB database. .. py:class:: DB LevelDB database .. py:method:: __init__(name, create_if_missing=False, error_if_exists=False, paranoid_checks=None, write_buffer_size=None, max_open_files=None, lru_cache_size=None, block_size=None, block_restart_interval=None, max_file_size=None, compression='snappy', bloom_filter_bits=0, comparator=None, comparator_name=None) Open the underlying database handle. Most arguments have the same name as the corresponding LevelDB parameters; see the LevelDB documentation for a detailed description. Arguments defaulting to `None` are only propagated to LevelDB if specified, e.g. not specifying a `write_buffer_size` means the LevelDB defaults are used. Most arguments are optional; only the database name is required. See the descriptions for :cpp:class:`DB`, :cpp:func:`DB::Open`, :cpp:class:`Cache`, :cpp:class:`FilterPolicy`, and :cpp:class:`Comparator` in the LevelDB C++ API for more information. .. versionadded:: 1.0.0 `max_file_size` argument :param str name: name of the database (directory name) :param bool create_if_missing: whether a new database should be created if needed :param bool error_if_exists: whether to raise an exception if the database already exists :param bool paranoid_checks: whether to enable paranoid checks :param int write_buffer_size: size of the write buffer (in bytes) :param int max_open_files: maximum number of files to keep open :param int lru_cache_size: size of the LRU cache (in bytes) :param int block_size: block size (in bytes) :param int block_restart_interval: block restart interval for delta encoding of keys :param bool max_file_size: maximum file size (in bytes) :param bool compression: whether to use Snappy compression (enabled by default)) :param int bloom_filter_bits: the number of bits to use per key for a bloom filter; the default of 0 means that no bloom filter will be used :param callable comparator: a custom comparator callable that takes two byte strings and returns an integer :param bytes comparator_name: name for the custom comparator .. py:attribute:: name The (directory) name of this :py:class:`DB` instance. This is a *read-only* attribute and must be set at instantiation time. *New in version 1.1.0.* .. py:method:: close() Close the database. This closes the database and releases associated resources such as open file pointers and caches. Any further operations on the closed database will raise :py:exc:`RuntimeError`. .. warning:: Closing a database while other threads are busy accessing the same database may result in hard crashes, since database operations do not perform any synchronisation/locking on the database object (for performance reasons) and simply assume it is available (and open). Applications should make sure not to close databases that are concurrently used from other threads. See the description for :cpp:class:`DB` in the LevelDB C++ API for more information. This method deletes the underlying DB handle in the LevelDB C++ API and also frees other related objects. .. py:attribute:: closed Boolean attribute indicating whether the database is closed. .. py:method:: get(key, default=None, verify_checksums=False, fill_cache=True) Get the value for the specified key, or `default` if no value was set. See the description for :cpp:func:`DB::Get` in the LevelDB C++ API for more information. .. versionadded:: 0.4 `default` argument :param bytes key: key to retrieve :param default: default value if key is not found :param bool verify_checksums: whether to verify checksums :param bool fill_cache: whether to fill the cache :return: value for the specified key, or `None` if not found :rtype: bytes .. py:method:: put(key, value, sync=False) Set a value for the specified key. See the description for :cpp:func:`DB::Put` in the LevelDB C++ API for more information. :param bytes key: key to set :param bytes value: value to set :param bool sync: whether to use synchronous writes .. method:: delete(key, sync=False) Delete the key/value pair for the specified key. See the description for :cpp:func:`DB::Delete` in the LevelDB C++ API for more information. :param bytes key: key to delete :param bool sync: whether to use synchronous writes .. py:method:: write_batch(transaction=False, sync=False) Create a new :py:class:`WriteBatch` instance for this database. See the :py:class:`WriteBatch` API for more information. Note that this method does not write a batch to the database; it only creates a new write batch instance. :param bool transaction: whether to enable transaction-like behaviour when the batch is used in a ``with`` block :param bool sync: whether to use synchronous writes :return: new :py:class:`WriteBatch` instance :rtype: :py:class:`WriteBatch` .. py:method:: iterator(reverse=False, start=None, stop=None, include_start=True, include_stop=False, prefix=None, include_key=True, include_value=True, verify_checksums=False, fill_cache=True) Create a new :py:class:`Iterator` instance for this database. All arguments are optional, and not all arguments can be used together, because some combinations make no sense. In particular: * `start` and `stop` cannot be used if a `prefix` is specified. * `include_start` and `include_stop` are only used if `start` and `stop` are specified. Note: due to the way the `prefix` support is implemented, this feature only works reliably when the default DB comparator is used. See the :py:class:`Iterator` API for more information about iterators. :param bool reverse: whether the iterator should iterate in reverse order :param bytes start: the start key (inclusive by default) of the iterator range :param bytes stop: the stop key (exclusive by default) of the iterator range :param bool include_start: whether to include the start key in the range :param bool include_stop: whether to include the stop key in the range :param bytes prefix: prefix that all keys in the the range must have :param bool include_key: whether to include keys in the returned data :param bool include_value: whether to include values in the returned data :param bool verify_checksums: whether to verify checksums :param bool fill_cache: whether to fill the cache :return: new :py:class:`Iterator` instance :rtype: :py:class:`Iterator` .. py:method:: raw_iterator(verify_checksums=False, fill_cache=True) Create a new :py:class:`RawIterator` instance for this database. See the :py:class:`RawIterator` API for more information. .. py:method:: snapshot() Create a new :py:class:`Snapshot` instance for this database. See the :py:class:`Snapshot` API for more information. .. py:method:: get_property(name) Get the specified property from LevelDB. This returns the property value or `None` if no value is available. Example property name: ``b'leveldb.stats'``. See the description for :cpp:func:`DB::GetProperty` in the LevelDB C++ API for more information. :param bytes name: name of the property :return: property value or `None` :rtype: bytes .. py:method:: compact_range(start=None, stop=None) Compact underlying storage for the specified key range. See the description for :cpp:func:`DB::CompactRange` in the LevelDB C++ API for more information. :param bytes start: start key of range to compact (optional) :param bytes stop: stop key of range to compact (optional) .. py:method:: approximate_size(start, stop) Return the approximate file system size for the specified range. See the description for :cpp:func:`DB::GetApproximateSizes` in the LevelDB C++ API for more information. :param bytes start: start key of the range :param bytes stop: stop key of the range :return: approximate size :rtype: int .. py:method:: approximate_sizes(\*ranges) Return the approximate file system sizes for the specified ranges. This method takes a variable number of arguments. Each argument denotes a range as a `(start, stop)` tuple, where `start` and `stop` are both byte strings. Example:: db.approximate_sizes( (b'a-key', b'other-key'), (b'some-other-key', b'yet-another-key')) See the description for :cpp:func:`DB::GetApproximateSizes` in the LevelDB C++ API for more information. :param ranges: variable number of `(start, stop`) tuples :return: approximate sizes for the specified ranges :rtype: list .. py:method:: prefixed_db(prefix) Return a new :py:class:`PrefixedDB` instance for this database. See the :py:class:`PrefixedDB` API for more information. :param bytes prefix: prefix to use :return: new :py:class:`PrefixedDB` instance :rtype: :py:class:`PrefixedDB` Prefixed database ----------------- .. py:class:: PrefixedDB A :py:class:`DB`-like object that transparently prefixes all database keys. Do not instantiate directly; use :py:meth:`DB.prefixed_db` instead. .. py:attribute:: prefix The prefix used by this :py:class:`PrefixedDB`. .. py:attribute:: db The underlying :py:class:`DB` instance. .. py:method:: get(...) See :py:meth:`DB.get`. .. py:method:: put(...) See :py:meth:`DB.put`. .. py:method:: delete(...) See :py:meth:`DB.delete`. .. py:method:: write_batch(...) See :py:meth:`DB.write_batch`. .. py:method:: iterator(...) See :py:meth:`DB.iterator`. .. py:method:: snapshot(...) See :py:meth:`DB.snapshot`. .. py:method:: prefixed_db(...) Create another :py:class:`PrefixedDB` instance with an additional key prefix, which will be appended to the prefix used by this :py:class:`PrefixedDB` instance. See :py:meth:`DB.prefixed_db`. Database maintenance -------------------- Existing databases can be repaired or destroyed using these module level functions: .. py:function:: repair_db(name, paranoid_checks=None, write_buffer_size=None, max_open_files=None, lru_cache_size=None, block_size=None, block_restart_interval=None, max_file_size=None, compression='snappy', bloom_filter_bits=0, comparator=None, comparator_name=None) Repair the specified database. See :py:class:`DB` for a description of the arguments. See the description for :cpp:func:`RepairDB` in the LevelDB C++ API for more information. .. py:function:: destroy_db(name) Destroy the specified database. :param str name: name of the database (directory name) See the description for :cpp:func:`DestroyDB` in the LevelDB C++ API for more information. Write batch =========== .. py:class:: WriteBatch Write batch for batch put/delete operations Instances of this class can be used as context managers (Python's ``with`` block). When the ``with`` block terminates, the write batch will automatically write itself to the database without an explicit call to :py:meth:`WriteBatch.write`:: with db.write_batch() as b: b.put(b'key', b'value') The `transaction` argument to :py:meth:`DB.write_batch` specifies whether the batch should be written after an exception occurred in the ``with`` block. By default, the batch is written (this is like a ``try`` statement with a ``finally`` clause), but if transaction mode is enabled`, the batch will be discarded (this is like a ``try`` statement with an ``else`` clause). Note: methods on a :py:class:`WriteBatch` do not take a `sync` argument; this flag can be specified for the complete write batch when it is created using :py:meth:`DB.write_batch`. Do not instantiate directly; use :py:meth:`DB.write_batch` instead. See the descriptions for :cpp:class:`WriteBatch` and :cpp:func:`DB::Write` in the LevelDB C++ API for more information. .. py:method:: put(key, value) Set a value for the specified key. This is like :py:meth:`DB.put`, but operates on the write batch instead. .. py:method:: delete(key) Delete the key/value pair for the specified key. This is like :py:meth:`DB.delete`, but operates on the write batch instead. .. py:method:: clear() Clear the batch. This discards all updates buffered in this write batch. .. py:method:: write() Write the batch to the associated database. If you use the write batch as a context manager (in a ``with`` block), this method will be invoked automatically. .. py:method:: append(source) Copy the operations in `source` (another :py:class:`WriteBatch` instance) to this batch. .. py:method:: approximate_size() Return the size of the database changes caused by this batch. Snapshot ======== .. py:class:: Snapshot Database snapshot A snapshot provides a consistent view over keys and values. After making a snapshot, puts and deletes on the database will not be visible by the snapshot. Do not keep unnecessary references to instances of this class around longer than needed, because LevelDB will not release the resources required for this snapshot until a snapshot is released. Do not instantiate directly; use :py:meth:`DB.snapshot` instead. See the descriptions for :cpp:func:`DB::GetSnapshot` and :cpp:func:`DB::ReleaseSnapshot` in the LevelDB C++ API for more information. .. py:method:: get(...) Get the value for the specified key, or `None` if no value was set. Same as :py:meth:`DB.get`, but operates on the snapshot instead. .. py:method:: iterator(...) Create a new :py:class:`Iterator` instance for this snapshot. Same as :py:meth:`DB.iterator`, but operates on the snapshot instead. .. py:method:: raw_iterator(...) Create a new :py:class:`RawIterator` instance for this snapshot. Same as :py:meth:`DB.raw_iterator`, but operates on the snapshot instead. .. py:method:: close() Close the snapshot. Can also be accomplished using a context manager. See :py:meth:`Iterator.close` for an example. .. versionadded:: 0.8 .. py:method:: release() Alias for :py:meth:`Snapshot.close`. *Release* is the terminology used in the LevelDB C++ API. .. versionadded:: 0.8 Iterator ======== Regular iterators ----------------- Plyvel's :py:class:`Iterator` is intended to be used like a normal Python iterator, so you can just use a standard ``for`` loop to iterate over it. Directly invoking methods on the :py:class:`Iterator` returned by :py:meth:`DB.iterator` method is only required for additional functionality. .. py:class:: Iterator Iterator to iterate over (ranges of) a database The next item in the iterator can be obtained using the :py:func:`next` built-in or by looping over the iterator using a ``for`` loop. Do not instantiate directly; use :py:meth:`DB.iterator` or :py:meth:`Snapshot.iterator` instead. See the descriptions for :cpp:func:`DB::NewIterator` and :cpp:class:`Iterator` in the LevelDB C++ API for more information. .. py:method:: prev() Move one step back and return the previous entry. This returns the same value as the most recent :py:func:`next` call (if any). .. py:method:: seek_to_start() Move the iterator to the start key (or the begin). This "rewinds" the iterator, so that it is in the same state as when first created. This means calling :py:func:`next` afterwards will return the first entry. .. py:method:: seek_to_stop() Move the iterator to the stop key (or the end). This "fast-forwards" the iterator past the end. After this call the iterator is exhausted, which means a call to :py:func:`next` raises StopIteration, but :py:meth:`~Iterator.prev` will work. .. py:method:: seek(target) Move the iterator to the specified `target`. This moves the iterator to the the first key that sorts equal or after the specified `target` within the iterator range (`start` and `stop`). .. py:method:: close() Close the iterator. This closes the iterator and releases the associated resources. Any further operations on the closed iterator will raise :py:exc:`RuntimeError`. To automatically close an iterator, a context manager can be used:: with db.iterator() as it: for k, v in it: pass # do something it.seek_to_start() # raises RuntimeError .. versionadded:: 0.6 Raw iterators ------------- The raw iteration API mimics the C++ iterator interface provided by LevelDB. See the LevelDB documentation for a detailed description. .. py:class:: RawIterator Raw iterator to iterate over a database .. versionadded:: 0.7 .. py:method:: valid() Check whether the iterator is currently valid. .. py:method:: seek_to_first() Seek to the first key (if any). .. py:method:: seek_to_last() Seek to the last key (if any). .. py:method:: seek(target) Seek to or past the specified key (if any). .. py:method:: next() Move the iterator one step forward. May raise :py:exc:`IteratorInvalidError`. .. py:method:: prev() Move the iterator one step backward. May raise :py:exc:`IteratorInvalidError`. .. py:method:: key() Return the current key. May raise :py:exc:`IteratorInvalidError`. .. py:method:: value() Return the current value. May raise :py:exc:`IteratorInvalidError`. .. py:method:: item() Return the current key and value as a tuple. May raise :py:exc:`IteratorInvalidError`. .. py:method:: close() Close the iterator. Can also be accomplished using a context manager. See :py:meth:`Iterator.close`. Errors ====== Plyvel uses standard exceptions like ``TypeError``, ``ValueError``, and ``RuntimeError`` as much as possible. For LevelDB specific errors, Plyvel may raise a few custom exceptions, which are described below. .. py:exception:: Error Generic LevelDB error This class is also the "parent" error for other LevelDB errors (:py:exc:`IOError` and :py:exc:`CorruptionError`). Other exceptions from this module extend from this class. .. py:exception:: IOError LevelDB IO error This class extends both the main LevelDB Error class from this module and Python's built-in IOError. .. py:exception:: CorruptionError LevelDB corruption error .. py:exception:: IteratorInvalidError Used by :py:class:`RawIterator` to signal invalid iterator state. .. vim: set tabstop=3 shiftwidth=3: plyvel-1.5.1/doc/conf.py000066400000000000000000000012701455126310100150730ustar00rootroot00000000000000import datetime import os.path version_file = os.path.join( os.path.dirname(__file__), '../plyvel/_version.py') with open(version_file) as fp: exec(fp.read(), globals(), locals()) extensions = ['sphinx.ext.autodoc', 'sphinx.ext.coverage'] templates_path = ['_templates'] exclude_patterns = ['_build'] source_suffix = '.rst' master_doc = 'index' project = u'Plyvel' copyright = u'2012‒{}, Wouter Bolsterlee'.format( datetime.datetime.now().year) version = __version__ release = __version__ autodoc_default_options = {"members": True, "undoc-members": True} autodoc_member_order = 'bysource' html_domain_indices = False html_show_sourcelink = False html_show_sphinx = False plyvel-1.5.1/doc/developer.rst000066400000000000000000000034631455126310100163210ustar00rootroot00000000000000=========================== Contributing and developing =========================== .. _Plyvel project page: https://github.com/wbolster/plyvel Reporting issues ================ Plyvel uses Github's issue tracker. See the `Plyvel project page`_ on Github. Obtaining the source code ========================= The Plyvel source code can be found on Github. See the `Plyvel project page`_ on Github. Compiling from source ===================== A simple ``make`` suffices to build the Plyvel extension. Note that the ``setup.py`` script does *not* invoke Cython, so that installations using ``pip install`` do not need to depend on Cython. A few remarks about the code: * Plyvel is mostly written in Cython. The LevelDB API is described in `leveldb.pxd`, and subsequently used from Cython. * The custom comparator support is written in C++ since it contains a C++ class that extends a LevelDB C++ class. The Python C API is used for the callbacks into Python. This custom class is made available in Cython using `comparator.pxd`. Running the tests ================= Almost all Plyvel code is covered by the unit tests. Plyvel uses *pytest* and *tox* for running those tests. Type ``make test`` to run the unit tests, or run ``tox`` to run the tests against multiple Python versions. Producing binary packages ========================= To build a non-portable binary package for a single platform:: python setup.py bdist_wheel See the comments at the top of the ``Dockerfile`` for instructions on how to build portable ``manylinux1`` wheels for multiple Python versions that should work on many Linux platforms. Generating the documentation ============================ The documentation is written in ReStructuredText (reST) format and processed using *Sphinx*. Type ``make doc`` to build the HTML documentation. plyvel-1.5.1/doc/index.rst000066400000000000000000000035451455126310100154440ustar00rootroot00000000000000====== Plyvel ====== **Plyvel** is a fast and feature-rich Python interface to LevelDB_. .. _LevelDB: https://github.com/google/leveldb Plyvel's key features are: * **Rich feature set** Plyvel wraps most of the LevelDB C++ API and adds some features of its own. In addition to basic features like getting, putting and deleting data, Plyvel allows you to use write batches, database snapshots, very flexible iterators, prefixed databases, bloom filters, custom cache sizes, custom comparators, and other goodness LevelDB has to offer. * **Friendly Pythonic API** Plyvel has a friendly and well-designed API that uses Python idioms like iterators and context managers (``with`` blocks), without sacrificing the power or performance of the underlying LevelDB C++ API. * **High performance** Plyvel executes all performance-critical code at C speed (using Cython_), which means Plyvel is a good fit for high-performance applications. .. _Cython: http://cython.org/ * **Extensive documentation** Plyvel comes with extensive documentation, including a user guide and API reference material. .. note Do you like Plyvel? You should know that Plyvel is a hobby project, written and maintained by me, Wouter Bolsterlee, in my spare time. Please consider making a small donation_ to let me know you appreciate my work. Thanks! .. _donation: https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=4FF4VZ5LTJ73N .. rubric:: Documentation contents .. toctree:: :maxdepth: 2 installation user api news developer license .. rubric:: External links * `Online documentation `_ (Read the docs) * `Project page `_ with source code and issue tracker (Github) * `Python Package Index (PyPI) page `_ with released tarballs plyvel-1.5.1/doc/installation.rst000066400000000000000000000057771455126310100170470ustar00rootroot00000000000000================== Installation guide ================== .. highlight:: sh This guide provides installation instructions for Plyvel. Build and install Plyvel ======================== The recommended (and easiest) way to install Plyvel is to install it into a virtual environment (*virtualenv*):: $ virtualenv envname $ source envname/bin/activate Now you can automatically install the latest Plyvel release from the `Python Package Index `_ (PyPI) using ``pip``:: (envname) $ pip install plyvel (In case you're feeling old-fashioned: downloading a source tarball, unpacking it and installing it manually with ``python setup.py install`` should also work.) The Plyvel source package does not include a copy of LevelDB itself. Plyvel requires LevelDB development headers and an installed shared library for LevelDB during build time, and the same installed shared library at runtime. To build from source, make sure you have a shared LevelDB library and the development headers installed where the compiler and linker can find them. For Debian or Ubuntu something like ``apt-get install libleveldb1v5 libleveldb-dev`` should suffice. For Linux, Plyvel also ships as pre-built binary packages (``manylinux1`` wheels) that have LevelDB embedded. Simply running ``pip install plyvel`` does the right thing with a modern ``pip`` on a modern Linux platform, even without any LevelDB libraries on your system. .. note:: Plyvel 1.x depends on LevelDB >= 1.20, which at the time of writing (early 2018) is more recent than the versions packaged by various Linux distributions. Using an older version will result in compile-time errors. The easiest solution is to use the pre-built binary packages. Alternatively, install LevelDB manually on your system. The Dockerfile in the Plyvel source repository, which is used for building the official binary packages, shows how to do this. .. warning:: The above installation method applies only to released packages available from PyPI. If you are building and installing from a source tree acquired through other means, e.g. checked out from source control, you will need to run Cython first. If you don't, you will see errors about missing source files. See the :doc:`developer documentation ` for more information. Verify that it works ==================== After installation, this command should not give any output:: (envname) $ python -c 'import plyvel' If you see an ``ImportError`` complaining about undefined symbols, e.g. .. code-block:: text ImportError: ./plyvel.so: undefined symbol: _ZN7leveldb10WriteBatch5ClearEv …then the installer (actually, the linker) was unable to find the LevelDB library on your system when building Plyvel. Install LevelDB or set the proper environment variables for the compiler and linker and try ``pip install --upgrade --force-reinstall plyvel``. .. rubric:: Next steps Continue with the :doc:`user guide ` to see how to use Plyvel. .. vim: set spell spelllang=en: plyvel-1.5.1/doc/license.rst000066400000000000000000000000341455126310100157450ustar00rootroot00000000000000.. include:: ../LICENSE.rst plyvel-1.5.1/doc/news.rst000066400000000000000000000000311455126310100152740ustar00rootroot00000000000000.. include:: ../NEWS.rst plyvel-1.5.1/doc/user.rst000066400000000000000000000423021455126310100153050ustar00rootroot00000000000000========== User guide ========== This user guide gives an overview of Plyvel. It covers: * opening and closing databases, * storing and retrieving data, * working with write batches, * using snapshots, * iterating over your data, * using prefixed databases, and * implementing custom comparators. Note: this document assumes basic familiarity with LevelDB; visit the `LevelDB homepage`_ for more information about its features and design. .. _`LevelDB homepage`: https://github.com/google/leveldb Getting started =============== After :doc:`installing Plyvel `, we can simply import ``plyvel``:: >>> import plyvel Let's open a new database by creating a new :py:class:`DB` instance:: >>> db = plyvel.DB('/tmp/testdb/', create_if_missing=True) That's all there is to it. At this point ``/tmp/testdb/`` contains a fresh LevelDB database (assuming the directory did not contain a LevelDB database already). For real-world applications, you probably want to tweak things like the size of the memory cache and the number of bits to use for the (optional) bloom filter. These settings, and many others, can be specified as arguments to the :py:class:`DB` constructor. For this tutorial we'll just use the LevelDB defaults. To close the database we just opened, use :py:meth:`DB.close` and inspect the ``closed`` property:: >>> db.closed False >>> db.close() >>> db.closed True Alternatively, you can just delete the variable that points to it, but this might not close the database immediately, e.g. because active iterators are using it:: >>> del db Note that the remainder of this tutorial assumes an open database, so you probably want to skip the above if you're performing all the steps in this tutorial yourself. Basic operations ================ Now that we have our database, we can use the basic LevelDB operations: putting, getting, and deleting data. Let's look at these in turn. First we'll add some data to the database by calling :py:meth:`DB.put` with a key/value pair:: >>> db.put(b'key', b'value') >>> db.put(b'another-key', b'another-value') To get the data out again, use :py:meth:`DB.get`:: >>> db.get(b'key') 'value' If you try to retrieve a key that does not exist, a ``None`` value is returned:: >>> print(db.get(b'yet-another-key')) None Optionally, you can specify a default value, just like :py:meth:`dict.get`:: >>> print(db.get(b'yet-another-key', b'default-value')) 'default-value' Finally, to delete data from the database, use :py:meth:`DB.delete`:: >>> db.delete(b'key') >>> db.delete(b'another-key') At this point our database is empty again. Note that, in addition to the basic use shown above, the :py:meth:`~DB.put`, :py:meth:`~DB.get`, and :py:meth:`~DB.delete` methods accept optional keyword arguments that influence their behaviour, e.g. for synchronous writes or reads that will not fill the cache. Write batches ============= LevelDB provides *write batches* for bulk data modification. Since batches are faster than repeatedly calling :py:meth:`DB.put` or :py:meth:`DB.delete`, batches are perfect for bulk loading data. Let's write some data:: >>> wb = db.write_batch() >>> for i in xrange(100000): wb.put(str(i).encode(), str(i).encode() * 100) ... >>> wb.write() Since write batches are committed in an atomic way, either the complete batch is written, or not at all, so if your machine crashes while LevelDB writes the batch to disk, the database will not end up containing partial or inconsistent data. This makes write batches very useful for multiple modifications to the database that should be applied as a group. Write batches can also act as context managers. The following code does the same as the example above, but there is no call to :py:meth:`WriteBatch.write` anymore: >>> with db.write_batch() as wb: ... for i in xrange(100000): ... wb.put(str(i).encode(), str(i).encode() * 100) If the ``with`` block raises an exception, pending modifications in the write batch will still be written to the database. This means each modification using :py:meth:`~WriteBatch.put` or :py:meth:`~WriteBatch.delete` that happened before the exception was raised will be applied to the database:: >>> with db.write_batch() as wb: ... wb.put(b'key-1', b'value-1') ... raise ValueError("Something went wrong!") ... wb.put(b'key-2', b'value-2') At this point the database contains ``key-1``, but not ``key-2``. Sometimes this behaviour is undesirable. If you want to discard all pending modifications in the write batch if an exception occurs, you can simply set the `transaction` argument:: >>> with db.write_batch(transaction=True) as wb: ... wb.put(b'key-3', b'value-3') ... raise ValueError("Something went wrong!") ... wb.put(b'key-4', b'value-4') In this case the database will not be modified, because the ``with`` block raised an exception. In this example this means that neither ``key-3`` nor ``key-4`` will be saved. .. note:: Write batches will never silently suppress exceptions. Exceptions will be propagated regardless of the value of the `transaction` argument, so in the examples above you will still see the ValueError. Snapshots ========= A snapshot is a consistent read-only view over the entire database. Any data that is modified after the snapshot was taken, will not be seen by the snapshot. Let's store a value: >>> db.put(b'key', b'first-value') Now we'll make a snapshot using :py:meth:`DB.snapshot`:: >>> sn = db.snapshot() >>> sn.get(b'key') 'first-value' At this point any modifications to the database will not be visible by the snapshot:: >>> db.put(b'key', b'second-value') >>> sn.get(b'key') 'first-value' Long-lived snapshots may consume significant resources in your LevelDB database, since the snapshot prevents LevelDB from cleaning up old data that is still accessible by the snapshot. This means that you should never keep a snapshot around longer than necessary. The snapshot and its associated resources will be released automatically when the snapshot reference count drops to zero, which (for local variables) happens when the variable goes out of scope (or after you've issued a ``del`` statement). If you want explicit control over the lifetime of a snapshot, you can also clean it up yourself using :py:meth:`Snapshot.close`:: >>> sn.close() Alternatively, you can use the snapshot as a context manager: >>> with db.snapshot() as sn: ... sn.get(b'key') Iterators ========= All key/value pairs in a LevelDB database will be sorted by key. Because of this, data can be efficiently retrieved in sorted order. This is what iterators are for. Iterators allow you to efficiently iterate over all sorted key/value pairs in the database, or more likely, a range of the database. Let's fill the database with some data first: >>> db.put(b'key-1', b'value-1') >>> db.put(b'key-5', b'value-5') >>> db.put(b'key-3', b'value-3') >>> db.put(b'key-2', b'value-2') >>> db.put(b'key-4', b'value-4') Now we can iterate over all data using a simple ``for`` loop, which will return all key/value pairs in lexicographical key order:: >>> for key, value in db: ... print(key) ... print(value) ... key-1 value-1 key-2 value-2 key-3 value-3 key-4 value-4 key-5 While the complete database can be iterated over by just looping over the :py:class:`DB` instance, this is generally not useful. The :py:meth:`DB.iterator` method allows you to obtain more specific iterators. This method takes several optional arguments to specify how the iterator should behave. Iterating over a key range -------------------------- Limiting the range of values that you want the iterator to iterate over can be achieved by supplying `start` and/or `stop` arguments:: >>> for key, value in db.iterator(start=b'key-2', stop=b'key-4'): ... print(key) ... key-2 key-3 Any combination of `start` and `stop` arguments is possible. For example, to iterate from a specific start key until the end of the database:: >>> for key, value in db.iterator(start=b'key-3'): ... print(key) ... key-3 key-4 key-5 By default the start key is *inclusive* and the stop key is *exclusive*. This matches the behaviour of Python's built-in :py:func:`range` function. If you want different behaviour, you can use the `include_start` and `include_stop` arguments:: >>> for key, value in db.iterator(start=b'key-2', include_start=False, ... stop=b'key-5', include_stop=True): ... print(key) key-3 key-4 key-5 Instead of specifying `start` and `stop` keys, you can also specify a `prefix` for keys. In this case the iterator will only return key/value pairs whose key starts with the specified prefix. In our example, all keys have the same prefix, so this will return all key/value pairs: >>> for key, value in db.iterator(prefix=b'ke'): ... print(key) key-1 key-2 key-3 key-4 key-5 >>> for key, value in db.iterator(prefix=b'key-4'): ... print(key) key-4 Limiting the returned data -------------------------- If you're only interested in either the key or the value, you can use the `include_key` and `include_value` arguments to omit data you don't need:: >>> list(db.iterator(start=b'key-2', stop=b'key-4', include_value=False)) ['key-2', 'key-3'] >>> list(db.iterator(start=b'key-2', stop=b'key-4', include_key=False)) ['value-2', 'value-3'] Only requesting the data that you are interested in results in slightly faster iterators, since Plyvel will avoid unnecessary memory copies and object construction in this case. Iterating in reverse order -------------------------- LevelDB also supports reverse iteration. Just set the `reverse` argument to `True` to obtain a reverse iterator:: >>> list(db.iterator(start=b'key-2', stop=b'key-4', include_value=False, reverse=True)) ['key-3', 'key-2'] Note that the `start` and `stop` keys are the same; the only difference is the `reverse` argument. Iterating over snapshots ------------------------ In addition to directly iterating over the database, LevelDB also supports iterating over snapshots using the :py:meth:`Snapshot.iterator` method. This method works exactly the same as :py:meth:`DB.iterator`, except that it operates on the snapshot instead of the complete database. Closing iterators ----------------- It is generally not required to close an iterator explicitly, since it will be closed when it goes out of scope (or is garbage collected). However, due to the way LevelDB is designed, each iterator operates on an implicit database snapshot, which can be an expensive resource depending on how the database is used during the iterator's lifetime. The :py:meth:`Iterator.close` method gives explicit control over when those resources are released:: >>> it = db.iterator() >>> it.close() Alternatively, to ensure that an iterator is immediately closed after use, you can also use it as a context manager using the ``with`` statement:: >>> with db.iterator() as it: ... for k, v in it: ... pass Non-linear iteration -------------------- In the examples above, we've only used Python's standard iteration methods using a ``for`` loop and the :py:func:`list` constructor. This suffices for most applications, but sometimes more advanced iterator tricks can be useful. Plyvel exposes pretty much all features of the LevelDB iterators using extra functions on the :py:class:`Iterator` instance that :py:meth:`DB.iterator` and :py:meth:`Snapshot.iterator` returns. For instance, you can step forward and backward over the same iterator. For forward stepping, Python's standard :py:func:`next` built-in function can be used (this is also what a standard ``for`` loop does). For backward stepping, you will need to call the :py:meth:`~Iterator.prev()` method on the iterator:: >>> it = db.iterator(include_value=False) >>> next(it) 'key-1' >>> next(it) 'key-2' >>> next(it) 'key-3' >>> it.prev() 'key-3' >>> next(it) 'key-3' >>> next(it) 'key-4' >>> next(it) 'key-5' >>> next(it) Traceback (most recent call last): ... StopIteration >>> it.prev() 'key-5' Note that for reverse iterators, the definition of 'forward' and 'backward' is inverted, i.e. calling ``next(it)`` on a reverse iterator will return the key that sorts *before* the key that was most recently returned. Additionally, Plyvel supports seeking on iterators:: >>> it = db.iterator(include_value=False) >>> it.seek(b'key-3') >>> next(it) 'key-3' >>> it.seek_to_start() >>> next(it) 'key-1' See the :py:class:`Iterator` API reference for more information about advanced iterator usage. Raw iterators ------------- In addition to the iterators describe above, which adhere to the Python iterator protocol, there is also a *raw iterator* API that mimics the C++ iterator API provided by LevelDB. Since this interface is only intended for advanced use cases, it is not covered in this user guide. See the API reference for :py:meth:`DB.raw_iterator` and :py:class:`RawIterator` for more information. Prefixed databases ================== LevelDB databases have a single key space. A common way to split a LevelDB database into separate partitions is to use a prefix for each partition. Plyvel makes this very easy to do using the :py:meth:`DB.prefixed_db` method: >>> my_sub_db = db.prefixed_db(b'example-') The ``my_sub_db`` variable in this example points to an instance of the :py:class:`PrefixedDB` class. This class behaves mostly like a normal Plyvel :py:class:`DB` instance, but all operations will transparently add the key prefix to all keys that it accepts (e.g. in :py:meth:`PrefixedDB.get`), and strip the key prefix from all keys that it returns (e.g. from :py:meth:`PrefixedDB.iterator`). Examples:: >>> my_sub_db.get(b'some-key') # this looks up b'example-some-key' >>> my_sub_db.put(b'some-key', b'value') # this sets b'example-some-key' Almost all functionality available on :py:class:`DB` is also available from :py:class:`PrefixedDB`: write batches, iterators, snapshots, and also iterators over snapshots. A :py:class:`PrefixedDB` is simply a lightweight object that delegates to the the real :py:class:`DB`, which is accessible using the :py:attr:`~PrefixedDB.db` attribute: >>> real_db = my_sub_db.db You can even nest key spaces by creating prefixed prefixed databases using :py:meth:`PrefixedDB.prefixed_db`: >>> my_sub_sub_db = my_sub_db.prefixed_db(b'other-prefix') Custom comparators ================== LevelDB provides an ordered data store, which means all keys are stored in sorted order. By default, a byte-wise comparator that works like :c:func:`strcmp()` is used, but this behaviour can be changed by providing a custom comparator. Plyvel allows you to use a Python callable as a custom LevelDB comparator. The signature for a comparator callable is simple: it takes two byte strings and should return either a positive number, zero, or a negative number, depending on whether the first byte string is greater than, equal to or less than the second byte string. (These are the same semantics as the built-in :py:func:`cmp()`, which has been removed in Python 3 in favour of the so-called ‘rich’ comparison methods.) A simple comparator function for case insensitive comparisons might look like this:: def comparator(a, b): a = a.lower() b = b.lower() if a < b: # a sorts before b return -1 if a > b: # a sorts after b return 1 # a and b are equal return 0 (This is a toy example. It only works properly for byte strings with characters in the ASCII range.) To use this comparator, pass the `comparator` and `comparator_name` arguments to the :py:class:`DB` constructor:: >>> db = DB('/path/to/database/', ... comparator=comparator, # the function defined above ... comparator_name=b'CaseInsensitiveComparator') The comparator name, which must be a byte string, will be stored in the database. LevelDB refuses to open existing databases if the provided comparator name does not match the one in the database. LevelDB invokes the comparator callable repeatedly during many of its operations, including storing and retrieving data, but also during background compactions. Background compaction uses threads that are ‘invisible’ from Python. This means that custom comparator callables *must not* raise any exceptions, since there is no proper way to recover from those. If an exception happens nonetheless, Plyvel will print the traceback to `stderr` and immediately abort your program to avoid database corruption. A final thing to keep in mind is that custom comparators written in Python come with a considerable performance impact. Experiments with simple Python comparator functions like the example above show a 4× slowdown for bulk writes compared to the built-in LevelDB comparator. .. rubric:: Next steps The user guide should be enough to get you started with Plyvel. A complete description of the Plyvel API is available from the :doc:`API reference `. .. vim: set spell spelllang=en: plyvel-1.5.1/plyvel/000077500000000000000000000000001455126310100143425ustar00rootroot00000000000000plyvel-1.5.1/plyvel/__init__.py000066400000000000000000000005231455126310100164530ustar00rootroot00000000000000""" Plyvel, a fast and feature-rich Python interface to LevelDB. """ # Only import the symbols that are part of the public API from ._plyvel import ( # noqa __leveldb_version__, DB, repair_db, destroy_db, Error, IOError, CorruptionError, IteratorInvalidError, ) from ._version import __version__ # noqa plyvel-1.5.1/plyvel/_plyvel.pyx000066400000000000000000001070751455126310100165700ustar00rootroot00000000000000# cython: embedsignature=True, language_level=3 # # Note about API documentation: # # The API reference for all classes and methods is maintained in # a separate file: doc/api.rst. The Sphinx 'autodoc' feature does not # work too well for this project (requires module compilation, chokes on # syntax differences, does not work with documentation hosting sites). # Make sure the API reference and the actual code are kept in sync! # """ Plyvel, a Python LevelDB interface. Use plyvel.DB() to create or open a database. """ import sys import threading from weakref import ref as weakref_ref cimport cython from cpython cimport bool from cpython.buffer cimport ( Py_buffer, PyObject_GetBuffer, PyBuffer_Release, PyBUF_SIMPLE, ) from libc.stdint cimport uint64_t from libc.stdlib cimport malloc, free from libc.string cimport const_char from libcpp.string cimport string from libcpp cimport bool as c_bool cimport plyvel.leveldb as leveldb from plyvel.leveldb cimport ( BytewiseComparator, Cache, Comparator, DestroyDB, NewBloomFilterPolicy, NewLRUCache, Options, Range, ReadOptions, RepairDB, Slice, Status, WriteOptions, ) from plyvel.comparator cimport NewPlyvelCallbackComparator __leveldb_version__ = '%d.%d' % (leveldb.kMajorVersion, leveldb.kMinorVersion) # # Errors and error handling # class Error(Exception): pass class IOError(Error, IOError): pass class CorruptionError(Error): pass class IteratorInvalidError(Error): pass cdef int raise_for_status(Status st) except -1: if st.ok(): return 0 if st.IsIOError(): raise IOError(st.ToString()) if st.IsCorruption(): raise CorruptionError(st.ToString()) # Generic fallback raise Error(st.ToString()) # # Utilities # cdef inline db_get(DB db, bytes key, object default, ReadOptions read_options): cdef string value cdef Status st cdef Slice key_slice = Slice(key, len(key)) with nogil: st = db._db.Get(read_options, key_slice, &value) if st.IsNotFound(): return default raise_for_status(st) return value cdef bytes to_file_system_name(name): if isinstance(name, bytes): return name if not isinstance(name, unicode): raise TypeError( "'name' arg must be a byte string or a unicode string") encoding = sys.getfilesystemencoding() or 'ascii' try: return name.encode(encoding) except UnicodeEncodeError as exc: raise ValueError( "Cannot convert unicode 'name' to a file system name: %s" % exc) cdef bytes bytes_increment(bytes s): # Increment the last byte that is not 0xff, and returned a new byte # string truncated after the position that was incremented. We use # a temporary bytearray to construct a new byte string, since that # works the same in Python 2 and Python 3. b = bytearray(s) cdef int i = len(s) - 1 while i >= 0: if b[i] == 0xff: i = i - 1 continue # Found byte smaller than 0xff: increment and truncate b[i] += 1 return bytes(b[:i + 1]) # Input contained only 0xff bytes return None cdef int parse_options(Options *options, c_bool create_if_missing, c_bool error_if_exists, object paranoid_checks, object write_buffer_size, object max_open_files, object lru_cache_size, object block_size, object block_restart_interval, object max_file_size, object compression, int bloom_filter_bits, object comparator, bytes comparator_name) except -1: cdef size_t c_lru_cache_size options.create_if_missing = create_if_missing options.error_if_exists = error_if_exists if paranoid_checks is not None: options.paranoid_checks = paranoid_checks if write_buffer_size is not None: options.write_buffer_size = write_buffer_size if max_open_files is not None: options.max_open_files = max_open_files if lru_cache_size is not None: c_lru_cache_size = lru_cache_size with nogil: options.block_cache = NewLRUCache(c_lru_cache_size) if block_size is not None: options.block_size = block_size if block_restart_interval is not None: options.block_restart_interval = block_restart_interval if max_file_size is not None: options.max_file_size = max_file_size if compression is None: options.compression = leveldb.kNoCompression else: if isinstance(compression, bytes): compression = compression.decode('UTF-8') if not isinstance(compression, unicode): raise TypeError("'compression' must be None or a string") if compression == u'snappy': options.compression = leveldb.kSnappyCompression else: raise ValueError("'compression' must be None or 'snappy'") if bloom_filter_bits > 0: with nogil: options.filter_policy = NewBloomFilterPolicy(bloom_filter_bits) if (comparator is None) != (comparator_name is None): raise ValueError( "'comparator' and 'comparator_name' must be specified together") if comparator is not None: if not callable(comparator): raise TypeError("custom comparator object must be callable") options.comparator = NewPlyvelCallbackComparator( comparator_name, comparator) # # Database # @cython.final cdef class DB: cdef leveldb.DB* _db cdef Options options cdef readonly object name cdef object lock cdef dict iterators def __init__(self, name, *, bool create_if_missing=False, bool error_if_exists=False, paranoid_checks=None, write_buffer_size=None, max_open_files=None, lru_cache_size=None, block_size=None, block_restart_interval=None, max_file_size=None, compression='snappy', int bloom_filter_bits=0, object comparator=None, bytes comparator_name=None): cdef Status st cdef string fsname self.name = name fsname = to_file_system_name(name) parse_options( &self.options, create_if_missing, error_if_exists, paranoid_checks, write_buffer_size, max_open_files, lru_cache_size, block_size, block_restart_interval, max_file_size, compression, bloom_filter_bits, comparator, comparator_name) with nogil: st = leveldb.DB_Open(self.options, fsname, &self._db) raise_for_status(st) # Keep weak references to open iterators, since deleting a C++ # DB instance results in a segfault if associated Iterator # instances are not deleted beforehand (as mentioned in # leveldb/db.h). We don't use weakref.WeakValueDictionary here # for performance reasons. self.lock = threading.Lock() self.iterators = dict() cpdef close(self): # If the constructor raised an exception (and hence never # completed), self.iterators can be None. In that case no # iterators need to be cleaned anyway. cdef BaseIterator iterator if self.iterators is not None: with self.lock: while self.iterators: iterator = self.iterators.popitem()[1]() if iterator is not None: iterator.close() if self._db is not NULL: del self._db self._db = NULL if self.options.block_cache is not NULL: del self.options.block_cache self.options.block_cache = NULL if self.options.filter_policy is not NULL: del self.options.filter_policy self.options.filter_policy = NULL if self.options.comparator is not NULL: # The built-in BytewiseComparator must not be deleted if self.options.comparator is not BytewiseComparator(): del self.options.comparator self.options.comparator = NULL property closed: def __get__(self): return self._db is NULL def __dealloc__(self): self.close() def __repr__(self): return '' % ( self.name, ' (closed)' if self.closed else '', hex(id(self)), ) def get(self, bytes key not None, default=None, *, bool verify_checksums=False, bool fill_cache=True): if self._db is NULL: raise RuntimeError("Database is closed") cdef ReadOptions read_options read_options.verify_checksums = verify_checksums read_options.fill_cache = fill_cache return db_get(self, key, default, read_options) def put(self, bytes key not None, value not None, *, bool sync=False): if self._db is NULL: raise RuntimeError("Database is closed") cdef WriteOptions write_options = WriteOptions() write_options.sync = sync cdef Slice key_slice = Slice(key, len(key)) cdef Py_buffer value_buffer cdef Status st PyObject_GetBuffer(value, &value_buffer, PyBUF_SIMPLE) try: with nogil: st = self._db.Put( write_options, key_slice, Slice(value_buffer.buf, value_buffer.len)) finally: PyBuffer_Release(&value_buffer) raise_for_status(st) def delete(self, bytes key not None, *, bool sync=False): if self._db is NULL: raise RuntimeError("Database is closed") cdef Status st cdef WriteOptions write_options write_options.sync = sync cdef Slice key_slice = Slice(key, len(key)) with nogil: st = self._db.Delete(write_options, key_slice) raise_for_status(st) def write_batch(self, *, bool transaction=False, bool sync=False): if self._db is NULL: raise RuntimeError("Database is closed") return WriteBatch(self, None, transaction, sync) def __iter__(self): if self._db is NULL: raise RuntimeError("Database is closed") return self.iterator() def iterator(self, *, reverse=False, start=None, stop=None, include_start=True, include_stop=False, prefix=None, include_key=True, include_value=True, bool verify_checksums=False, bool fill_cache=True): return Iterator( self, # db None, # db_prefix reverse, start, stop, include_start, include_stop, prefix, include_key, include_value, verify_checksums, fill_cache, None, # snapshot ) def raw_iterator(self, *, bool verify_checksums=False, bool fill_cache=True): return RawIterator( self, # db verify_checksums, fill_cache, None, # snapshot ) def snapshot(self): return Snapshot(db=self) def get_property(self, bytes name not None): if self._db is NULL: raise RuntimeError("Database is closed") cdef Slice sl = Slice(name, len(name)) cdef string value cdef c_bool result with nogil: result = self._db.GetProperty(sl, &value) return value if result else None def compact_range(self, *, bytes start=None, bytes stop=None): if self._db is NULL: raise RuntimeError("Database is closed") cdef Slice start_slice cdef Slice stop_slice if start is not None: start_slice = Slice(start, len(start)) if stop is not None: stop_slice = Slice(stop, len(stop)) with nogil: self._db.CompactRange(&start_slice, &stop_slice) def approximate_size(self, bytes start not None, bytes stop not None): if self._db is NULL: raise RuntimeError("Database is closed") return self.approximate_sizes((start, stop))[0] def approximate_sizes(self, *ranges): if self._db is NULL: raise RuntimeError("Database is closed") cdef int n_ranges = len(ranges) cdef Range *c_ranges = malloc(n_ranges * sizeof(Range)) cdef uint64_t *sizes = malloc(n_ranges * sizeof(uint64_t)) try: for i in xrange(n_ranges): start, stop = ranges[i] if not isinstance(start, bytes) or not isinstance(stop, bytes): raise TypeError( "Start and stop of range must be byte strings") c_ranges[i] = Range( Slice(start, len(start)), Slice(stop, len(stop))) with nogil: self._db.GetApproximateSizes(c_ranges, n_ranges, sizes) return [sizes[i] for i in xrange(n_ranges)] finally: free(c_ranges) free(sizes) def prefixed_db(self, bytes prefix not None): return PrefixedDB(db=self, prefix=prefix) def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): self.close() cdef class PrefixedDB: cdef readonly DB db cdef readonly bytes prefix def __init__(self, *, DB db not None, bytes prefix not None): self.db = db self.prefix = prefix def __repr__(self): return '' % ( self.prefix, hex(id(self)), ) def get(self, bytes key not None, default=None, *, bool verify_checksums=False, bool fill_cache=True): return self.db.get( self.prefix + key, default=default, verify_checksums=verify_checksums, fill_cache=fill_cache) def put(self, bytes key not None, value not None, *, bool sync=False): return self.db.put(self.prefix + key, value, sync=sync) def delete(self, bytes key not None, *, bool sync=False): return self.db.delete(self.prefix + key, sync=sync) def write_batch(self, *, transaction=False, bool sync=False): return WriteBatch(self.db, self.prefix, transaction, sync) def __iter__(self): return self.iterator() def iterator(self, *, reverse=False, start=None, stop=None, include_start=True, include_stop=False, prefix=None, include_key=True, include_value=True, bool verify_checksums=False, bool fill_cache=True): return Iterator( self.db, self.prefix, reverse, start, stop, include_start, include_stop, prefix, include_key, include_value, verify_checksums, fill_cache, None, # snapshot ) def snapshot(self): return Snapshot(db=self.db, prefix=self.prefix) def prefixed_db(self, bytes prefix not None): return PrefixedDB(db=self.db, prefix=self.prefix + prefix) def repair_db(name, *, paranoid_checks=None, write_buffer_size=None, max_open_files=None, lru_cache_size=None, block_size=None, block_restart_interval=None, max_file_size=None, compression='snappy', int bloom_filter_bits=0, comparator=None, bytes comparator_name=None): cdef Options options = Options() cdef Status st cdef string fsname fsname = to_file_system_name(name) create_if_missing = False error_if_exists = True parse_options( &options, create_if_missing, error_if_exists, paranoid_checks, write_buffer_size, max_open_files, lru_cache_size, block_size, block_restart_interval, max_file_size, compression, bloom_filter_bits, comparator, comparator_name) with nogil: st = RepairDB(fsname, options) raise_for_status(st) def destroy_db(name): cdef Options options = Options() cdef Status st cdef string fsname fsname = to_file_system_name(name) with nogil: st = DestroyDB(fsname, options) raise_for_status(st) # # Write batch # @cython.final cdef class WriteBatch: cdef leveldb.WriteBatch* _write_batch cdef WriteOptions write_options cdef DB db cdef bytes prefix cdef c_bool transaction def __init__(self, DB db not None, bytes prefix, bool transaction, sync): self.db = db self.prefix = prefix self.transaction = transaction self.write_options = WriteOptions() if sync is not None: self.write_options.sync = sync self._write_batch = new leveldb.WriteBatch() def __dealloc__(self): del self._write_batch def put(self, bytes key not None, value not None): if self.db._db is NULL: raise RuntimeError("Database is closed") if self.prefix is not None: key = self.prefix + key cdef Slice key_slice = Slice(key, len(key)) cdef Py_buffer value_buffer PyObject_GetBuffer(value, &value_buffer, PyBUF_SIMPLE) try: with nogil: self._write_batch.Put( key_slice, Slice(value_buffer.buf, value_buffer.len)) finally: PyBuffer_Release(&value_buffer) def delete(self, bytes key not None): if self.db._db is NULL: raise RuntimeError("Database is closed") if self.prefix is not None: key = self.prefix + key cdef Slice key_slice = Slice(key, len(key)) with nogil: self._write_batch.Delete(key_slice) def clear(self): if self.db._db is NULL: raise RuntimeError("Database is closed") with nogil: self._write_batch.Clear() def write(self): if self.db._db is NULL: raise RuntimeError("Database is closed") cdef Status st with nogil: st = self.db._db.Write(self.write_options, self._write_batch) raise_for_status(st) def approximate_size(self): if self.db._db is NULL: raise RuntimeError("Database is closed") return self._write_batch.ApproximateSize() def append(self, WriteBatch source not None): if self.db._db is NULL: raise RuntimeError("Database is closed") self._write_batch.Append(source._write_batch[0]) def __enter__(self): if self.db._db is NULL: raise RuntimeError("Database is closed") return self def __exit__(self, exc_type, exc_val, exc_tb): if self.db._db is NULL: raise RuntimeError("Database is closed") if self.transaction and exc_type is not None: # Exception occurred in transaction; do not write the batch self.clear() return self.write() self.clear() # # Iterator # cdef enum IteratorState: BEFORE_START AFTER_STOP IN_BETWEEN IN_BETWEEN_ALREADY_POSITIONED cdef enum IteratorDirection: FORWARD REVERSE cdef class BaseIterator: cdef DB db cdef leveldb.Iterator* _iter # Iterators need to be weak referencable to ensure a proper cleanup # from DB.close() cdef object __weakref__ def __init__(self, DB db, bool verify_checksums, bool fill_cache, Snapshot snapshot): if db._db is NULL: raise RuntimeError("Database or iterator is closed") self.db = db cdef ReadOptions read_options read_options.verify_checksums = verify_checksums read_options.fill_cache = fill_cache if snapshot is not None: read_options.snapshot = snapshot._snapshot with nogil: self._iter = db._db.NewIterator(read_options) # Store a weak reference on the db (needed when closing db) iterator_id = id(self) ref_dict = db.iterators ref_dict[iterator_id] = weakref_ref( self, lambda wr: ref_dict.pop(iterator_id)) cpdef close(self): if self._iter is not NULL: del self._iter self._iter = NULL def __dealloc__(self): self.close() def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): self.close() return False # propagate exceptions @cython.final cdef class Iterator(BaseIterator): cdef IteratorDirection direction cdef IteratorState state cdef Comparator* comparator cdef bytes start cdef bytes stop cdef Slice start_slice cdef Slice stop_slice cdef c_bool include_start cdef c_bool include_stop cdef c_bool include_key cdef c_bool include_value cdef bytes db_prefix cdef size_t db_prefix_len def __init__(self, DB db, bytes db_prefix, bool reverse, bytes start, bytes stop, bool include_start, bool include_stop, bytes prefix, bool include_key, bool include_value, bool verify_checksums, bool fill_cache, Snapshot snapshot): super(Iterator, self).__init__( db=db, verify_checksums=verify_checksums, fill_cache=fill_cache, snapshot=snapshot) self.comparator = db.options.comparator self.direction = FORWARD if not reverse else REVERSE if db_prefix is None: self.db_prefix_len = 0 else: # This is an iterator on a PrefixedDB. self.db_prefix = db_prefix self.db_prefix_len = len(db_prefix) # Transform args so that the database key prefix is taken # into account. if prefix is not None: # Both database key prefix and prefix on the iterator prefix = db_prefix + prefix else: # Adapt start and stop keys to use the database key # prefix. if start is None: start = db_prefix include_start = True else: start = db_prefix + start if stop is None: stop = bytes_increment(db_prefix) include_stop = False else: stop = db_prefix + stop if prefix is not None: if start is not None or stop is not None: raise TypeError( "'prefix' cannot be used together with 'start' or 'stop'") # Use prefix to construct start and stop keys, and ignore # include_start and include_stop args start = prefix stop = bytes_increment(prefix) include_start = True include_stop = False if start is not None: self.start = start self.start_slice = Slice(start, len(start)) if stop is not None: self.stop = stop self.stop_slice = Slice(stop, len(stop)) self.include_start = include_start self.include_stop = include_stop self.include_key = include_key self.include_value = include_value if self.direction == FORWARD: self.seek_to_start() else: self.seek_to_stop() raise_for_status(self._iter.status()) def __iter__(self): return self cdef object current(self): """Return the current iterator key/value. This is an internal helper function that is not exposed in the external Python API. """ cdef Slice key_slice cdef bytes key = None cdef Slice value_slice cdef bytes value = None # Only build Python strings that will be returned. Also chop off # the db prefix (for PrefixedDB iterators). if self.include_key: key_slice = self._iter.key() key = key_slice.data()[self.db_prefix_len:key_slice.size()] if self.include_value: value_slice = self._iter.value() value = value_slice.data()[:value_slice.size()] if self.include_key and self.include_value: return (key, value) if self.include_key: return key if self.include_value: return value return None def __next__(self): """Return the next iterator entry. Note: Cython will also create a .next() method that does the same as this method. """ if self.direction == FORWARD: return self.real_next() else: return self.real_prev() def prev(self): if self.direction == FORWARD: return self.real_prev() else: return self.real_next() cdef real_next(self): if self._iter is NULL: raise RuntimeError("Database or iterator is closed") if self.state == IN_BETWEEN: with nogil: self._iter.Next() if not self._iter.Valid(): self.state = AFTER_STOP raise StopIteration elif self.state == IN_BETWEEN_ALREADY_POSITIONED: self.state = IN_BETWEEN elif self.state == BEFORE_START: if self.start is None: with nogil: self._iter.SeekToFirst() else: with nogil: self._iter.Seek(self.start_slice) if not self._iter.Valid(): # Iterator is empty raise StopIteration if self.start is not None and not self.include_start: # Start key is excluded, so skip past it if the db # contains it. if self.comparator.Compare(self._iter.key(), self.start_slice) == 0: with nogil: self._iter.Next() if not self._iter.Valid(): raise StopIteration self.state = IN_BETWEEN elif self.state == AFTER_STOP: raise StopIteration raise_for_status(self._iter.status()) # Check range boundaries if self.stop is not None: n = 1 if self.include_stop else 0 if self.comparator.Compare(self._iter.key(), self.stop_slice) >= n: self.state = AFTER_STOP raise StopIteration return self.current() cdef real_prev(self): if self._iter is NULL: raise RuntimeError("Database or iterator is closed") if self.state == IN_BETWEEN: pass elif self.state == IN_BETWEEN_ALREADY_POSITIONED: assert self._iter.Valid() with nogil: self._iter.Prev() if not self._iter.Valid(): # The .seek() resulted in the first key in the database self.state = BEFORE_START raise StopIteration raise_for_status(self._iter.status()) elif self.state == BEFORE_START: raise StopIteration elif self.state == AFTER_STOP: if self.stop is None: # No stop key, seek to last entry with nogil: self._iter.SeekToLast() else: # Seek to stop key with nogil: self._iter.Seek(self.stop_slice) if self._iter.Valid(): # Move one step back if stop is exclusive. if not self.include_stop: with nogil: self._iter.Prev() else: # Stop key did not exist; position at the last # database entry instead. with nogil: self._iter.SeekToLast() # Make sure the iterator is not past the stop key if self._iter.Valid() and self.comparator.Compare(self._iter.key(), self.stop_slice) > 0: with nogil: self._iter.Prev() if not self._iter.Valid(): # No entries left raise StopIteration # After all the stepping back, we might even have ended up # *before* the start key. In this case the iterator does not # yield any items. if self.start is not None and self.comparator.Compare(self.start_slice, self._iter.key()) >= 0: raise StopIteration raise_for_status(self._iter.status()) # Unlike .real_next(), first obtain the value, then move the # iterator pointer (not the other way around), so that # repeatedly calling it.prev() and next(it) will work as # designed. out = self.current() with nogil: self._iter.Prev() if not self._iter.Valid(): # Moved before the first key in the database self.state = BEFORE_START else: if self.start is None: # Iterator is valid self.state = IN_BETWEEN else: # Check range boundaries n = 0 if self.include_start else 1 if self.comparator.Compare( self._iter.key(), self.start_slice) >= n: # Iterator is valid and within range boundaries self.state = IN_BETWEEN else: # Iterator is valid, but has moved before the # 'start' key self.state = BEFORE_START raise_for_status(self._iter.status()) return out def seek_to_start(self): if self._iter is NULL: raise RuntimeError("Database or iterator is closed") self.state = BEFORE_START def seek_to_stop(self): if self._iter is NULL: raise RuntimeError("Database or iterator is closed") self.state = AFTER_STOP def seek(self, bytes target not None): if self._iter is NULL: raise RuntimeError("Database or iterator is closed") if self.db_prefix is not None: target = self.db_prefix + target cdef Slice target_slice = Slice(target, len(target)) # Seek only within the start/stop boundaries if self.start is not None and self.comparator.Compare( target_slice, self.start_slice) < 0: target_slice = self.start_slice if self.stop is not None and self.comparator.Compare( target_slice, self.stop_slice) > 0: target_slice = self.stop_slice with nogil: self._iter.Seek(target_slice) if not self._iter.Valid(): # Moved past the end (or empty database) self.state = AFTER_STOP return self.state = IN_BETWEEN_ALREADY_POSITIONED raise_for_status(self._iter.status()) @cython.final cdef class RawIterator(BaseIterator): def valid(self): if self._iter is NULL: raise RuntimeError("Database or iterator is closed") return self._iter.Valid() def seek_to_first(self): if self._iter is NULL: raise RuntimeError("Database or iterator is closed") with nogil: self._iter.SeekToFirst() raise_for_status(self._iter.status()) def seek_to_last(self): if self._iter is NULL: raise RuntimeError("Database or iterator is closed") with nogil: self._iter.SeekToLast() raise_for_status(self._iter.status()) def seek(self, bytes target not None): if self._iter is NULL: raise RuntimeError("Database or iterator is closed") cdef Slice target_slice = Slice(target, len(target)) with nogil: self._iter.Seek(target_slice) raise_for_status(self._iter.status()) def next(self): if self._iter is NULL: raise RuntimeError("Database or iterator is closed") if not self._iter.Valid(): raise IteratorInvalidError() with nogil: self._iter.Next() raise_for_status(self._iter.status()) def prev(self): if self._iter is NULL: raise RuntimeError("Database or iterator is closed") if not self._iter.Valid(): raise IteratorInvalidError() with nogil: self._iter.Prev() raise_for_status(self._iter.status()) cpdef key(self): if self._iter is NULL: raise RuntimeError("Database or iterator is closed") if not self._iter.Valid(): raise IteratorInvalidError() cdef Slice key_slice key_slice = self._iter.key() return key_slice.data()[:key_slice.size()] cpdef value(self): if self._iter is NULL: raise RuntimeError("Database or iterator is closed") if not self._iter.Valid(): raise IteratorInvalidError() cdef Slice value_slice value_slice = self._iter.value() return value_slice.data()[:value_slice.size()] def item(self): return self.key(), self.value() # # Snapshot # @cython.final cdef class Snapshot: cdef leveldb.Snapshot* _snapshot cdef DB db cdef bytes prefix def __init__(self, *, DB db not None, bytes prefix=None): if db._db is NULL: raise RuntimeError("Cannot operate on closed LevelDB database") self.db = db self.prefix = prefix with nogil: self._snapshot = db._db.GetSnapshot() def __dealloc__(self): self.close() cpdef close(self): if self.db._db is NULL or self._snapshot is NULL: return # nothing to do with nogil: self.db._db.ReleaseSnapshot(self._snapshot) self._snapshot = NULL def release(self): self.close() def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): self.close() return False # propagate exceptions def get(self, bytes key not None, default=None, *, bool verify_checksums=False, bool fill_cache=True): if self.db._db is NULL or self._snapshot is NULL: raise RuntimeError("Database or snapshot is closed") cdef ReadOptions read_options read_options.verify_checksums = verify_checksums read_options.fill_cache = fill_cache read_options.snapshot = self._snapshot if self.prefix is not None: key = self.prefix + key return db_get(self.db, key, default, read_options) def __iter__(self): return self.iterator() def iterator(self, *, reverse=False, start=None, stop=None, include_start=True, include_stop=False, prefix=None, include_key=True, include_value=True, bool verify_checksums=False, bool fill_cache=True): if self.db._db is NULL or self._snapshot is NULL: raise RuntimeError("Database or snapshot is closed") return Iterator( db=self.db, db_prefix=self.prefix, reverse=reverse, start=start, stop=stop, include_start=include_start, include_stop=include_stop, prefix=prefix, include_key=include_key, include_value=include_value, verify_checksums=verify_checksums, fill_cache=fill_cache, snapshot=self) def raw_iterator(self, *, bool verify_checksums=False, bool fill_cache=True): if self.db._db is NULL or self._snapshot is NULL: raise RuntimeError("Database or snapshot is closed") return RawIterator( db=self.db, verify_checksums=verify_checksums, fill_cache=fill_cache, snapshot=self) plyvel-1.5.1/plyvel/_version.py000066400000000000000000000002761455126310100165450ustar00rootroot00000000000000""" Plyvel version module. """ # Note: don't add any non-trivial logic here; this file is also loaded # from setup.py file when the module has not yet been compiled! __version__ = '1.5.1' plyvel-1.5.1/plyvel/comparator.cpp000066400000000000000000000057631455126310100172300ustar00rootroot00000000000000/* * Custom Python comparator callback support code for Plyvel. */ #include "Python.h" #include #include #include #include "comparator.h" class PlyvelCallbackComparator : public leveldb::Comparator { public: PlyvelCallbackComparator(const char* name, PyObject* comparator) : name(name), comparator(comparator) { Py_INCREF(comparator); zero = PyLong_FromLong(0); /* LevelDB uses a background thread for compaction, and with custom * comparators this background thread calls back into Python code, * which means the GIL must be initialized. */ PyEval_InitThreads(); } ~PlyvelCallbackComparator() { Py_DECREF(comparator); Py_DECREF(zero); } void bailout(const char* message) const { PyErr_Print(); std::cerr << "FATAL ERROR: " << message << std::endl; std::cerr << "Aborting to avoid database corruption..." << std::endl; abort(); } int Compare(const leveldb::Slice& a, const leveldb::Slice& b) const { int ret; PyObject* bytes_a; PyObject* bytes_b; PyObject* compare_result; PyGILState_STATE gstate; gstate = PyGILState_Ensure(); /* Create two Python byte strings */ bytes_a = PyBytes_FromStringAndSize(a.data(), a.size()); bytes_b = PyBytes_FromStringAndSize(b.data(), b.size()); if ((bytes_a == NULL) || (bytes_b == NULL)) { this->bailout("Plyvel comparator could not allocate byte strings"); } /* Invoke comparator callable */ compare_result = PyObject_CallFunctionObjArgs(comparator, bytes_a, bytes_b, 0); if (compare_result == NULL) { this->bailout("Exception raised from custom Plyvel comparator"); } /* The comparator callable can return any Python object. Compare it * to our "0" value to get a -1, 0, or 1 for LevelDB. */ if (PyObject_RichCompareBool(compare_result, zero, Py_GT) == 1) { ret = 1; } else if (PyObject_RichCompareBool(compare_result, zero, Py_LT) == 1) { ret = -1; } else { ret = 0; } if (PyErr_Occurred()) { this->bailout("Exception raised while comparing custom Plyvel comparator result with 0"); } Py_DECREF(compare_result); Py_DECREF(bytes_a); Py_DECREF(bytes_b); PyGILState_Release(gstate); return ret; } const char* Name() const { return name.c_str(); } void FindShortestSeparator(std::string*, const leveldb::Slice&) const { } void FindShortSuccessor(std::string*) const { } private: std::string name; PyObject* comparator; PyObject* zero; }; /* * This function is the only API used by the Plyvel Cython code. */ leveldb::Comparator* NewPlyvelCallbackComparator(const char* name, PyObject* comparator) { return new PlyvelCallbackComparator(name, comparator); } plyvel-1.5.1/plyvel/comparator.h000066400000000000000000000002741455126310100166650ustar00rootroot00000000000000#ifndef PLYVEL_COMPARATOR_H #define PLYVEL_COMPARATOR_H #include leveldb::Comparator* NewPlyvelCallbackComparator(const char* name, PyObject* comparator); #endif plyvel-1.5.1/plyvel/comparator.pxd000066400000000000000000000003351455126310100172270ustar00rootroot00000000000000# distutils: language = c++ from libc.string cimport const_char from .leveldb cimport Comparator cdef extern from "comparator.h": Comparator* NewPlyvelCallbackComparator(const_char* name, object comparator) nogil plyvel-1.5.1/plyvel/leveldb.pxd000066400000000000000000000107331455126310100165000ustar00rootroot00000000000000# distutils: language = c++ # distutils: libraries = leveldb from libc.stdint cimport uint64_t from libc.string cimport const_char from libcpp cimport bool from libcpp.string cimport string cdef extern from "leveldb/db.h" namespace "leveldb": int kMajorVersion int kMinorVersion cdef cppclass Snapshot: pass cdef cppclass Range: Slice start Slice limit Range() nogil Range(Slice& s, Slice& l) nogil cdef cppclass DB: Status Put(WriteOptions& options, Slice& key, Slice& value) nogil Status Delete(WriteOptions& options, Slice& key) nogil Status Write(WriteOptions& options, WriteBatch* updates) nogil Status Get(ReadOptions& options, Slice& key, string* value) nogil Iterator* NewIterator(ReadOptions& options) nogil Snapshot* GetSnapshot() nogil void ReleaseSnapshot(Snapshot* snapshot) nogil bool GetProperty(Slice& property, string* value) nogil void GetApproximateSizes(Range* range, int n, uint64_t* sizes) nogil void CompactRange(Slice* begin, Slice* end) nogil # The DB::open() method is static, and hence not a member of the DB # class defined above Status DB_Open "leveldb::DB::Open"(Options& options, string& name, DB** dbptr) nogil cdef Status DestroyDB(string& name, Options& options) nogil cdef Status RepairDB(string& dbname, Options& options) nogil cdef extern from "leveldb/status.h" namespace "leveldb": cdef cppclass Status: bool ok() nogil bool IsNotFound() nogil bool IsCorruption() nogil bool IsIOError() nogil string ToString() nogil cdef extern from "leveldb/options.h" namespace "leveldb": cdef enum CompressionType: kNoCompression kSnappyCompression cdef cppclass Options: Comparator* comparator bool create_if_missing bool error_if_exists bool paranoid_checks # Env* env # Logger* info_log size_t write_buffer_size int max_open_files Cache* block_cache size_t block_size int block_restart_interval size_t max_file_size CompressionType compression FilterPolicy* filter_policy Options() nogil cdef cppclass ReadOptions: bool verify_checksums bool fill_cache Snapshot* snapshot ReadOptions() nogil cdef cppclass WriteOptions: bool sync WriteOptions() nogil cdef extern from "leveldb/slice.h" namespace "leveldb": cdef cppclass Slice: Slice() nogil Slice(const_char* d, size_t n) nogil Slice(string& s) nogil Slice(const_char* s) nogil const_char* data() nogil size_t size() nogil bool empty() nogil # char operator[](size_t n) nogil void clear() nogil void remove_prefix(size_t n) nogil string ToString() nogil int compare(Slice& b) nogil bool starts_with(Slice& x) nogil cdef extern from "leveldb/write_batch.h" namespace "leveldb": cdef cppclass WriteBatch: WriteBatch() nogil void Put(Slice& key, Slice& value) nogil void Delete(Slice& key) nogil void Clear() nogil size_t ApproximateSize() nogil void Append(WriteBatch& source) nogil # Status Iterate(Handler* handler) cdef extern from "leveldb/iterator.h" namespace "leveldb": cdef cppclass Iterator: Iterator() nogil bool Valid() nogil void SeekToFirst() nogil void SeekToLast() nogil void Seek(Slice& target) nogil void Next() nogil void Prev() nogil Slice key() nogil Slice value() nogil Status status() nogil # void RegisterCleanup(CleanupFunction function, void* arg1, void* arg2); cdef extern from "leveldb/comparator.h" namespace "leveldb": cdef cppclass Comparator: int Compare(Slice& a, Slice&b) nogil const_char* Name() nogil Comparator* BytewiseComparator() nogil cdef extern from "leveldb/filter_policy.h" namespace "leveldb": cdef cppclass FilterPolicy: const_char* Name() nogil void CreateFilter(Slice* keys, int n, string* dst) nogil bool KeyMayMatch(Slice& key, Slice& filter) nogil FilterPolicy* NewBloomFilterPolicy(int bits_per_key) nogil cdef extern from "leveldb/cache.h" namespace "leveldb": cdef cppclass Cache: # Treat as opaque structure pass Cache* NewLRUCache(size_t capacity) nogil plyvel-1.5.1/requirements-dev.txt000066400000000000000000000001001455126310100170560ustar00rootroot00000000000000-r requirements-test.txt cibuildwheel Cython >= 0.17 sphinx tox plyvel-1.5.1/requirements-test.txt000066400000000000000000000000421455126310100172640ustar00rootroot00000000000000pytest>=3.6 pytest-cov setuptools plyvel-1.5.1/scripts/000077500000000000000000000000001455126310100145165ustar00rootroot00000000000000plyvel-1.5.1/scripts/1.1.9-0001-fix-inlining-failure.patch000066400000000000000000000007561455126310100226110ustar00rootroot00000000000000Fixes the following error: error: inlining failed in call to ‘always_inline’ ‘size_t snappy::AdvanceToNextTag(const uint8_t**, size_t*)’: function body can be overwritten at link time --- snappy-stubs-internal.h +++ snappy-stubs-internal.h @@ -100,7 +100,7 @@ // Inlining hints. #ifdef HAVE_ATTRIBUTE_ALWAYS_INLINE -#define SNAPPY_ATTRIBUTE_ALWAYS_INLINE __attribute__((always_inline)) +#define SNAPPY_ATTRIBUTE_ALWAYS_INLINE #else #define SNAPPY_ATTRIBUTE_ALWAYS_INLINE #endif plyvel-1.5.1/scripts/cibuildwheel-before-build.sh000077500000000000000000000002571455126310100220560ustar00rootroot00000000000000#!/bin/sh set -eux python --version cython --version git clean -xfd make cython if python --version | grep -q -F 3.9; then python setup.py sdist --dist-dir /output fi plyvel-1.5.1/scripts/install-leveldb.sh000077500000000000000000000007671455126310100201500ustar00rootroot00000000000000#!/bin/sh set -eux SUDO=$(command -v sudo || true) LEVELDB_VERSION=1.22 mkdir /opt/leveldb cd /opt/leveldb curl -sL leveldb.tar.gz https://codeload.github.com/google/leveldb/tar.gz/${LEVELDB_VERSION} | tar xzf - cd leveldb-* cmake \ -DCMAKE_BUILD_TYPE=Release \ -DBUILD_SHARED_LIBS=ON \ -DCMAKE_POSITION_INDEPENDENT_CODE=on \ -DLEVELDB_BUILD_TESTS=off \ -DLEVELDB_BUILD_BENCHMARKS=off \ . cmake --build . $SUDO "$(command -v cmake)" --build . --target install $SUDO ldconfig plyvel-1.5.1/scripts/install-snappy.sh000077500000000000000000000012331455126310100200320ustar00rootroot00000000000000#!/bin/sh set -eux SUDO=$(command -v sudo || true) SNAPPY_VERSION=1.1.9 mkdir /opt/snappy cd /opt/snappy curl -sL https://codeload.github.com/google/snappy/tar.gz/${SNAPPY_VERSION} | tar xzf - cd ./snappy-* # See https://github.com/google/snappy/blob/${SNAPPY_VERSION}/.gitmodules git clone --depth 1 \ https://github.com/google/benchmark.git third_party/benchmark git clone --depth 1 \ https://github.com/google/googletest.git third_party/googletest patch < /1.1.9-0001-fix-inlining-failure.patch cmake -DBUILD_SHARED_LIBS=ON -DCMAKE_POSITION_INDEPENDENT_CODE=on . cmake --build . $SUDO "$(command -v cmake)" --build . --target install $SUDO ldconfig plyvel-1.5.1/setup.cfg000066400000000000000000000001251455126310100146460ustar00rootroot00000000000000[build_sphinx] source-dir = doc/ build-dir = doc/build/ [tool:pytest] addopts = -sv plyvel-1.5.1/setup.py000066400000000000000000000034171455126310100145460ustar00rootroot00000000000000from os.path import join, dirname from setuptools import setup from setuptools.extension import Extension import platform CURRENT_DIR = dirname(__file__) with open(join(CURRENT_DIR, 'plyvel/_version.py')) as fp: exec(fp.read(), globals(), locals()) def get_file_contents(filename): with open(join(CURRENT_DIR, filename)) as fp: return fp.read() extra_compile_args = ['-Wall', '-g', '-x', 'c++', '-std=c++11'] if platform.system() == 'Darwin': extra_compile_args += ['-stdlib=libc++'] ext_modules = [ Extension( 'plyvel._plyvel', sources=['plyvel/_plyvel.cpp', 'plyvel/comparator.cpp'], libraries=['leveldb'], extra_compile_args=extra_compile_args, ) ] setup( name='plyvel', description="Plyvel, a fast and feature-rich Python interface to LevelDB", long_description=get_file_contents('README.rst'), url="https://github.com/wbolster/plyvel", version=__version__, # noqa: F821 author="Wouter Bolsterlee", author_email="wouter@bolsterl.ee", ext_modules=ext_modules, packages=['plyvel'], license="BSD License", classifiers=[ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "Intended Audience :: Information Technology", "Intended Audience :: Science/Research", "License :: OSI Approved :: BSD License", "Operating System :: POSIX", "Programming Language :: C++", "Programming Language :: Cython", "Programming Language :: Python", "Programming Language :: Python :: 2", "Programming Language :: Python :: 3", "Topic :: Database", "Topic :: Database :: Database Engines/Servers", "Topic :: Software Development :: Libraries :: Python Modules", ] ) plyvel-1.5.1/test/000077500000000000000000000000001455126310100140065ustar00rootroot00000000000000plyvel-1.5.1/test/__init__.py000066400000000000000000000000001455126310100161050ustar00rootroot00000000000000plyvel-1.5.1/test/test_plyvel.py000066400000000000000000000732231455126310100167410ustar00rootroot00000000000000# encoding: UTF-8 from __future__ import unicode_literals import itertools import os import random import shutil import stat import sys import tempfile import threading import time import pytest import plyvel # # Fixtures # @pytest.fixture def db_dir(request): name = tempfile.mkdtemp() def finalize(): shutil.rmtree(name) request.addfinalizer(finalize) return name @pytest.fixture def db(request): name = tempfile.mkdtemp() db = plyvel.DB(name, create_if_missing=True, error_if_exists=True) def finalize(): db.close() shutil.rmtree(name) request.addfinalizer(finalize) return db # # Actual tests # def test_version(): v = plyvel.__leveldb_version__ assert v.startswith('1.') def test_open_read_only_dir(db_dir): # Opening a DB in a read-only dir should not work os.chmod(db_dir, stat.S_IRUSR | stat.S_IXUSR) with pytest.raises(plyvel.IOError): plyvel.DB(db_dir) def test_open_no_create(db_dir): with pytest.raises(plyvel.Error): plyvel.DB(db_dir, create_if_missing=False) def test_open_fresh(db_dir): db = plyvel.DB(db_dir, create_if_missing=True) db.close() with pytest.raises(plyvel.Error): plyvel.DB(db_dir, error_if_exists=True) def test_open_no_compression(db_dir): plyvel.DB(db_dir, compression=None, create_if_missing=True) def test_open_many_options(db_dir): plyvel.DB( db_dir, create_if_missing=True, error_if_exists=False, paranoid_checks=True, write_buffer_size=16 * 1024 * 1024, max_open_files=512, lru_cache_size=64 * 1024 * 1024, block_size=2 * 1024, block_restart_interval=32, compression='snappy', bloom_filter_bits=10) def test_invalid_open(db_dir): with pytest.raises(TypeError): plyvel.DB(123) with pytest.raises(TypeError): plyvel.DB(db_dir, write_buffer_size='invalid') with pytest.raises(TypeError): plyvel.DB(db_dir, lru_cache_size='invalid') with pytest.raises(ValueError): plyvel.DB(db_dir, compression='invalid', create_if_missing=True) @pytest.mark.skipif(sys.getfilesystemencoding() != 'utf-8', reason="requires UTF-8 file system encoding") def test_open_unicode_name(db_dir): db_dir = os.path.join(db_dir, 'úñîçøđê_name') os.makedirs(db_dir) plyvel.DB(db_dir, create_if_missing=True) def test_open_close(db_dir): # Create a database with options that result in additional object # allocation (e.g. LRU cache). db = plyvel.DB( db_dir, create_if_missing=True, lru_cache_size=1024 * 1024, bloom_filter_bits=10) db.put(b'key', b'value') wb = db.write_batch() sn = db.snapshot() it = db.iterator() snapshot_it = sn.iterator() # Close the database db.close() assert db.closed # Expect runtime errors for operations on the database, with pytest.raises(RuntimeError): db.get(b'key') with pytest.raises(RuntimeError): db.put(b'key', b'value') with pytest.raises(RuntimeError): db.delete(b'key') # ... on write batches, with pytest.raises(RuntimeError): wb.put(b'key', b'value') # ... on snapshots, pytest.raises(RuntimeError, db.snapshot) with pytest.raises(RuntimeError): sn.get(b'key') # ... on iterators, with pytest.raises(RuntimeError): next(it) # ... and on snapshot iterators, with pytest.raises(RuntimeError): next(snapshot_it) def test_large_lru_cache(db_dir): # Use a 2 GB size (does not fit in a 32-bit signed int) plyvel.DB(db_dir, create_if_missing=True, lru_cache_size=2 * 1024**3) def test_put(db): db.put(b'foo', b'bar') db.put(b'foo', b'bar', sync=False) db.put(b'foo', b'bar', sync=True) for i in range(1000): key = ('key-%d' % i).encode('ascii') value = ('value-%d' % i).encode('ascii') db.put(key, value) pytest.raises(TypeError, db.put, b'foo', 12) pytest.raises(TypeError, db.put, 12, 'foo') def test_get(db): key = b'the-key' value = b'the-value' assert db.get(key) is None db.put(key, value) assert db.get(key) == value assert db.get(key, verify_checksums=True) == value assert db.get(key, verify_checksums=False) == value assert db.get(key, verify_checksums=None) == value assert db.get(key, fill_cache=True) == value assert db.get(key, fill_cache=False, verify_checksums=None) == value key2 = b'key-that-does-not-exist' value2 = b'default-value' assert db.get(key2) is None assert db.get(key2, value2) == value2 assert db.get(key2, default=value2) == value2 pytest.raises(TypeError, db.get, 1) pytest.raises(TypeError, db.get, 'key') pytest.raises(TypeError, db.get, None) pytest.raises(TypeError, db.get, b'foo', b'default', True) def test_delete(db): # Put and delete a key key = b'key-that-will-be-deleted' db.put(key, b'') assert db.get(key) is not None db.delete(key) assert db.get(key) is None # The .delete() method also takes write options db.put(key, b'') db.delete(key, sync=True) def test_null_bytes(db): key = b'key\x00\x01' value = b'\x00\x00\x01' db.put(key, value) assert db.get(key) == value db.delete(key) assert db.get(key) is None def test_bytes_like(db): b = b'bar' value = bytearray(b) db.put(b'quux', value) assert db.get(b'quux') == value value = memoryview(b) db.put(b'foo', value) assert db.get(b'foo') == value def test_write_batch(db): # Prepare a batch with some data batch = db.write_batch() for i in range(1000): batch.put(('batch-key-%d' % i).encode('ascii'), b'value') # Delete a key that was also set in the same (pending) batch batch.delete(b'batch-key-2') # The DB should not have any data before the batch is written assert db.get(b'batch-key-1') is None # ...but it should have data afterwards batch.write() assert db.get(b'batch-key-1') is not None assert db.get(b'batch-key-2') is None # Batches can be cleared batch = db.write_batch() batch.put(b'this-is-never-saved', b'') batch.clear() batch.write() assert db.get(b'this-is-never-saved') is None # Batches take write options batch = db.write_batch(sync=True) batch.put(b'batch-key-sync', b'') batch.write() def test_write_batch_context_manager(db): key = b'batch-key' assert db.get(key) is None with db.write_batch() as wb: wb.put(key, b'') assert db.get(key) is not None # Data should also be written when an exception is raised key = b'batch-key-exception' assert db.get(key) is None with pytest.raises(ValueError): with db.write_batch() as wb: wb.put(key, b'') raise ValueError() assert db.get(key) is not None def test_write_batch_transaction(db): with pytest.raises(ValueError): with db.write_batch(transaction=True) as wb: wb.put(b'key', b'value') raise ValueError() assert list(db.iterator()) == [] def test_write_batch_bytes_like(db): with db.write_batch() as wb: wb.put(b'a', bytearray(b'foo')) wb.put(b'b', memoryview(b'bar')) assert db.get(b'a') == b'foo' assert db.get(b'b') == b'bar' def test_write_batch_append(db): wb = db.write_batch() wb.put(b'a', b'aa') other_wb = db.write_batch() other_wb.put(b'b', b'bb') wb.append(other_wb) wb.write() assert db.get(b'a') == b'aa' assert db.get(b'b') == b'bb' def test_write_batch_approximate_size(db): wb = db.write_batch() initial_size = wb.approximate_size() wb.put(b'a', b'aa') wb.put(b'b', b'bb') wb.delete(b'b') final_size = wb.approximate_size() assert initial_size < final_size def test_iteration(db): entries = [] for i in range(100): entry = ( ('%03d' % i).encode('ascii'), ('%03d' % i).encode('ascii')) entries.append(entry) for k, v in entries: db.put(k, v) for entry, expected in zip(entries, db): assert entry == expected def test_iterator_closing(db): db.put(b'k', b'v') it = db.iterator() next(it) it.close() pytest.raises(RuntimeError, next, it) pytest.raises(RuntimeError, it.seek_to_stop) with db.iterator() as it: next(it) pytest.raises(RuntimeError, next, it) def test_iterator_return(db): db.put(b'key', b'value') for key, value in db: assert b'key' == key assert b'value' == value for key, value in db.iterator(): assert b'key' == key assert b'value' == value for key in db.iterator(include_value=False): assert b'key' == key for value in db.iterator(include_key=False): assert b'value' == value for ret in db.iterator(include_key=False, include_value=False): assert ret is None def assert_iterator_behaviour(db, iter_kwargs, expected_values): first, second, third = expected_values is_forward = not iter_kwargs.get('reverse', False) # Simple iteration it = db.iterator(**iter_kwargs) assert next(it) == first assert next(it) == second assert next(it) == third with pytest.raises(StopIteration): next(it) with pytest.raises(StopIteration): # second time may not cause a segfault next(it) # Single steps, both next() and .prev() it = db.iterator(**iter_kwargs) assert next(it) == first assert it.prev() == first assert next(it) == first assert it.prev() == first with pytest.raises(StopIteration): it.prev() assert next(it) == first assert next(it) == second assert next(it) == third with pytest.raises(StopIteration): next(it) assert it.prev() == third assert it.prev() == second # End of iterator it = db.iterator(**iter_kwargs) if is_forward: it.seek_to_stop() else: it.seek_to_start() with pytest.raises(StopIteration): next(it) assert it.prev() == third # Begin of iterator it = db.iterator(**iter_kwargs) if is_forward: it.seek_to_start() else: it.seek_to_stop() with pytest.raises(StopIteration): it.prev() assert next(it) == first def test_forward_iteration(db): db.put(b'1', b'1') db.put(b'2', b'2') db.put(b'3', b'3') assert_iterator_behaviour( db, iter_kwargs=dict(include_key=False), expected_values=(b'1', b'2', b'3')) def test_reverse_iteration(db): db.put(b'1', b'1') db.put(b'2', b'2') db.put(b'3', b'3') assert_iterator_behaviour( db, iter_kwargs=dict(reverse=True, include_key=False), expected_values=(b'3', b'2', b'1')) def test_range_iteration(db): db.put(b'1', b'1') db.put(b'2', b'2') db.put(b'3', b'3') db.put(b'4', b'4') db.put(b'5', b'5') actual = list(db.iterator(start=b'2', include_value=False)) expected = [b'2', b'3', b'4', b'5'] assert actual == expected actual = list(db.iterator(stop=b'3', include_value=False)) expected = [b'1', b'2'] assert actual == expected actual = list(db.iterator(start=b'0', stop=b'3', include_value=False)) expected = [b'1', b'2'] assert actual == expected assert list(db.iterator(start=b'3', stop=b'0')) == [] # Only start (inclusive) assert_iterator_behaviour( db, iter_kwargs=dict(start=b'3', include_key=False), expected_values=(b'3', b'4', b'5')) # Only start (exclusive) assert_iterator_behaviour( db, iter_kwargs=dict(start=b'2', include_key=False, include_start=False), expected_values=(b'3', b'4', b'5')) # Only stop (exclusive) assert_iterator_behaviour( db, iter_kwargs=dict(stop=b'4', include_key=False), expected_values=(b'1', b'2', b'3')) # Only stop (inclusive) assert_iterator_behaviour( db, iter_kwargs=dict(stop=b'3', include_key=False, include_stop=True), expected_values=(b'1', b'2', b'3')) # Both start and stop assert_iterator_behaviour( db, iter_kwargs=dict(start=b'2', stop=b'5', include_key=False), expected_values=(b'2', b'3', b'4')) def test_reverse_range_iteration(db): db.put(b'1', b'1') db.put(b'2', b'2') db.put(b'3', b'3') db.put(b'4', b'4') db.put(b'5', b'5') assert list(db.iterator(start=b'3', stop=b'0', reverse=True)) == [] # Only start (inclusive) assert_iterator_behaviour( db, iter_kwargs=dict(start=b'3', reverse=True, include_value=False), expected_values=(b'5', b'4', b'3')) # Only start (exclusive) assert_iterator_behaviour( db, iter_kwargs=dict(start=b'2', reverse=True, include_value=False, include_start=False), expected_values=(b'5', b'4', b'3')) # Only stop (exclusive) assert_iterator_behaviour( db, iter_kwargs=dict(stop=b'4', reverse=True, include_value=False), expected_values=(b'3', b'2', b'1')) # Only stop (inclusive) assert_iterator_behaviour( db, iter_kwargs=dict(stop=b'3', reverse=True, include_value=False, include_stop=True), expected_values=(b'3', b'2', b'1')) # Both start and stop assert_iterator_behaviour( db, iter_kwargs=dict(start=b'1', stop=b'4', reverse=True, include_value=False), expected_values=(b'3', b'2', b'1')) def test_out_of_range_iterations(db): db.put(b'1', b'1') db.put(b'3', b'3') db.put(b'4', b'4') db.put(b'5', b'5') db.put(b'7', b'7') def t(expected, **kwargs): kwargs['include_value'] = False assert b''.join((db.iterator(**kwargs))) == expected # Out of range start key t(b'3457', start=b'2') t(b'3457', start=b'2', include_start=False) # Out of range stop key t(b'5431', stop=b'6', reverse=True) t(b'5431', stop=b'6', include_stop=True, reverse=True) # Out of range prefix t(b'', prefix=b'0', include_start=True) t(b'', prefix=b'0', include_start=False) t(b'', prefix=b'8', include_stop=True, reverse=True) t(b'', prefix=b'8', include_stop=False, reverse=True) def test_range_empty_database(db): it = db.iterator() it.seek_to_start() # no-op (don't crash) it.seek_to_stop() # no-op (don't crash) it = db.iterator() with pytest.raises(StopIteration): next(it) it = db.iterator() with pytest.raises(StopIteration): it.prev() with pytest.raises(StopIteration): next(it) def test_iterator_single_entry(db): key = b'key' value = b'value' db.put(key, value) it = db.iterator(include_value=False) assert next(it) == key assert it.prev() == key assert next(it) == key assert it.prev() == key with pytest.raises(StopIteration): it.prev() assert next(it) == key with pytest.raises(StopIteration): next(it) def test_iterator_seeking(db): db.put(b'1', b'1') db.put(b'2', b'2') db.put(b'3', b'3') db.put(b'4', b'4') db.put(b'5', b'5') it = db.iterator(include_value=False) it.seek_to_start() with pytest.raises(StopIteration): it.prev() assert next(it) == b'1' it.seek_to_start() assert next(it) == b'1' it.seek_to_stop() with pytest.raises(StopIteration): next(it) assert it.prev() == b'5' # Seek to a specific key it.seek(b'2') assert next(it) == b'2' assert next(it) == b'3' assert list(it) == [b'4', b'5'] it.seek(b'2') assert it.prev() == b'1' # Seek to keys that sort between/before/after existing keys it.seek(b'123') assert next(it) == b'2' it.seek(b'6') with pytest.raises(StopIteration): next(it) it.seek(b'0') with pytest.raises(StopIteration): it.prev() assert next(it) == b'1' it.seek(b'4') it.seek(b'3') assert next(it) == b'3' # Seek in a reverse iterator it = db.iterator(include_value=False, reverse=True) it.seek(b'6') assert next(it) == b'5' assert next(it) == b'4' it.seek(b'1') with pytest.raises(StopIteration): next(it) assert it.prev() == b'1' # Seek in iterator with start key it = db.iterator(start=b'2', include_value=False) assert next(it) == b'2' it.seek(b'2') assert next(it) == b'2' it.seek(b'0') assert next(it) == b'2' it.seek_to_start() assert next(it) == b'2' # Seek in iterator with stop key it = db.iterator(stop=b'3', include_value=False) assert next(it) == b'1' it.seek(b'2') assert next(it) == b'2' it.seek(b'5') with pytest.raises(StopIteration): next(it) it.seek(b'5') assert it.prev() == b'2' it.seek_to_stop() with pytest.raises(StopIteration): next(it) it.seek_to_stop() assert it.prev() == b'2' # Seek in iterator with both start and stop keys it = db.iterator(start=b'2', stop=b'5', include_value=False) it.seek(b'0') assert next(it) == b'2' it.seek(b'5') with pytest.raises(StopIteration): next(it) it.seek(b'5') assert it.prev() == b'4' # Seek in reverse iterator with start and stop key it = db.iterator( reverse=True, start=b'2', stop=b'4', include_value=False) it.seek(b'5') assert next(it) == b'3' it.seek(b'1') assert it.prev() == b'2' it.seek_to_start() with pytest.raises(StopIteration): next(it) it.seek_to_stop() assert next(it) == b'3' def test_iterator_boundaries(db): db.put(b'1', b'1') db.put(b'2', b'2') db.put(b'3', b'3') db.put(b'4', b'4') db.put(b'5', b'5') def t(expected, **kwargs): kwargs.update(include_value=False) actual = b''.join(db.iterator(**kwargs)) assert actual == expected t(b'12345') t(b'345', start=b'2', include_start=False) t(b'1234', stop=b'5', include_start=False) t(b'12345', stop=b'5', include_stop=True) t(b'2345', start=b'2', stop=b'5', include_stop=True) t(b'2345', start=b'2', stop=b'5', include_stop=True) t(b'345', start=b'3', include_stop=False) t(b'45', start=b'3', include_start=False, include_stop=False) def test_iterator_prefix(db): keys = [ b'a1', b'a2', b'a3', b'aa4', b'aa5', b'b1', b'b2', b'b3', b'b4', b'b5', b'c1', b'c\xff', b'c\x00', b'\xff\xff', b'\xff\xffa', b'\xff\xff\xff', ] for key in keys: db.put(key, b'') pytest.raises(TypeError, db.iterator, prefix=b'abc', start=b'a') pytest.raises(TypeError, db.iterator, prefix=b'abc', stop=b'a') def t(*args, **kwargs): # Positional arguments are the expected ones, keyword # arguments are passed to db.iterator() kwargs.update(include_value=False) it = db.iterator(**kwargs) assert list(it) == list(args) t(*sorted(keys), prefix=b'') t(prefix=b'd') t(b'b1', prefix=b'b1') t(b'a1', b'a2', b'a3', b'aa4', b'aa5', prefix=b'a') t(b'aa4', b'aa5', prefix=b'aa') t(b'\xff\xff', b'\xff\xffa', b'\xff\xff\xff', prefix=b'\xff\xff') # The include_start and include_stop make no sense, so should # not affect the behaviour t(b'a1', b'a2', b'a3', b'aa4', b'aa5', prefix=b'a', include_start=False, include_stop=True) def test_snapshot(db): db.put(b'a', b'a') db.put(b'b', b'b') # Snapshot should have existing values, but not changed values snapshot = db.snapshot() assert snapshot.get(b'a') == b'a' assert list(snapshot.iterator(include_value=False)) == [b'a', b'b'] assert snapshot.get(b'c') is None db.delete(b'a') db.put(b'c', b'c') assert snapshot.get(b'c') is None assert snapshot.get(b'c', b'd') == b'd' assert snapshot.get(b'c', default=b'd') == b'd' assert list(snapshot.iterator(include_value=False)) == [b'a', b'b'] # New snapshot should reflect latest state snapshot = db.snapshot() assert snapshot.get(b'c') == b'c' assert list(snapshot.iterator(include_value=False)) == [b'b', b'c'] # Snapshots are directly iterable, just like DB assert list(k for k, v in snapshot) == [b'b', b'c'] def test_snapshot_closing(db): # Snapshots can be closed explicitly snapshot = db.snapshot() snapshot.close() with pytest.raises(RuntimeError): snapshot.get(b'a') snapshot.close() # no-op def test_snapshot_closing_database(db): # Closing the db should also render the snapshot unusable snapshot = db.snapshot() db.close() with pytest.raises(RuntimeError): snapshot.get(b'a') def test_snapshot_closing_context_manager(db): # Context manager db.put(b'a', b'a') snapshot = db.snapshot() with snapshot: assert snapshot.get(b'a') == b'a' with pytest.raises(RuntimeError): snapshot.get(b'a') def test_property(db): with pytest.raises(TypeError): db.get_property() with pytest.raises(TypeError): db.get_property(42) assert db.get_property(b'does-not-exist') is None properties = [ b'leveldb.stats', b'leveldb.sstables', b'leveldb.num-files-at-level0', ] for prop in properties: assert isinstance(db.get_property(prop), bytes) def test_compaction(db): db.compact_range() db.compact_range(start=b'a', stop=b'b') db.compact_range(start=b'a') db.compact_range(stop=b'b') def test_approximate_sizes(db_dir): # Write some data to a fresh database db = plyvel.DB(db_dir, create_if_missing=True, error_if_exists=True) value = b'a' * 100 with db.write_batch() as wb: for i in range(1000): key = bytes(i) * 100 wb.put(key, value) # Close and reopen the database db.close() del wb, db db = plyvel.DB(db_dir, create_if_missing=False) with pytest.raises(TypeError): db.approximate_size(1, 2) with pytest.raises(TypeError): db.approximate_sizes(None) with pytest.raises(TypeError): db.approximate_sizes((1, 2)) # Test single range assert db.approximate_size(b'1', b'2') >= 0 # Test multiple ranges assert db.approximate_sizes() == [] assert db.approximate_sizes((b'1', b'2'))[0] >= 0 ranges = [ (b'1', b'3'), (b'', b'\xff'), ] assert len(db.approximate_sizes(*ranges)) == len(ranges) def test_repair_db(db_dir): db = plyvel.DB(db_dir, create_if_missing=True) db.put(b'foo', b'bar') db.close() del db plyvel.repair_db(db_dir) db = plyvel.DB(db_dir) assert db.get(b'foo') == b'bar' def test_destroy_db(db_dir): db_dir = os.path.join(db_dir, 'subdir') db = plyvel.DB(db_dir, create_if_missing=True) db.put(b'foo', b'bar') db.close() del db plyvel.destroy_db(db_dir) assert not os.path.lexists(db_dir) def test_threading(db): randint = random.randint N_THREADS_PER_FUNC = 5 def bulk_insert(db): name = threading.current_thread().name v = name.encode('ascii') * randint(300, 700) for n in range(randint(1000, 8000)): rev = '{0:x}'.format(n)[::-1] k = '{0}: {1}'.format(rev, name).encode('ascii') db.put(k, v) def iterate_full(db): for i in range(randint(4, 7)): for key, value in db.iterator(reverse=True): pass def iterate_short(db): for i in range(randint(200, 700)): it = db.iterator() list(itertools.islice(it, randint(50, 100))) def close_db(db): time.sleep(1) db.close() funcs = [ bulk_insert, iterate_full, iterate_short, # XXX: This this will usually cause a segfault since # unexpectedly closing a database may crash threads using # iterators: # close_db, ] threads = [] for func in funcs: for n in range(N_THREADS_PER_FUNC): t = threading.Thread(target=func, args=(db,)) t.start() threads.append(t) for t in threads: t.join() def test_invalid_comparator(db_dir): with pytest.raises(ValueError): plyvel.DB(db_dir, comparator=None, comparator_name=b'invalid') with pytest.raises(TypeError): plyvel.DB(db_dir, comparator=lambda x, y: 1, comparator_name=12) with pytest.raises(TypeError): plyvel.DB(db_dir, comparator=b'not-a-callable', comparator_name=b'invalid') def test_comparator(db_dir): def comparator(a, b): a = a.lower() b = b.lower() if a < b: return -1 if a > b: return 1 else: return 0 comparator_name = b"CaseInsensitiveComparator" db = plyvel.DB( db_dir, create_if_missing=True, comparator=comparator, comparator_name=comparator_name) keys = [ b'aaa', b'BBB', b'ccc', ] with db.write_batch() as wb: for key in keys: wb.put(key, b'') expected = sorted(keys, key=lambda s: s.lower()) actual = list(db.iterator(include_value=False)) assert actual == expected def test_prefixed_db(db): for prefix in (b'a', b'b'): for i in range(1000): key = prefix + '{0:03d}'.format(i).encode('ascii') db.put(key, b'') db_a = db.prefixed_db(b'a') db_b = db.prefixed_db(b'b') # Access original db assert db_a.db is db # Basic operations key = b'123' assert db_a.get(key) is not None db_a.put(key, b'foo') assert db_a.get(key) == b'foo' db_a.delete(key) assert db_a.get(key) is None assert db_a.get(key, b'v') == b'v' assert db_a.get(key, default=b'v') == b'v' db_a.put(key, b'foo') assert db.get(b'a123') == b'foo' # Iterators assert len(list(db_a)) == 1000 it = db_a.iterator(include_value=False) assert next(it) == b'000' assert next(it) == b'001' assert len(list(it)) == 998 it = db_a.iterator(start=b'900') assert len(list(it)) == 100 it = db_a.iterator(stop=b'012', include_stop=False) assert len(list(it)) == 12 it = db_a.iterator(stop=b'012', include_stop=True) assert len(list(it)) == 13 it = db_a.iterator(include_stop=True) assert len(list(it)) == 1000 it = db_a.iterator(prefix=b'10') assert len(list(it)) == 10 it = db_a.iterator(include_value=False) it.seek(b'500') assert len(list(it)) == 500 it.seek_to_start() assert len(list(it)) == 1000 it.seek_to_stop() assert it.prev() == b'999' it = db_b.iterator() it.seek_to_start() with pytest.raises(StopIteration): it.prev() it.seek_to_stop() with pytest.raises(StopIteration): next(it) # Snapshots sn_a = db_a.snapshot() assert sn_a.get(b'042') == b'' db_a.put(b'042', b'new') assert sn_a.get(b'042') == b'' assert db_a.get(b'042') == b'new' assert db_a.get(b'foo', b'x') == b'x' assert db_a.get(b'foo', default=b'x') == b'x' # Snapshot iterators sn_a.iterator() it = sn_a.iterator( start=b'900', include_start=False, include_value=False) assert next(it) == b'901' assert len(list(it)) == 98 # Write batches wb = db_a.write_batch() wb.put(b'0002', b'foo') wb.delete(b'0003') wb.write() assert db_a.get(b'0002') == b'foo' assert db_a.get(b'0003') is None # Delete all data in db_a for key in db_a.iterator(include_value=False): db_a.delete(key) assert len(list(db_a)) == 0 # The complete db and the 'b' prefix should remain untouched assert len(list(db)) == 1000 assert len(list(db_b)) == 1000 # Prefixed prefixed databases (recursive) db_b12 = db_b.prefixed_db(b'12') it = db_b12.iterator(include_value=False) assert next(it) == b'0' assert len(list(it)) == 9 def test_raw_iterator(db): for i in range(1000): key = value = '{0:03d}'.format(i).encode('ascii') db.put(key, value) it = db.raw_iterator() it.seek_to_first() assert it.key() == b'000' assert it.value() == b'000' it.next() assert it.key() == b'001' assert it.value() == b'001' it.next() assert it.key() == b'002' it.next() it.next() assert it.key() == b'004' it.prev() assert it.key() == b'003' it.seek_to_first() it.prev() with pytest.raises(plyvel.IteratorInvalidError): it.value() assert not it.valid() it.seek(b'005') assert it.key() == b'005' it.seek(b'006abc') assert it.key() == b'007' it.seek_to_last() assert it.key() == b'999' it.next() with pytest.raises(plyvel.IteratorInvalidError): it.key() def test_raw_iterator_empty_db(db): it = db.raw_iterator() assert not it.valid() it.seek_to_first() assert not it.valid() it.seek_to_last() assert not it.valid() with pytest.raises(plyvel.IteratorInvalidError): it.key() def test_raw_iterator_snapshot(db): sn = db.snapshot() db.put(b'001', b'') it = sn.raw_iterator() it.seek_to_first() assert not it.valid() with pytest.raises(plyvel.IteratorInvalidError): it.key() def test_raw_iterator_closing(db): it = db.raw_iterator() it.close() with pytest.raises(RuntimeError): it.next() with pytest.raises(RuntimeError): with db.raw_iterator() as it: pass it.valid() def test_access_DB_name_attr(db_dir): db = plyvel.DB(db_dir, create_if_missing=True) assert db.name == db_dir def test_try_changing_DB_name_attr(db_dir): db = plyvel.DB(db_dir, create_if_missing=True) with pytest.raises(AttributeError): db.name = 'a' def test_context_manager(db_dir): key = b'the-key' value = b'the-value' with plyvel.DB(db_dir, create_if_missing=True) as db: db.put(key, value) assert db.get(key) == value assert db.closed plyvel-1.5.1/tox.ini000066400000000000000000000002201455126310100143340ustar00rootroot00000000000000[tox] envlist = py312, py311, py310, py39, py38, py37 [testenv] deps = -rrequirements-test.txt commands = make test allowlist_externals = make