tooz-2.0.0/0000775000175000017500000000000013616633265012541 5ustar zuulzuul00000000000000tooz-2.0.0/run-examples.sh0000775000175000017500000000030213616633007015505 0ustar zuulzuul00000000000000#!/bin/bash set -e python_version=$(python --version 2>&1) echo "Running using '$python_version'" for filename in examples/*.py; do echo "Activating '$filename'" python $filename done tooz-2.0.0/.zuul.yaml0000664000175000017500000001246413616633007014503 0ustar zuulzuul00000000000000- project: templates: - check-requirements - lib-forward-testing-python3 - openstack-cover-jobs - openstack-python3-train-jobs - periodic-stable-jobs - publish-openstack-docs-pti - release-notes-jobs-python3 check: jobs: - tooz-tox-py37-etcd - tooz-tox-py36-etcd - tooz-tox-py37-etcd3 - tooz-tox-py36-etcd3 - tooz-tox-py37-etcd3gw - tooz-tox-py36-etcd3gw - tooz-tox-py37-zookeeper - tooz-tox-py36-zookeeper - tooz-tox-py37-redis - tooz-tox-py36-redis - tooz-tox-py37-sentinel - tooz-tox-py36-sentinel - tooz-tox-py37-memcached - tooz-tox-py36-memcached - tooz-tox-py37-postgresql - tooz-tox-py36-postgresql - tooz-tox-py37-mysql - tooz-tox-py36-mysql - tooz-tox-py37-consul - tooz-tox-py36-consul gate: jobs: - tooz-tox-py37-etcd - tooz-tox-py36-etcd - tooz-tox-py37-etcd3 - tooz-tox-py36-etcd3 - tooz-tox-py37-etcd3gw - tooz-tox-py36-etcd3gw - tooz-tox-py37-zookeeper - tooz-tox-py36-zookeeper - tooz-tox-py37-redis - tooz-tox-py36-redis - tooz-tox-py37-sentinel - tooz-tox-py36-sentinel - tooz-tox-py37-memcached - tooz-tox-py36-memcached - tooz-tox-py37-postgresql - tooz-tox-py36-postgresql - tooz-tox-py37-mysql - tooz-tox-py36-mysql - tooz-tox-py37-consul - tooz-tox-py36-consul - job: name: tooz-tox-py37-consul parent: openstack-tox-py37 description: | Run tests using ``py37-consul`` environment. vars: tox_envlist: mysql-python - job: name: tooz-tox-py37-etcd parent: openstack-tox-py37 description: | Run tests using ``py37-etcd`` environment. vars: tox_envlist: py37-etcd - job: name: tooz-tox-py37-etcd3 parent: openstack-tox-py37 description: | Run tests using ``py37-etcd3`` environment. vars: tox_envlist: py37-etcd3 - job: name: tooz-tox-py37-etcd3gw parent: openstack-tox-py37 description: | Run tests using ``py37-etcd3gw`` environment. vars: tox_envlist: py37-etcd3gw - job: name: tooz-tox-py37-memcached parent: openstack-tox-py37 description: | Run tests using ``py37-memcached`` environment. vars: tox_envlist: py37-memcached - job: name: tooz-tox-py37-mysql parent: openstack-tox-py37 description: | Run tests using ``py37-mysql`` environment. vars: tox_envlist: py37-mysql - job: name: tooz-tox-py37-postgresql parent: openstack-tox-py37 description: | Run tests using ``py37-postgresql`` environment. vars: tox_envlist: py37-postgresql - job: name: tooz-tox-py37-redis parent: openstack-tox-py37 description: | Run tests using ``py37-redis`` environment. vars: tox_envlist: py37-redis pre-run: - playbooks/stop-redis.yaml - job: name: tooz-tox-py37-sentinel parent: openstack-tox-py37 description: | Run tests using ``py37-sentinel`` environment. vars: tox_envlist: py37-sentinel pre-run: - playbooks/stop-redis.yaml - job: name: tooz-tox-py37-zookeeper parent: openstack-tox-py37 description: | Run tests using ``py37-zookeeper`` environment. vars: tox_envlist: py37-zookeeper - job: name: tooz-tox-py36-consul parent: openstack-tox-py36 description: | Run tests using ``py36-consul`` environment. vars: tox_envlist: py36-consul - job: name: tooz-tox-py36-etcd parent: openstack-tox-py36 description: | Run tests using ``py36-etcd`` environment. vars: tox_envlist: py36-etcd - job: name: tooz-tox-py36-etcd3 parent: openstack-tox-py36 description: | Run tests using ``py36-etcd3`` environment. vars: tox_envlist: py36-etcd3 - job: name: tooz-tox-py36-etcd3gw parent: openstack-tox-py36 description: | Run tests using ``py36-etcd3gw`` environment. vars: tox_envlist: py36-etcd3gw - job: name: tooz-tox-py36-memcached parent: openstack-tox-py36 description: | Run tests using ``py36-memcached`` environment. vars: tox_envlist: py36-memcached - job: name: tooz-tox-py36-mysql parent: openstack-tox-py36 description: | Run tests using ``py36-mysql`` environment. vars: tox_envlist: py36-mysql - job: name: tooz-tox-py36-postgresql parent: openstack-tox-py36 description: | Run tests using ``py36-postgresql`` environment. vars: tox_envlist: py36-postgresql - job: name: tooz-tox-py36-redis parent: openstack-tox-py36 description: | Run tests using ``py36-redis`` environment. vars: tox_envlist: py36-redis pre-run: - playbooks/stop-redis.yaml - job: name: tooz-tox-py36-sentinel parent: openstack-tox-py36 description: | Run tests using ``py36-sentinel`` environment. vars: tox_envlist: py36-sentinel pre-run: - playbooks/stop-redis.yaml - job: name: tooz-tox-py36-zookeeper parent: openstack-tox-py36 description: | Run tests using ``py36-zookeeper`` environment. vars: tox_envlist: py36-zookeeper tooz-2.0.0/doc/0000775000175000017500000000000013616633265013306 5ustar zuulzuul00000000000000tooz-2.0.0/doc/requirements.txt0000664000175000017500000000065613616633007016573 0ustar zuulzuul00000000000000sphinx!=1.6.6,!=1.6.7,>=1.6.2,!=2.1.0 # BSD openstackdocstheme>=1.11.0 # Apache-2.0 reno>=1.8.0 # Apache-2.0 # Install dependencies for tooz so that autodoc works. python-consul>=0.4.7 # MIT License sysv-ipc>=0.6.8 # BSD License zake>=0.1.6 # Apache-2.0 redis>=2.10.0 # MIT psycopg2>=2.5 # LGPL/ZPL PyMySQL>=0.6.2 # MIT License pymemcache!=1.3.0,>=1.2.9 # Apache 2.0 License etcd3>=0.6.2 # Apache-2.0 etcd3gw>=0.1.0 # Apache-2.0 tooz-2.0.0/doc/source/0000775000175000017500000000000013616633265014606 5ustar zuulzuul00000000000000tooz-2.0.0/doc/source/reference/0000775000175000017500000000000013616633265016544 5ustar zuulzuul00000000000000tooz-2.0.0/doc/source/reference/index.rst0000664000175000017500000000273313616633007020404 0ustar zuulzuul00000000000000================ Module Reference ================ Interfaces ---------- .. autoclass:: tooz.coordination.CoordinationDriver :members: Consul ~~~~~~ .. autoclass:: tooz.drivers.consul.ConsulDriver :members: Etcd ~~~~ .. autoclass:: tooz.drivers.etcd.EtcdDriver :members: Etcd3 ~~~~~ .. autoclass:: tooz.drivers.etcd3.Etcd3Driver :members: Etcd3gw ~~~~~~~ .. autoclass:: tooz.drivers.etcd3gw.Etcd3Driver :members: File ~~~~ .. autoclass:: tooz.drivers.file.FileDriver :members: IPC ~~~ .. autoclass:: tooz.drivers.ipc.IPCDriver :members: Memcached ~~~~~~~~~ .. autoclass:: tooz.drivers.memcached.MemcachedDriver :members: Mysql ~~~~~ .. autoclass:: tooz.drivers.mysql.MySQLDriver :members: PostgreSQL ~~~~~~~~~~ .. autoclass:: tooz.drivers.pgsql.PostgresDriver :members: Redis ~~~~~ .. autoclass:: tooz.drivers.redis.RedisDriver :members: Zake ~~~~ .. autoclass:: tooz.drivers.zake.ZakeDriver :members: Zookeeper ~~~~~~~~~ .. autoclass:: tooz.drivers.zookeeper.KazooDriver :members: Exceptions ---------- .. autoclass:: tooz.ToozError .. autoclass:: tooz.coordination.ToozConnectionError .. autoclass:: tooz.coordination.OperationTimedOut .. autoclass:: tooz.coordination.GroupNotCreated .. autoclass:: tooz.coordination.GroupAlreadyExist .. autoclass:: tooz.coordination.MemberAlreadyExist .. autoclass:: tooz.coordination.MemberNotJoined .. autoclass:: tooz.coordination.GroupNotEmpty .. autofunction:: tooz.utils.raise_with_cause tooz-2.0.0/doc/source/install/0000775000175000017500000000000013616633265016254 5ustar zuulzuul00000000000000tooz-2.0.0/doc/source/install/index.rst0000664000175000017500000000155313616633007020113 0ustar zuulzuul00000000000000============ Installation ============ Python Versions =============== Tooz is tested under Python 2.7 and 3.4. .. _install-basic: Basic Installation ================== Tooz should be installed into the same site-packages area where the application and extensions are installed (either a virtualenv or the global site-packages). You may need administrative privileges to do that. The easiest way to install it is using pip_:: $ pip install tooz or:: $ sudo pip install tooz .. _pip: http://pypi.python.org/pypi/pip Download ======== Tooz releases are hosted on PyPI and can be downloaded from: http://pypi.python.org/pypi/tooz Source Code =========== The source is hosted on the OpenStack infrastructure: https://opendev.org/openstack/tooz/ Reporting Bugs ============== Please report bugs through the launchpad project: https://launchpad.net/python-tooz tooz-2.0.0/doc/source/index.rst0000664000175000017500000000137513616633007016447 0ustar zuulzuul00000000000000========================================= Tooz -- Distributed System Helper Library ========================================= The Tooz project aims at centralizing the most common distributed primitives like group membership protocol, lock service and leader election by providing a coordination API helping developers to build distributed applications. [#f1]_ .. toctree:: :maxdepth: 2 install/index user/index reference/index .. rubric:: Indices and tables * :ref:`genindex` * :ref:`modindex` * :ref:`search` .. [#f1] It should be noted that even though it is designed with OpenStack integration in mind, and that is where most of its *current* integration is it aims to be generally usable and useful in any project. tooz-2.0.0/doc/source/conf.py0000664000175000017500000002020613616633007016077 0ustar zuulzuul00000000000000# -*- coding: utf-8 -*- # # tooz documentation build configuration file # # This file is execfile()d with the current directory set to its containing dir. # # Note that not all possible configuration values are present in this # autogenerated file. # # All configuration values have a default; values that are commented out # serve to show the default. import datetime import subprocess # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. #sys.path.insert(0, os.path.abspath('.')) # -- General configuration ----------------------------------------------------- # If your documentation needs a minimal Sphinx version, state it here. #needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = [ 'sphinx.ext.autodoc', 'sphinx.ext.todo', 'sphinx.ext.graphviz', 'sphinx.ext.extlinks', 'openstackdocstheme', 'sphinx.ext.inheritance_diagram', 'sphinx.ext.viewcode', 'stevedore.sphinxext', ] # openstackdocstheme options repository_name = 'openstack/tooz' bug_project = 'tooz' bug_tag = '' # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] # The suffix of source filenames. source_suffix = '.rst' # The encoding of source files. #source_encoding = 'utf-8-sig' # The master toctree document. master_doc = 'index' # General information about the project. project = u'tooz' copyright = u'%s, OpenStack Foundation' % datetime.date.today().year # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. version = subprocess.Popen(['sh', '-c', 'cd ../..; python setup.py --version'], stdout=subprocess.PIPE).stdout.read().decode('utf-8') version = version.strip() # The full version, including alpha/beta/rc tags. release = version # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. #language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: #today = '' # Else, today_fmt is used as the format for a strftime call. #today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. exclude_patterns = [] # The reST default role (used for this markup: `text`) to use for all documents. #default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. #add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). #add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. #show_authors = False # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' # A list of ignored prefixes for module index sorting. #modindex_common_prefix = [] # -- Options for HTML output --------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. html_theme = 'openstackdocs' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. #html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. #html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". #html_title = None # A shorter title for the navigation bar. Default is the same as html_title. #html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. #html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. #html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". # html_static_path = ['_static'] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. html_last_updated_fmt = '%Y-%m-%d %H:%M' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. #html_use_smartypants = True # Custom sidebar templates, maps document names to template names. #html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. #html_additional_pages = {} # If false, no module index is generated. #html_domain_indices = True # If false, no index is generated. #html_use_index = True # If true, the index is split into individual pages for each letter. #html_split_index = False # If true, links to the reST sources are added to the pages. #html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. #html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. #html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. #html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). #html_file_suffix = None # Output file base name for HTML help builder. htmlhelp_basename = 'toozdoc' # -- Options for LaTeX output -------------------------------------------------- latex_elements = { # The paper size ('letterpaper' or 'a4paper'). #'papersize': 'letterpaper', # The font size ('10pt', '11pt' or '12pt'). #'pointsize': '10pt', # Additional stuff for the LaTeX preamble. #'preamble': '', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ ('index', 'tooz.tex', u'tooz Documentation', u'eNovance', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of # the title page. #latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. #latex_use_parts = False # If true, show page references after internal links. #latex_show_pagerefs = False # If true, show URL addresses after external links. #latex_show_urls = False # Documents to append as an appendix to all manuals. #latex_appendices = [] # If false, no module index is generated. #latex_domain_indices = True # -- Options for manual page output -------------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). # man_pages = [ # ('index', 'tooz', u'tooz Documentation', # [u'eNovance'], 1) # ] # If true, show URL addresses after external links. #man_show_urls = False # -- Options for Texinfo output ------------------------------------------------ # Grouping the document tree into Texinfo files. List of tuples # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ ('index', 'tooz', u'tooz Documentation', u'OpenStack Foundation', 'tooz', 'One line description of project.', 'Miscellaneous'), ] # Documents to append as an appendix to all manuals. #texinfo_appendices = [] # If false, no module index is generated. #texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. #texinfo_show_urls = 'footnote' # extlinks = { # } autodoc_default_options = { 'members': None, 'special-members': None, 'show_inheritance': None, } tooz-2.0.0/doc/source/user/0000775000175000017500000000000013616633265015564 5ustar zuulzuul00000000000000tooz-2.0.0/doc/source/user/drivers.rst0000664000175000017500000001611113616633007017766 0ustar zuulzuul00000000000000======= Drivers ======= Tooz is provided with several drivers implementing the provided coordination API. While all drivers provides the same set of features with respect to the API, some of them have different characteristics: Zookeeper --------- **Driver:** :py:class:`tooz.drivers.zookeeper.KazooDriver` **Characteristics:** :py:attr:`tooz.drivers.zookeeper.KazooDriver.CHARACTERISTICS` **Entrypoint name:** ``zookeeper`` or ``kazoo`` **Summary:** The zookeeper is the reference implementation and provides the most solid features as it's possible to build a cluster of zookeeper servers that is resilient towards network partitions for example. **Test driver:** :py:class:`tooz.drivers.zake.ZakeDriver` **Characteristics:** :py:attr:`tooz.drivers.zake.ZakeDriver.CHARACTERISTICS` **Test driver entrypoint name:** ``zake`` Considerations ~~~~~~~~~~~~~~ - Primitives are based on sessions (and typically require careful selection of session heartbeat periodicity and server side configuration of session expiry). Memcached --------- **Driver:** :py:class:`tooz.drivers.memcached.MemcachedDriver` **Characteristics:** :py:attr:`tooz.drivers.memcached.MemcachedDriver.CHARACTERISTICS` **Entrypoint name:** ``memcached`` **Summary:** The memcached driver is a basic implementation and provides little resiliency, though it's much simpler to setup. A lot of the features provided in tooz are based on timeout (heartbeats, locks, etc) so are less resilient than other backends. Considerations ~~~~~~~~~~~~~~ - Less resilient than other backends such as zookeeper and redis. - Primitives are often based on TTL(s) that may expire before being renewed. - Lacks certain primitives (compare and delete) so certain functionality is fragile and/or broken due to this. Redis ----- **Driver:** :py:class:`tooz.drivers.redis.RedisDriver` **Characteristics:** :py:attr:`tooz.drivers.redis.RedisDriver.CHARACTERISTICS` **Entrypoint name:** ``redis`` **Summary:** The redis driver is a basic implementation and provides reasonable resiliency when used with `redis-sentinel`_. A lot of the features provided in tooz are based on timeout (heartbeats, locks, etc) so are less resilient than other backends. Considerations ~~~~~~~~~~~~~~ - Less resilient than other backends such as zookeeper. - Primitives are often based on TTL(s) that may expire before being renewed. IPC --- **Driver:** :py:class:`tooz.drivers.ipc.IPCDriver` **Characteristics:** :py:attr:`tooz.drivers.ipc.IPCDriver.CHARACTERISTICS` **Entrypoint name:** ``ipc`` **Summary:** The IPC driver is based on Posix IPC API and implements a lock mechanism and some basic group primitives (with **huge** limitations). Considerations ~~~~~~~~~~~~~~ - The lock can **only** be distributed locally to a computer processes. File ---- **Driver:** :py:class:`tooz.drivers.file.FileDriver` **Characteristics:** :py:attr:`tooz.drivers.file.FileDriver.CHARACTERISTICS` **Entrypoint name:** ``file`` **Summary:** The file driver is a **simple** driver based on files and directories. It implements a lock based on POSIX or Window file level locking mechanism and some basic group primitives (with **huge** limitations). Considerations ~~~~~~~~~~~~~~ - The lock can **only** be distributed locally to a computer processes. - Certain concepts provided by it are **not** crash tolerant. PostgreSQL ---------- **Driver:** :py:class:`tooz.drivers.pgsql.PostgresDriver` **Characteristics:** :py:attr:`tooz.drivers.pgsql.PostgresDriver.CHARACTERISTICS` **Entrypoint name:** ``postgresql`` **Summary:** The postgresql driver is a driver providing only a distributed lock (for now) and is based on the `PostgreSQL database server`_ and its API(s) that provide for `advisory locks`_ to be created and used by applications. When a lock is acquired it will release either when explicitly released or automatically when the database session ends (for example if the program using the lock crashes). Considerations ~~~~~~~~~~~~~~ - Lock that may be acquired restricted by ``max_locks_per_transaction * (max_connections + max_prepared_transactions)`` upper bound (PostgreSQL server configuration settings). MySQL ----- **Driver:** :py:class:`tooz.drivers.mysql.MySQLDriver` **Characteristics:** :py:attr:`tooz.drivers.mysql.MySQLDriver.CHARACTERISTICS` **Entrypoint name:** ``mysql`` **Summary:** The MySQL driver is a driver providing only distributed locks (for now) and is based on the `MySQL database server`_ supported `get_lock`_ primitives. When a lock is acquired it will release either when explicitly released or automatically when the database session ends (for example if the program using the lock crashes). Considerations ~~~~~~~~~~~~~~ - Does **not** work correctly on some MySQL versions. - Does **not** work when MySQL replicates from one server to another (locks are local to the server that they were created from). Etcd ---- **Driver:** :py:class:`tooz.drivers.etcd.EtcdDriver` **Characteristics:** :py:attr:`tooz.drivers.etcd.EtcdDriver.CHARACTERISTICS` **Entrypoint name:** ``etcd`` **Summary:** The etcd driver is a driver providing only distributed locks (for now) and is based on the `etcd server`_ supported key/value storage and associated primitives. Etcd3 ----- **Driver:** :py:class:`tooz.drivers.etcd3.Etcd3Driver` **Characteristics:** :py:attr:`tooz.drivers.etcd3.Etcd3Driver.CHARACTERISTICS` **Entrypoint name:** ``etcd3`` **Summary:** The etcd3 driver is a driver providing only distributed locks (for now) and is based on the `etcd server`_ supported key/value storage and associated primitives. Etcd3 Gateway ------------- **Driver:** :py:class:`tooz.drivers.etcd3gw.Etcd3Driver` **Characteristics:** :py:attr:`tooz.drivers.etcd3gw.Etcd3Driver.CHARACTERISTICS` **Entrypoint name:** ``etcd3+http`` **Summary:** The etcd3gw driver is a driver providing only distributed locks (for now) and is based on the `etcd server`_ supported key/value storage and associated primitives. It relies on the `GRPC Gateway`_ to provide HTTP access to etcd3. Consul ------ **Driver:** :py:class:`tooz.drivers.consul.ConsulDriver` **Characteristics:** :py:attr:`tooz.drivers.consul.ConsulDriver.CHARACTERISTICS` **Entrypoint name:** ``consul`` **Summary:** The `consul`_ driver is a driver providing only distributed locks (for now) and is based on the consul server key/value storage and/or primitives. When a lock is acquired it will release either when explicitly released or automatically when the consul session ends (for example if the program using the lock crashes). Characteristics --------------- .. autoclass:: tooz.coordination.Characteristics .. _etcd server: https://coreos.com/etcd/ .. _consul: https://www.consul.io/ .. _advisory locks: http://www.postgresql.org/docs/8.2/interactive/\ explicit-locking.html#ADVISORY-LOCKS .. _get_lock: http://dev.mysql.com/doc/refman/5.5/en/\ miscellaneous-functions.html#function_get-lock .. _PostgreSQL database server: http://postgresql.org .. _MySQL database server: http://mysql.org .. _redis-sentinel: http://redis.io/topics/sentinel .. _GRPC Gateway: https://github.com/grpc-ecosystem/grpc-gateway tooz-2.0.0/doc/source/user/history.rst0000664000175000017500000000004013616633007020003 0ustar zuulzuul00000000000000.. include:: ../../../ChangeLog tooz-2.0.0/doc/source/user/index.rst0000664000175000017500000000026013616633007017415 0ustar zuulzuul00000000000000================== User Documentation ================== .. toctree:: :maxdepth: 2 tutorial/index drivers compatibility .. toctree:: :maxdepth: 1 history tooz-2.0.0/doc/source/user/tutorial/0000775000175000017500000000000013616633265017427 5ustar zuulzuul00000000000000tooz-2.0.0/doc/source/user/tutorial/leader_election.rst0000664000175000017500000000217513616633007023276 0ustar zuulzuul00000000000000================= Leader Election ================= Each group can elect its own leader. There can be only one leader at a time in a group. Only members that are running for the election can be elected. As soon as one of leader steps down or dies, a new member that was running for the election will be elected. .. literalinclude:: ../../../../examples/leader_election.py :language: python The method :meth:`tooz.coordination.CoordinationDriver.watch_elected_as_leader` allows to register for a function to be called back when the member is elected as a leader. Using this function indicates that the run is therefore running for the election. The member can stop running by unregistering all its callbacks with :meth:`tooz.coordination.CoordinationDriver.unwatch_elected_as_leader`. It can also temporarily try to step down as a leader with :meth:`tooz.coordination.CoordinationDriver.stand_down_group_leader`. If another member is in the run for election, it may be elected instead. To retrieve the leader of a group, even when not being part of the group, the method :meth:`tooz.coordination.CoordinationDriver.get_leader()` can be used. tooz-2.0.0/doc/source/user/tutorial/lock.rst0000664000175000017500000000101013616633007021073 0ustar zuulzuul00000000000000====== Lock ====== Tooz provides distributed locks. A lock is identified by a name, and a lock can only be acquired by one coordinator at a time. .. literalinclude:: ../../../../examples/lock.py :language: python The method :meth:`tooz.coordination.CoordinationDriver.get_lock` allows to create a lock identified by a name. Once you retrieve this lock, you can use it as a context manager or use the :meth:`tooz.locking.Lock.acquire` and :meth:`tooz.locking.Lock.release` methods to acquire and release the lock. tooz-2.0.0/doc/source/user/tutorial/index.rst0000664000175000017500000000046613616633007021270 0ustar zuulzuul00000000000000============================== Using Tooz in Your Application ============================== This tutorial is a step-by-step walk-through demonstrating how to use tooz in your application. .. toctree:: :maxdepth: 2 coordinator group_membership leader_election lock hashring partitioner tooz-2.0.0/doc/source/user/tutorial/hashring.rst0000664000175000017500000000052113616633007021754 0ustar zuulzuul00000000000000=========== Hash ring =========== Tooz provides a consistent hash ring implementation. It can be used to map objects (represented via binary keys) to one or several nodes. When the node list changes, the rebalancing of objects across the ring is kept minimal. .. literalinclude:: ../../../../examples/hashring.py :language: python tooz-2.0.0/doc/source/user/tutorial/partitioner.rst0000664000175000017500000000060313616633007022512 0ustar zuulzuul00000000000000============= Partitioner ============= Tooz provides a partitioner object based on its consistent hash ring implementation. It can be used to map Python objects to one or several nodes. The partitioner object automatically keeps track of nodes joining and leaving the group, so the rebalancing is managed. .. literalinclude:: ../../../../examples/partitioner.py :language: python tooz-2.0.0/doc/source/user/tutorial/group_membership.rst0000664000175000017500000000304113616633007023520 0ustar zuulzuul00000000000000================ Group Membership ================ Basic operations ================ One of the feature provided by the coordinator is the ability to handle group membership. Once a group is created, any coordinator can join the group and become a member of it. Any coordinator can be notified when a members joins or leaves the group. .. literalinclude:: ../../../../examples/group_membership.py :language: python Note that all the operation are asynchronous. That means you cannot be sure that your group has been created or joined before you call the :meth:`tooz.coordination.CoordAsyncResult.get` method. You can also leave a group using the :meth:`tooz.coordination.CoordinationDriver.leave_group` method. The list of all available groups is retrievable via the :meth:`tooz.coordination.CoordinationDriver.get_groups` method. Watching Group Changes ====================== It's possible to watch and get notified when the member list of a group changes. That's useful to run callback functions whenever something happens in that group. .. literalinclude:: ../../../../examples/group_membership_watch.py :language: python Using :meth:`tooz.coordination.CoordinationDriver.watch_join_group` and :meth:`tooz.coordination.CoordinationDriver.watch_leave_group` your application can be notified each time a member join or leave a group. To stop watching an event, the two methods :meth:`tooz.coordination.CoordinationDriver.unwatch_join_group` and :meth:`tooz.coordination.CoordinationDriver.unwatch_leave_group` allow to unregister a particular callback. tooz-2.0.0/doc/source/user/tutorial/coordinator.rst0000664000175000017500000000307013616633007022476 0ustar zuulzuul00000000000000====================== Creating A Coordinator ====================== The principal object provided by tooz is the *coordinator*. It allows you to use various features, such as group membership, leader election or distributed locking. The features provided by tooz coordinator are implemented using different drivers. When creating a coordinator, you need to specify which back-end driver you want it to use. Different drivers may provide different set of capabilities. If a driver does not support a feature, it will raise a :class:`~tooz.NotImplemented` exception. This example program loads a basic coordinator using the ZooKeeper based driver. .. literalinclude:: ../../../../examples/coordinator.py :language: python The second argument passed to the coordinator must be a unique identifier identifying the running program. After the coordinator is created, it can be used to use the various features provided. In order to keep the connection to the coordination server active, the method :meth:`~tooz.coordination.CoordinationDriver.heartbeat` method must be called regularly. This will ensure that the coordinator is not considered dead by other program participating in the coordination. Unless you want to call it manually, you can use tooz builtin heartbeat manager by passing the `start_heart` argument. .. literalinclude:: ../../../../examples/coordinator_heartbeat.py :language: python heartbeat at different moment or intervals. Note that certain drivers, such as `memcached` are heavily based on timeout, so the interval used to run the heartbeat is important. tooz-2.0.0/doc/source/user/compatibility.rst0000664000175000017500000000657113616633007021172 0ustar zuulzuul00000000000000============= Compatibility ============= Grouping ======== APIs ---- * :py:meth:`~tooz.coordination.CoordinationDriver.watch_join_group` * :py:meth:`~tooz.coordination.CoordinationDriver.unwatch_join_group` * :py:meth:`~tooz.coordination.CoordinationDriver.watch_leave_group` * :py:meth:`~tooz.coordination.CoordinationDriver.unwatch_leave_group` * :py:meth:`~tooz.coordination.CoordinationDriver.create_group` * :py:meth:`~tooz.coordination.CoordinationDriver.get_groups` * :py:meth:`~tooz.coordination.CoordinationDriver.join_group` * :py:meth:`~tooz.coordination.CoordinationDriver.leave_group` * :py:meth:`~tooz.coordination.CoordinationDriver.delete_group` * :py:meth:`~tooz.coordination.CoordinationDriver.get_members` * :py:meth:`~tooz.coordination.CoordinationDriver.get_member_capabilities` * :py:meth:`~tooz.coordination.CoordinationDriver.update_capabilities` Driver support -------------- .. list-table:: :header-rows: 1 * - Driver - Supported * - :py:class:`~tooz.drivers.consul.ConsulDriver` - No * - :py:class:`~tooz.drivers.etcd.EtcdDriver` - No * - :py:class:`~tooz.drivers.file.FileDriver` - Yes * - :py:class:`~tooz.drivers.ipc.IPCDriver` - No * - :py:class:`~tooz.drivers.memcached.MemcachedDriver` - Yes * - :py:class:`~tooz.drivers.mysql.MySQLDriver` - No * - :py:class:`~tooz.drivers.pgsql.PostgresDriver` - No * - :py:class:`~tooz.drivers.redis.RedisDriver` - Yes * - :py:class:`~tooz.drivers.zake.ZakeDriver` - Yes * - :py:class:`~tooz.drivers.zookeeper.KazooDriver` - Yes Leaders ======= APIs ---- * :py:meth:`~tooz.coordination.CoordinationDriver.watch_elected_as_leader` * :py:meth:`~tooz.coordination.CoordinationDriver.unwatch_elected_as_leader` * :py:meth:`~tooz.coordination.CoordinationDriver.stand_down_group_leader` * :py:meth:`~tooz.coordination.CoordinationDriver.get_leader` Driver support -------------- .. list-table:: :header-rows: 1 * - Driver - Supported * - :py:class:`~tooz.drivers.consul.ConsulDriver` - No * - :py:class:`~tooz.drivers.etcd.EtcdDriver` - No * - :py:class:`~tooz.drivers.file.FileDriver` - No * - :py:class:`~tooz.drivers.ipc.IPCDriver` - No * - :py:class:`~tooz.drivers.memcached.MemcachedDriver` - Yes * - :py:class:`~tooz.drivers.mysql.MySQLDriver` - No * - :py:class:`~tooz.drivers.pgsql.PostgresDriver` - No * - :py:class:`~tooz.drivers.redis.RedisDriver` - Yes * - :py:class:`~tooz.drivers.zake.ZakeDriver` - Yes * - :py:class:`~tooz.drivers.zookeeper.KazooDriver` - Yes Locking ======= APIs ---- * :py:meth:`~tooz.coordination.CoordinationDriver.get_lock` Driver support -------------- .. list-table:: :header-rows: 1 * - Driver - Supported * - :py:class:`~tooz.drivers.consul.ConsulDriver` - Yes * - :py:class:`~tooz.drivers.etcd.EtcdDriver` - Yes * - :py:class:`~tooz.drivers.file.FileDriver` - Yes * - :py:class:`~tooz.drivers.ipc.IPCDriver` - Yes * - :py:class:`~tooz.drivers.memcached.MemcachedDriver` - Yes * - :py:class:`~tooz.drivers.mysql.MySQLDriver` - Yes * - :py:class:`~tooz.drivers.pgsql.PostgresDriver` - Yes * - :py:class:`~tooz.drivers.redis.RedisDriver` - Yes * - :py:class:`~tooz.drivers.zake.ZakeDriver` - Yes * - :py:class:`~tooz.drivers.zookeeper.KazooDriver` - Yes tooz-2.0.0/setup-etcd-env.sh0000775000175000017500000000132413616633007015735 0ustar zuulzuul00000000000000#!/bin/bash set -eux if [ -z "$(which etcd)" ]; then ETCD_VERSION=3.1.3 case `uname -s` in Darwin) OS=darwin SUFFIX=zip ;; Linux) OS=linux SUFFIX=tar.gz ;; *) echo "Unsupported OS" exit 1 esac case `uname -m` in x86_64) MACHINE=amd64 ;; *) echo "Unsupported machine" exit 1 esac TARBALL_NAME=etcd-v${ETCD_VERSION}-$OS-$MACHINE test ! -d "$TARBALL_NAME" && curl -L https://github.com/coreos/etcd/releases/download/v${ETCD_VERSION}/${TARBALL_NAME}.${SUFFIX} | tar xz export PATH=$PATH:$TARBALL_NAME fi $* tooz-2.0.0/requirements.txt0000664000175000017500000000076313616633007016025 0ustar zuulzuul00000000000000# The order of packages is significant, because pip processes them in the order # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. pbr>=1.6 # Apache-2.0 stevedore>=1.16.0 # Apache-2.0 six>=1.9.0 # MIT voluptuous>=0.8.9 # BSD License msgpack>=0.4.0 # Apache-2.0 fasteners>=0.7 # Apache-2.0 tenacity>=3.2.1 # Apache-2.0 futurist>=1.2.0 # Apache-2.0 oslo.utils>=3.15.0 # Apache-2.0 oslo.serialization>=1.10.0 # Apache-2.0 tooz-2.0.0/README.rst0000664000175000017500000000140613616633007014223 0ustar zuulzuul00000000000000Tooz ==== .. image:: https://img.shields.io/pypi/v/tooz.svg :target: https://pypi.org/project/tooz/ :alt: Latest Version .. image:: https://img.shields.io/pypi/dm/tooz.svg :target: https://pypi.org/project/tooz/ :alt: Downloads The Tooz project aims at centralizing the most common distributed primitives like group membership protocol, lock service and leader election by providing a coordination API helping developers to build distributed applications. * Free software: Apache license * Documentation: https://docs.openstack.org/tooz/latest/ * Source: https://opendev.org/openstack/tooz * Bugs: https://bugs.launchpad.net/python-tooz/ * Release notes: https://docs.openstack.org/releasenotes/tooz Join us ------- - https://launchpad.net/python-tooz tooz-2.0.0/PKG-INFO0000664000175000017500000000417613616633265013646 0ustar zuulzuul00000000000000Metadata-Version: 2.1 Name: tooz Version: 2.0.0 Summary: Coordination library for distributed systems. Home-page: https://docs.openstack.org/tooz/latest/ Author: OpenStack Author-email: openstack-discuss@lists.openstack.org License: Apache-2 Description: Tooz ==== .. image:: https://img.shields.io/pypi/v/tooz.svg :target: https://pypi.org/project/tooz/ :alt: Latest Version .. image:: https://img.shields.io/pypi/dm/tooz.svg :target: https://pypi.org/project/tooz/ :alt: Downloads The Tooz project aims at centralizing the most common distributed primitives like group membership protocol, lock service and leader election by providing a coordination API helping developers to build distributed applications. * Free software: Apache license * Documentation: https://docs.openstack.org/tooz/latest/ * Source: https://opendev.org/openstack/tooz * Bugs: https://bugs.launchpad.net/python-tooz/ * Release notes: https://docs.openstack.org/releasenotes/tooz Join us ------- - https://launchpad.net/python-tooz Platform: UNKNOWN Classifier: Environment :: OpenStack Classifier: Intended Audience :: Developers Classifier: Intended Audience :: Information Technology Classifier: License :: OSI Approved :: Apache Software License Classifier: Operating System :: POSIX :: Linux Classifier: Programming Language :: Python Classifier: Programming Language :: Python :: 3 Classifier: Programming Language :: Python :: 3.6 Classifier: Programming Language :: Python :: 3.7 Classifier: Programming Language :: Python :: 3 :: Only Classifier: Programming Language :: Python :: Implementation :: CPython Classifier: Topic :: System :: Distributed Computing Requires-Python: >=3.6 Provides-Extra: consul Provides-Extra: etcd Provides-Extra: etcd3 Provides-Extra: etcd3gw Provides-Extra: zake Provides-Extra: redis Provides-Extra: postgresql Provides-Extra: mysql Provides-Extra: zookeeper Provides-Extra: memcached Provides-Extra: ipc Provides-Extra: test tooz-2.0.0/run-tests.sh0000775000175000017500000000076613616633007015047 0ustar zuulzuul00000000000000#!/bin/bash set -e set -x if [ -n "$TOOZ_TEST_DRIVERS" ] then IFS="," for TOOZ_TEST_DRIVER in $TOOZ_TEST_DRIVERS do IFS=" " TOOZ_TEST_DRIVER=(${TOOZ_TEST_DRIVER}) SETUP_ENV_SCRIPT="./setup-${TOOZ_TEST_DRIVER[0]}-env.sh" [ -x $SETUP_ENV_SCRIPT ] || unset SETUP_ENV_SCRIPT $SETUP_ENV_SCRIPT pifpaf -e TOOZ_TEST run "${TOOZ_TEST_DRIVER[@]}" -- $* done unset IFS else for d in $TOOZ_TEST_URLS do TOOZ_TEST_URL=$d $* done fi tooz-2.0.0/setup.py0000664000175000017500000000200413616633007014241 0ustar zuulzuul00000000000000# Copyright (c) 2013 Hewlett-Packard Development Company, L.P. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. # THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT import setuptools # In python < 2.7.4, a lazy loading of package `pbr` will break # setuptools if some other modules registered functions in `atexit`. # solution from: http://bugs.python.org/issue15881#msg170215 try: import multiprocessing # noqa except ImportError: pass setuptools.setup( setup_requires=['pbr>=1.8'], pbr=True) tooz-2.0.0/setup.cfg0000664000175000017500000000354213616633265014366 0ustar zuulzuul00000000000000[metadata] name = tooz author = OpenStack author-email = openstack-discuss@lists.openstack.org summary = Coordination library for distributed systems. description-file = README.rst license = Apache-2 home-page = https://docs.openstack.org/tooz/latest/ python-requires = >=3.6 classifier = Environment :: OpenStack Intended Audience :: Developers Intended Audience :: Information Technology License :: OSI Approved :: Apache Software License Operating System :: POSIX :: Linux Programming Language :: Python Programming Language :: Python :: 3 Programming Language :: Python :: 3.6 Programming Language :: Python :: 3.7 Programming Language :: Python :: 3 :: Only Programming Language :: Python :: Implementation :: CPython Topic :: System :: Distributed Computing [files] packages = tooz [entry_points] tooz.backends = etcd = tooz.drivers.etcd:EtcdDriver etcd3 = tooz.drivers.etcd3:Etcd3Driver etcd3+http = tooz.drivers.etcd3gw:Etcd3Driver kazoo = tooz.drivers.zookeeper:KazooDriver zake = tooz.drivers.zake:ZakeDriver memcached = tooz.drivers.memcached:MemcachedDriver ipc = tooz.drivers.ipc:IPCDriver redis = tooz.drivers.redis:RedisDriver postgresql = tooz.drivers.pgsql:PostgresDriver mysql = tooz.drivers.mysql:MySQLDriver file = tooz.drivers.file:FileDriver zookeeper = tooz.drivers.zookeeper:KazooDriver consul = tooz.drivers.consul:ConsulDriver [extras] consul = python-consul>=0.4.7 # MIT License etcd = requests>=2.10.0 # Apache-2.0 etcd3 = etcd3>=0.6.2 # Apache-2.0 grpcio>=1.18.0 etcd3gw = etcd3gw>=0.1.0 # Apache-2.0 zake = zake>=0.1.6 # Apache-2.0 redis = redis>=2.10.0 # MIT postgresql = psycopg2>=2.5 # LGPL/ZPL mysql = PyMySQL>=0.6.2 # MIT License zookeeper = kazoo>=2.2 # Apache-2.0 memcached = pymemcache!=1.3.0,>=1.2.9 # Apache 2.0 License ipc = sysv-ipc>=0.6.8 # BSD License [egg_info] tag_build = tag_date = 0 tooz-2.0.0/playbooks/0000775000175000017500000000000013616633265014544 5ustar zuulzuul00000000000000tooz-2.0.0/playbooks/stop-redis.yaml0000664000175000017500000000062013616633007017511 0ustar zuulzuul00000000000000# On Ubuntu, just installing the redis-server package starts Redis. As a # result, when we try to start one via pifpaf it fails because the port # is already in use. # See https://bugs.launchpad.net/python-tooz/+bug/1828610 - hosts: all name: Stop Redis server before running tests tasks: - name: Stop Redis service: name: redis-server state: stopped become: true tooz-2.0.0/LICENSE0000664000175000017500000002363713616633007013553 0ustar zuulzuul00000000000000 Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. tooz-2.0.0/CONTRIBUTING.rst0000664000175000017500000000122013616633007015167 0ustar zuulzuul00000000000000If you would like to contribute to the development of OpenStack, you must follow the steps in this page: http://docs.openstack.org/infra/manual/developers.html If you already have a good understanding of how the system works and your OpenStack accounts are set up, you can skip to the development workflow section of this documentation to learn how changes to OpenStack should be submitted for review via the Gerrit tool: http://docs.openstack.org/infra/manual/developers.html#development-workflow Pull requests submitted through GitHub will be ignored. Bugs should be filed on Launchpad, not GitHub: https://bugs.launchpad.net/python-tooz/ tooz-2.0.0/.stestr.conf0000664000175000017500000000003113616633007014776 0ustar zuulzuul00000000000000[DEFAULT] test_path=tooz tooz-2.0.0/tooz/0000775000175000017500000000000013616633265013534 5ustar zuulzuul00000000000000tooz-2.0.0/tooz/locking.py0000664000175000017500000000746013616633007015535 0ustar zuulzuul00000000000000# -*- coding: utf-8 -*- # # Copyright (C) 2014 eNovance Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import abc import six import tooz from tooz import coordination class _LockProxy(object): def __init__(self, lock, *args, **kwargs): self.lock = lock self.args = args self.kwargs = kwargs def __enter__(self): return self.lock.__enter__(*self.args, **self.kwargs) def __exit__(self, exc_type, exc_val, exc_tb): self.lock.__exit__(exc_type, exc_val, exc_tb) @six.add_metaclass(abc.ABCMeta) class Lock(object): def __init__(self, name): if not name: raise ValueError("Locks must be provided a name") self._name = name @property def name(self): return self._name def __call__(self, *args, **kwargs): return _LockProxy(self, *args, **kwargs) def __enter__(self, *args, **kwargs): acquired = self.acquire(*args, **kwargs) if not acquired: msg = u'Acquiring lock %s failed' % self.name raise coordination.LockAcquireFailed(msg) return self def __exit__(self, exc_type, exc_val, exc_tb): self.release() def is_still_owner(self): """Checks if the lock is still owned by the acquiree. :returns: returns true if still acquired (false if not) and false if the lock was never acquired in the first place or raises ``NotImplemented`` if not implemented. """ raise tooz.NotImplemented @abc.abstractmethod def release(self): """Attempts to release the lock, returns true if released. The behavior of releasing a lock which was not acquired in the first place is undefined (it can range from harmless to releasing some other users lock).. :returns: returns true if released (false if not) :rtype: bool """ def break_(self): """Forcefully release the lock. This is mostly used for testing purposes, to simulate an out of band operation that breaks the lock. Backends may allow waiters to acquire immediately if a lock is broken, or they should raise an exception. Releasing should be successful for objects that believe they hold the lock but do not have the lock anymore. However, they should be careful not to re-break the lock by releasing it, since they may not be the holder anymore. :returns: returns true if forcefully broken (false if not) or raises ``NotImplemented`` if not implemented. """ raise tooz.NotImplemented @abc.abstractmethod def acquire(self, blocking=True, shared=False): """Attempts to acquire the lock. :param blocking: If True, blocks until the lock is acquired. If False, returns right away. Otherwise, the value is used as a timeout value and the call returns maximum after this number of seconds. :param shared: If False, the lock is exclusive. If True, the lock can be shareable or raises ``NotImplemented`` if not implemented. :returns: returns true if acquired (false if not) :rtype: bool """ tooz-2.0.0/tooz/coordination.py0000775000175000017500000007625513616633007016612 0ustar zuulzuul00000000000000# -*- coding: utf-8 -*- # # Copyright (C) 2016 Red Hat, Inc. # Copyright (C) 2013-2014 eNovance Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import abc import collections from concurrent import futures import enum import logging import threading from oslo_utils import encodeutils from oslo_utils import netutils from oslo_utils import timeutils import six from stevedore import driver import tenacity import tooz from tooz import _retry from tooz import partitioner from tooz import utils LOG = logging.getLogger(__name__) TOOZ_BACKENDS_NAMESPACE = "tooz.backends" class Characteristics(enum.Enum): """Attempts to describe the characteristic that a driver supports.""" DISTRIBUTED_ACROSS_THREADS = 'DISTRIBUTED_ACROSS_THREADS' """Coordinator components when used by multiple **threads** work the same as if those components were only used by a single thread.""" DISTRIBUTED_ACROSS_PROCESSES = 'DISTRIBUTED_ACROSS_PROCESSES' """Coordinator components when used by multiple **processes** work the same as if those components were only used by a single thread.""" DISTRIBUTED_ACROSS_HOSTS = 'DISTRIBUTED_ACROSS_HOSTS' """Coordinator components when used by multiple **hosts** work the same as if those components were only used by a single thread.""" NON_TIMEOUT_BASED = 'NON_TIMEOUT_BASED' """The driver has the following property: * Its operations are not based on the timeout of other clients, but on some other more robust mechanisms. """ LINEARIZABLE = 'LINEARIZABLE' """The driver has the following properties: * Ensures each operation must take place before its completion time. * Any operation invoked subsequently must take place after the invocation and by extension, after the original operation itself. """ SEQUENTIAL = 'SEQUENTIAL' """The driver has the following properties: * Operations can take effect before or after completion – but all operations retain the constraint that operations from any given process must take place in that processes order. """ CAUSAL = 'CAUSAL' """The driver has the following properties: * Does **not** have to enforce the order of every operation from a process, perhaps, only causally related operations must occur in order. """ SERIALIZABLE = 'SERIALIZABLE' """The driver has the following properties: * The history of **all** operations is equivalent to one that took place in some single atomic order but with unknown invocation and completion times - it places no bounds on time or order. """ SAME_VIEW_UNDER_PARTITIONS = 'SAME_VIEW_UNDER_PARTITIONS' """When a client is connected to a server and that server is partitioned from a group of other servers it will (somehow) have the same view of data as a client connected to a server on the other side of the partition (typically this is accomplished by write availability being lost and therefore nothing can change). """ SAME_VIEW_ACROSS_CLIENTS = 'SAME_VIEW_ACROSS_CLIENTS' """A client connected to one server will *always* have the same view every other client will have (no matter what server those other clients are connected to). Typically this is a sacrifice in write availability because before a write can be acknowledged it must be acknowledged by *all* servers in a cluster (so that all clients that are connected to those servers read the exact *same* thing). """ class Hooks(list): def run(self, *args, **kwargs): return list(map(lambda cb: cb(*args, **kwargs), self)) class Event(object): """Base class for events.""" class MemberJoinedGroup(Event): """A member joined a group event.""" def __init__(self, group_id, member_id): self.group_id = group_id self.member_id = member_id def __repr__(self): return "<%s: group %s: +member %s>" % (self.__class__.__name__, self.group_id, self.member_id) class MemberLeftGroup(Event): """A member left a group event.""" def __init__(self, group_id, member_id): self.group_id = group_id self.member_id = member_id def __repr__(self): return "<%s: group %s: -member %s>" % (self.__class__.__name__, self.group_id, self.member_id) class LeaderElected(Event): """A leader as been elected.""" def __init__(self, group_id, member_id): self.group_id = group_id self.member_id = member_id class Heart(object): """Coordination drivers main liveness pump (its heart).""" def __init__(self, driver, thread_cls=threading.Thread, event_cls=threading.Event): self._thread_cls = thread_cls self._dead = event_cls() self._runner = None self._driver = driver self._beats = 0 @property def beats(self): """How many times the heart has beaten.""" return self._beats def is_alive(self): """Returns if the heart is beating.""" return not (self._runner is None or not self._runner.is_alive()) def _beat_forever_until_stopped(self): """Inner beating loop.""" retry = tenacity.Retrying( wait=tenacity.wait_fixed(1), before_sleep=tenacity.before_sleep_log(LOG, logging.WARNING), ) while not self._dead.is_set(): with timeutils.StopWatch() as w: wait_until_next_beat = retry(self._driver.heartbeat) ran_for = w.elapsed() has_to_sleep_for = wait_until_next_beat - ran_for if has_to_sleep_for < 0: LOG.warning( "Heartbeating took too long to execute (it ran for" " %0.2f seconds which is %0.2f seconds longer than" " the next heartbeat idle time). This may cause" " timeouts (in locks, leadership, ...) to" " happen (which will not end well).", ran_for, ran_for - wait_until_next_beat) self._beats += 1 # NOTE(harlowja): use the event object for waiting and # not a sleep function since doing that will allow this code # to terminate early if stopped via the stop() method vs # having to wait until the sleep function returns. # NOTE(jd): Wait for only the half time of what we should. # This is a measure of safety, better be too soon than too late. self._dead.wait(has_to_sleep_for / 2.0) def start(self, thread_cls=None): """Starts the heart beating thread (noop if already started).""" if not self.is_alive(): self._dead.clear() self._beats = 0 if thread_cls is None: thread_cls = self._thread_cls self._runner = thread_cls(target=self._beat_forever_until_stopped) self._runner.daemon = True self._runner.start() def stop(self): """Requests the heart beating thread to stop beating.""" self._dead.set() def wait(self, timeout=None): """Wait up to given timeout for the heart beating thread to stop.""" self._runner.join(timeout) return self._runner.is_alive() class CoordinationDriver(object): requires_beating = False """ Usage requirement that if true requires that the :py:meth:`~.heartbeat` be called periodically (at a given rate) to avoid locks, sessions and other from being automatically closed/discarded by the coordinators backing store. """ CHARACTERISTICS = () """ Tuple of :py:class:`~tooz.coordination.Characteristics` introspectable enum member(s) that can be used to interogate how this driver works. """ def __init__(self, member_id, parsed_url, options): super(CoordinationDriver, self).__init__() self._member_id = member_id self._started = False self._hooks_join_group = collections.defaultdict(Hooks) self._hooks_leave_group = collections.defaultdict(Hooks) self._hooks_elected_leader = collections.defaultdict(Hooks) self.requires_beating = ( CoordinationDriver.heartbeat != self.__class__.heartbeat ) self.heart = Heart(self) def _has_hooks_for_group(self, group_id): return (group_id in self._hooks_join_group or group_id in self._hooks_leave_group) def join_partitioned_group( self, group_id, weight=1, partitions=partitioner.Partitioner.DEFAULT_PARTITION_NUMBER): """Join a group and get a partitioner. A partitioner allows to distribute a bunch of objects across several members using a consistent hash ring. Each object gets assigned (at least) one member responsible for it. It's then possible to check which object is owned by any member of the group. This method also creates if necessary, and joins the group with the selected weight. :param group_id: The group to create a partitioner for. :param weight: The weight to use in the hashring for this node. :param partitions: The number of partitions to create. :return: A :py:class:`~tooz.partitioner.Partitioner` object. """ self.join_group_create(group_id, capabilities={'weight': weight}) return partitioner.Partitioner(self, group_id, partitions=partitions) def leave_partitioned_group(self, partitioner): """Leave a partitioned group. This leaves the partitioned group and stop the partitioner. :param group_id: The group to create a partitioner for. """ leave = self.leave_group(partitioner.group_id) partitioner.stop() return leave.get() @staticmethod def run_watchers(timeout=None): """Run the watchers callback. This may also activate :py:meth:`.run_elect_coordinator` (depending on driver implementation). """ raise tooz.NotImplemented @staticmethod def run_elect_coordinator(): """Try to leader elect this coordinator & activate hooks on success.""" raise tooz.NotImplemented def watch_join_group(self, group_id, callback): """Call a function when group_id sees a new member joined. The callback functions will be executed when `run_watchers` is called. :param group_id: The group id to watch :param callback: The function to execute when a member joins this group """ self._hooks_join_group[group_id].append(callback) def unwatch_join_group(self, group_id, callback): """Stop executing a function when a group_id sees a new member joined. :param group_id: The group id to unwatch :param callback: The function that was executed when a member joined this group """ try: # Check if group_id is in hooks to avoid creating a default empty # entry in hooks list. if group_id not in self._hooks_join_group: raise ValueError self._hooks_join_group[group_id].remove(callback) except ValueError: raise WatchCallbackNotFound(group_id, callback) if not self._hooks_join_group[group_id]: del self._hooks_join_group[group_id] def watch_leave_group(self, group_id, callback): """Call a function when group_id sees a new member leaving. The callback functions will be executed when `run_watchers` is called. :param group_id: The group id to watch :param callback: The function to execute when a member leaves this group """ self._hooks_leave_group[group_id].append(callback) def unwatch_leave_group(self, group_id, callback): """Stop executing a function when a group_id sees a new member leaving. :param group_id: The group id to unwatch :param callback: The function that was executed when a member left this group """ try: # Check if group_id is in hooks to avoid creating a default empty # entry in hooks list. if group_id not in self._hooks_leave_group: raise ValueError self._hooks_leave_group[group_id].remove(callback) except ValueError: raise WatchCallbackNotFound(group_id, callback) if not self._hooks_leave_group[group_id]: del self._hooks_leave_group[group_id] def watch_elected_as_leader(self, group_id, callback): """Call a function when member gets elected as leader. The callback functions will be executed when `run_watchers` is called. :param group_id: The group id to watch :param callback: The function to execute when a member leaves this group """ self._hooks_elected_leader[group_id].append(callback) def unwatch_elected_as_leader(self, group_id, callback): """Call a function when member gets elected as leader. The callback functions will be executed when `run_watchers` is called. :param group_id: The group id to watch :param callback: The function to execute when a member leaves this group """ try: self._hooks_elected_leader[group_id].remove(callback) except ValueError: raise WatchCallbackNotFound(group_id, callback) if not self._hooks_elected_leader[group_id]: del self._hooks_elected_leader[group_id] @staticmethod def stand_down_group_leader(group_id): """Stand down as the group leader if we are. :param group_id: The group where we don't want to be a leader anymore """ raise tooz.NotImplemented @property def is_started(self): return self._started def start(self, start_heart=False): """Start the service engine. If needed, the establishment of a connection to the servers is initiated. """ if self._started: raise tooz.ToozError( "Can not start a driver which has not been stopped") self._start() if self.requires_beating and start_heart: self.heart.start() self._started = True # Tracks which group are joined self._joined_groups = set() def _start(self): pass def stop(self): """Stop the service engine. If needed, the connection to servers is closed and the client will disappear from all joined groups. """ if not self._started: raise tooz.ToozError( "Can not stop a driver which has not been started") if self.heart.is_alive(): self.heart.stop() self.heart.wait() # Some of the drivers modify joined_groups when being called to leave # so clone it so that we aren't modifying something while iterating. joined_groups = self._joined_groups.copy() leaving = [self.leave_group(group) for group in joined_groups] for fut in leaving: try: fut.get() except tooz.ToozError: # Whatever happens, ignore. Maybe we got booted out/never # existed in the first place, or something is down, but we just # want to call _stop after whatever happens to not leak any # connection. pass self._stop() self._started = False def _stop(self): pass @staticmethod def create_group(group_id): """Request the creation of a group asynchronously. :param group_id: the id of the group to create :type group_id: ascii bytes :returns: None :rtype: CoordAsyncResult """ raise tooz.NotImplemented @staticmethod def get_groups(): """Return the list composed by all groups ids asynchronously. :returns: the list of all created group ids :rtype: CoordAsyncResult """ raise tooz.NotImplemented @staticmethod def join_group(group_id, capabilities=b""): """Join a group and establish group membership asynchronously. :param group_id: the id of the group to join :type group_id: ascii bytes :param capabilities: the capabilities of the joined member :type capabilities: object :returns: None :rtype: CoordAsyncResult """ raise tooz.NotImplemented @_retry.retry() def join_group_create(self, group_id, capabilities=b""): """Join a group and create it if necessary. If the group cannot be joined because it does not exist, it is created before being joined. This function will keep retrying until it can create the group and join it. Since nothing is transactional, it may have to retry several times if another member is creating/deleting the group at the same time. :param group_id: Identifier of the group to join and create :param capabilities: the capabilities of the joined member """ req = self.join_group(group_id, capabilities) try: req.get() except GroupNotCreated: req = self.create_group(group_id) try: req.get() except GroupAlreadyExist: # The group might have been created in the meantime, ignore pass # Now retry to join the group raise _retry.TryAgain @staticmethod def leave_group(group_id): """Leave a group asynchronously. :param group_id: the id of the group to leave :type group_id: ascii bytes :returns: None :rtype: CoordAsyncResult """ raise tooz.NotImplemented @staticmethod def delete_group(group_id): """Delete a group asynchronously. :param group_id: the id of the group to leave :type group_id: ascii bytes :returns: Result :rtype: CoordAsyncResult """ raise tooz.NotImplemented @staticmethod def get_members(group_id): """Return the set of all members ids of the specified group. :returns: set of all created group ids :rtype: CoordAsyncResult """ raise tooz.NotImplemented @staticmethod def get_member_capabilities(group_id, member_id): """Return the capabilities of a member asynchronously. :param group_id: the id of the group of the member :type group_id: ascii bytes :param member_id: the id of the member :type member_id: ascii bytes :returns: capabilities of a member :rtype: CoordAsyncResult """ raise tooz.NotImplemented @staticmethod def get_member_info(group_id, member_id): """Return the statistics and capabilities of a member asynchronously. :param group_id: the id of the group of the member :type group_id: ascii bytes :param member_id: the id of the member :type member_id: ascii bytes :returns: capabilities and statistics of a member :rtype: CoordAsyncResult """ raise tooz.NotImplemented @staticmethod def update_capabilities(group_id, capabilities): """Update member capabilities in the specified group. :param group_id: the id of the group of the current member :type group_id: ascii bytes :param capabilities: the capabilities of the updated member :type capabilities: object :returns: None :rtype: CoordAsyncResult """ raise tooz.NotImplemented @staticmethod def get_leader(group_id): """Return the leader for a group. :param group_id: the id of the group: :returns: the leader :rtype: CoordAsyncResult """ raise tooz.NotImplemented @staticmethod def get_lock(name): """Return a distributed lock. This is a exclusive lock, a second call to acquire() will block or return False. :param name: The lock name that is used to identify it across all nodes. """ raise tooz.NotImplemented @staticmethod def heartbeat(): """Update member status to indicate it is still alive. Method to run once in a while to be sure that the member is not dead and is still an active member of a group. :return: The number of seconds to wait before sending a new heartbeat. """ pass @six.add_metaclass(abc.ABCMeta) class CoordAsyncResult(object): """Representation of an asynchronous task. Every call API returns an CoordAsyncResult object on which the result or the status of the task can be requested. """ @abc.abstractmethod def get(self, timeout=None): """Retrieve the result of the corresponding asynchronous call. :param timeout: block until the timeout expire. :type timeout: float """ @abc.abstractmethod def done(self): """Returns True if the task is done, False otherwise.""" class CoordinatorResult(CoordAsyncResult): """Asynchronous result that references a future.""" def __init__(self, fut, failure_translator=None): self._fut = fut self._failure_translator = failure_translator def get(self, timeout=None): try: if self._failure_translator: with self._failure_translator(): return self._fut.result(timeout=timeout) else: return self._fut.result(timeout=timeout) except futures.TimeoutError as e: utils.raise_with_cause(OperationTimedOut, encodeutils.exception_to_unicode(e), cause=e) def done(self): return self._fut.done() class CoordinationDriverWithExecutor(CoordinationDriver): EXCLUDE_OPTIONS = None def __init__(self, member_id, parsed_url, options): self._options = utils.collapse(options, exclude=self.EXCLUDE_OPTIONS) self._executor = utils.ProxyExecutor.build( self.__class__.__name__, self._options) super(CoordinationDriverWithExecutor, self).__init__( member_id, parsed_url, options) def start(self, start_heart=False): self._executor.start() super(CoordinationDriverWithExecutor, self).start(start_heart) def stop(self): super(CoordinationDriverWithExecutor, self).stop() self._executor.stop() class CoordinationDriverCachedRunWatchers(CoordinationDriver): """Coordination driver with a `run_watchers` implementation. This implementation of `run_watchers` is based on a cache of the group members between each run of `run_watchers` that is being updated between each run. """ def __init__(self, member_id, parsed_url, options): super(CoordinationDriverCachedRunWatchers, self).__init__( member_id, parsed_url, options) # A cache for group members self._group_members = collections.defaultdict(set) self._joined_groups = set() def _init_watch_group(self, group_id): if group_id not in self._group_members: members = self.get_members(group_id) self._group_members[group_id] = members.get() def watch_join_group(self, group_id, callback): self._init_watch_group(group_id) super(CoordinationDriverCachedRunWatchers, self).watch_join_group( group_id, callback) def unwatch_join_group(self, group_id, callback): super(CoordinationDriverCachedRunWatchers, self).unwatch_join_group( group_id, callback) if (not self._has_hooks_for_group(group_id) and group_id in self._group_members): del self._group_members[group_id] def watch_leave_group(self, group_id, callback): self._init_watch_group(group_id) super(CoordinationDriverCachedRunWatchers, self).watch_leave_group( group_id, callback) def unwatch_leave_group(self, group_id, callback): super(CoordinationDriverCachedRunWatchers, self).unwatch_leave_group( group_id, callback) if (not self._has_hooks_for_group(group_id) and group_id in self._group_members): del self._group_members[group_id] def run_watchers(self, timeout=None): with timeutils.StopWatch(duration=timeout) as w: result = [] group_with_hooks = set(self._hooks_join_group.keys()).union( set(self._hooks_leave_group.keys())) for group_id in group_with_hooks: try: group_members = self.get_members(group_id).get( timeout=w.leftover(return_none=True)) except GroupNotCreated: group_members = set() if (group_id in self._joined_groups and self._member_id not in group_members): self._joined_groups.discard(group_id) old_group_members = self._group_members.get(group_id, set()) for member_id in (group_members - old_group_members): result.extend( self._hooks_join_group[group_id].run( MemberJoinedGroup(group_id, member_id))) for member_id in (old_group_members - group_members): result.extend( self._hooks_leave_group[group_id].run( MemberLeftGroup(group_id, member_id))) self._group_members[group_id] = group_members return result def get_coordinator(backend_url, member_id, characteristics=frozenset(), **kwargs): """Initialize and load the backend. :param backend_url: the backend URL to use :type backend: str :param member_id: the id of the member :type member_id: ascii bytes :param characteristics: set :type characteristics: set of :py:class:`.Characteristics` that will be matched to the requested driver (this **will** become a **required** parameter in a future tooz version) :param kwargs: additional coordinator options (these take precedence over options of the **same** name found in the ``backend_url`` arguments query string) """ parsed_url = netutils.urlsplit(backend_url) parsed_qs = six.moves.urllib.parse.parse_qs(parsed_url.query) if kwargs: options = {} for (k, v) in six.iteritems(kwargs): options[k] = [v] for (k, v) in six.iteritems(parsed_qs): if k not in options: options[k] = v else: options = parsed_qs d = driver.DriverManager( namespace=TOOZ_BACKENDS_NAMESPACE, name=parsed_url.scheme, invoke_on_load=True, invoke_args=(member_id, parsed_url, options)).driver characteristics = set(characteristics) driver_characteristics = set(getattr(d, 'CHARACTERISTICS', set())) missing_characteristics = characteristics - driver_characteristics if missing_characteristics: raise ToozDriverChosenPoorly("Desired characteristics %s" " is not a strict subset of driver" " characteristics %s, %s" " characteristics were not found" % (characteristics, driver_characteristics, missing_characteristics)) return d # TODO(harlowja): We'll have to figure out a way to remove this 'alias' at # some point in the future (when we have a better way to tell people it has # moved without messing up their exception catching hierarchy). ToozError = tooz.ToozError class ToozDriverChosenPoorly(tooz.ToozError): """Raised when a driver does not match desired characteristics.""" class ToozConnectionError(tooz.ToozError): """Exception raised when the client cannot connect to the server.""" class OperationTimedOut(tooz.ToozError): """Exception raised when an operation times out.""" class LockAcquireFailed(tooz.ToozError): """Exception raised when a lock acquire fails in a context manager.""" class GroupNotCreated(tooz.ToozError): """Exception raised when the caller request an nonexistent group.""" def __init__(self, group_id): self.group_id = group_id super(GroupNotCreated, self).__init__( "Group %s does not exist" % group_id) class GroupAlreadyExist(tooz.ToozError): """Exception raised trying to create an already existing group.""" def __init__(self, group_id): self.group_id = group_id super(GroupAlreadyExist, self).__init__( "Group %s already exists" % group_id) class MemberAlreadyExist(tooz.ToozError): """Exception raised trying to join a group already joined.""" def __init__(self, group_id, member_id): self.group_id = group_id self.member_id = member_id super(MemberAlreadyExist, self).__init__( "Member %s has already joined %s" % (member_id, group_id)) class MemberNotJoined(tooz.ToozError): """Exception raised trying to access a member not in a group.""" def __init__(self, group_id, member_id): self.group_id = group_id self.member_id = member_id super(MemberNotJoined, self).__init__("Member %s has not joined %s" % (member_id, group_id)) class GroupNotEmpty(tooz.ToozError): "Exception raised when the caller try to delete a group with members." def __init__(self, group_id): self.group_id = group_id super(GroupNotEmpty, self).__init__("Group %s is not empty" % group_id) class WatchCallbackNotFound(tooz.ToozError): """Exception raised when unwatching a group. Raised when the caller tries to unwatch a group with a callback that does not exist. """ def __init__(self, group_id, callback): self.group_id = group_id self.callback = callback super(WatchCallbackNotFound, self).__init__( 'Callback %s is not registered on group %s' % (callback.__name__, group_id)) # TODO(harlowja,jd): We'll have to figure out a way to remove this 'alias' at # some point in the future (when we have a better way to tell people it has # moved without messing up their exception catching hierarchy). SerializationError = utils.SerializationError tooz-2.0.0/tooz/hashring.py0000664000175000017500000001125713616633007015711 0ustar zuulzuul00000000000000# -*- coding: utf-8 -*- # # Copyright (C) 2016 Red Hat, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import bisect import hashlib import six import tooz from tooz import utils class UnknownNode(tooz.ToozError): """Node is unknown.""" def __init__(self, node): super(UnknownNode, self).__init__("Unknown node `%s'" % node) self.node = node class HashRing(object): """Map objects onto nodes based on their consistent hash.""" DEFAULT_PARTITION_NUMBER = 2**5 def __init__(self, nodes, partitions=DEFAULT_PARTITION_NUMBER): """Create a new hashring. :param nodes: List of nodes where objects will be mapped onto. :param partitions: Number of partitions to spread objects onto. """ self.nodes = {} self._ring = dict() self._partitions = [] self._partition_number = partitions self.add_nodes(set(nodes)) def add_node(self, node, weight=1): """Add a node to the hashring. :param node: Node to add. :param weight: How many resource instances this node should manage compared to the other nodes (default 1). Higher weights will be assigned more resources. Three nodes A, B and C with weights 1, 2 and 3 will each handle 1/6, 1/3 and 1/2 of the resources, respectively. """ return self.add_nodes((node,), weight) def add_nodes(self, nodes, weight=1): """Add nodes to the hashring with equal weight :param nodes: Nodes to add. :param weight: How many resource instances this node should manage compared to the other nodes (default 1). Higher weights will be assigned more resources. Three nodes A, B and C with weights 1, 2 and 3 will each handle 1/6, 1/3 and 1/2 of the resources, respectively. """ for node in nodes: key = utils.to_binary(node, 'utf-8') key_hash = hashlib.md5(key) for r in six.moves.range(self._partition_number * weight): key_hash.update(key) self._ring[self._hash2int(key_hash)] = node self.nodes[node] = weight self._partitions = sorted(self._ring.keys()) def remove_node(self, node): """Remove a node from the hashring. Raises py:exc:`UnknownNode` :param node: Node to remove. """ try: weight = self.nodes.pop(node) except KeyError: raise UnknownNode(node) key = utils.to_binary(node, 'utf-8') key_hash = hashlib.md5(key) for r in six.moves.range(self._partition_number * weight): key_hash.update(key) del self._ring[self._hash2int(key_hash)] self._partitions = sorted(self._ring.keys()) @staticmethod def _hash2int(key): return int(key.hexdigest(), 16) def _get_partition(self, data): hashed_key = self._hash2int(hashlib.md5(data)) position = bisect.bisect(self._partitions, hashed_key) return position if position < len(self._partitions) else 0 def _get_node(self, partition): return self._ring[self._partitions[partition]] def get_nodes(self, data, ignore_nodes=None, replicas=1): """Get the set of nodes which the supplied data map onto. :param data: A byte identifier to be mapped across the ring. :param ignore_nodes: Set of nodes to ignore. :param replicas: Number of replicas to use. :return: A set of nodes whose length depends on the number of replicas. """ partition = self._get_partition(data) ignore_nodes = set(ignore_nodes) if ignore_nodes else set() candidates = set(self.nodes.keys()) - ignore_nodes replicas = min(replicas, len(candidates)) nodes = set() while len(nodes) < replicas: node = self._get_node(partition) if node not in ignore_nodes: nodes.add(node) partition = (partition + 1 if partition + 1 < len(self._partitions) else 0) return nodes def __getitem__(self, key): return self.get_nodes(key) def __len__(self): return len(self._partitions) tooz-2.0.0/tooz/partitioner.py0000664000175000017500000000762613616633007016453 0ustar zuulzuul00000000000000# -*- coding: utf-8 -*- # # Copyright (C) 2016 Red Hat, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import six from tooz import hashring class Partitioner(object): """Partition set of objects across several members. Objects to be partitioned should implement the __tooz_hash__ method to identify themselves across the consistent hashring. This should method return bytes. """ DEFAULT_PARTITION_NUMBER = hashring.HashRing.DEFAULT_PARTITION_NUMBER def __init__(self, coordinator, group_id, partitions=DEFAULT_PARTITION_NUMBER): members = coordinator.get_members(group_id) self.partitions = partitions self.group_id = group_id self._coord = coordinator caps = [(m, self._coord.get_member_capabilities(self.group_id, m)) for m in members.get()] self._coord.watch_join_group(self.group_id, self._on_member_join) self._coord.watch_leave_group(self.group_id, self._on_member_leave) self.ring = hashring.HashRing([], partitions=self.partitions) for m_id, cap in caps: self.ring.add_node(m_id, cap.get().get("weight", 1)) def _on_member_join(self, event): weight = self._coord.get_member_capabilities( self.group_id, event.member_id).get().get("weight", 1) self.ring.add_node(event.member_id, weight) def _on_member_leave(self, event): self.ring.remove_node(event.member_id) @staticmethod def _hash_object(obj): if hasattr(obj, "__tooz_hash__"): return obj.__tooz_hash__() return six.text_type(obj).encode('utf8') def members_for_object(self, obj, ignore_members=None, replicas=1): """Return the members responsible for an object. :param obj: The object to check owning for. :param member_id: The member to check if it owns the object. :param ignore_members: Group members to ignore. :param replicas: Number of replicas for the object. """ return self.ring.get_nodes(self._hash_object(obj), ignore_nodes=ignore_members, replicas=replicas) def belongs_to_member(self, obj, member_id, ignore_members=None, replicas=1): """Return whether an object belongs to a member. :param obj: The object to check owning for. :param member_id: The member to check if it owns the object. :param ignore_members: Group members to ignore. :param replicas: Number of replicas for the object. """ return member_id in self.members_for_object( obj, ignore_members=ignore_members, replicas=replicas) def belongs_to_self(self, obj, ignore_members=None, replicas=1): """Return whether an object belongs to this coordinator. :param obj: The object to check owning for. :param ignore_members: Group members to ignore. :param replicas: Number of replicas for the object. """ return self.belongs_to_member(obj, self._coord._member_id, ignore_members=ignore_members, replicas=replicas) def stop(self): """Stop the partitioner.""" self._coord.unwatch_join_group(self.group_id, self._on_member_join) self._coord.unwatch_leave_group(self.group_id, self._on_member_leave) tooz-2.0.0/tooz/tests/0000775000175000017500000000000013616633265014676 5ustar zuulzuul00000000000000tooz-2.0.0/tooz/tests/test_coordination.py0000664000175000017500000012110713616633007020773 0ustar zuulzuul00000000000000# -*- coding: utf-8 -*- # # Copyright © 2013-2015 eNovance Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import threading import time from concurrent import futures import mock from six.moves.urllib import parse from testtools import matchers from testtools import testcase import tooz import tooz.coordination from tooz import tests def try_to_lock_job(name, coord, url, member_id): if not coord: coord = tooz.coordination.get_coordinator( url, member_id) coord.start() lock2 = coord.get_lock(name) return lock2.acquire(blocking=False) class TestAPI(tests.TestWithCoordinator): def assertRaisesAny(self, exc_classes, callable_obj, *args, **kwargs): checkers = [matchers.MatchesException(exc_class) for exc_class in exc_classes] matcher = matchers.Raises(matchers.MatchesAny(*checkers)) callable_obj = testcase.Nullary(callable_obj, *args, **kwargs) self.assertThat(callable_obj, matcher) def test_connection_error_bad_host(self): if (tooz.coordination.Characteristics.DISTRIBUTED_ACROSS_HOSTS not in self._coord.CHARACTERISTICS): self.skipTest("This driver is not distributed across hosts") scheme = parse.urlparse(self.url).scheme coord = tooz.coordination.get_coordinator( "%s://localhost:1/f00" % scheme, self.member_id) self.assertRaises(tooz.coordination.ToozConnectionError, coord.start) def test_stop_first(self): c = tooz.coordination.get_coordinator(self.url, self.member_id) self.assertRaises(tooz.ToozError, c.stop) def test_create_group(self): self._coord.create_group(self.group_id).get() all_group_ids = self._coord.get_groups().get() self.assertIn(self.group_id, all_group_ids) def test_get_lock_release_broken(self): name = tests.get_random_uuid() memberid2 = tests.get_random_uuid() coord2 = tooz.coordination.get_coordinator(self.url, memberid2) coord2.start() lock1 = self._coord.get_lock(name) lock2 = coord2.get_lock(name) self.assertTrue(lock1.acquire(blocking=False)) self.assertFalse(lock2.acquire(blocking=False)) self.assertTrue(lock2.break_()) self.assertTrue(lock2.acquire(blocking=False)) self.assertFalse(lock1.release()) # Assert lock is not accidentally broken now memberid3 = tests.get_random_uuid() coord3 = tooz.coordination.get_coordinator(self.url, memberid3) coord3.start() lock3 = coord3.get_lock(name) self.assertFalse(lock3.acquire(blocking=False)) def test_create_group_already_exist(self): self._coord.create_group(self.group_id).get() create_group = self._coord.create_group(self.group_id) self.assertRaises(tooz.coordination.GroupAlreadyExist, create_group.get) def test_get_groups(self): groups_ids = [tests.get_random_uuid() for _ in range(0, 5)] for group_id in groups_ids: self._coord.create_group(group_id).get() created_groups = self._coord.get_groups().get() for group_id in groups_ids: self.assertIn(group_id, created_groups) def test_delete_group(self): self._coord.create_group(self.group_id).get() all_group_ids = self._coord.get_groups().get() self.assertIn(self.group_id, all_group_ids) self._coord.delete_group(self.group_id).get() all_group_ids = self._coord.get_groups().get() self.assertNotIn(self.group_id, all_group_ids) join_group = self._coord.join_group(self.group_id) self.assertRaises(tooz.coordination.GroupNotCreated, join_group.get) def test_delete_group_non_existent(self): delete = self._coord.delete_group(self.group_id) self.assertRaises(tooz.coordination.GroupNotCreated, delete.get) def test_delete_group_non_empty(self): self._coord.create_group(self.group_id).get() self._coord.join_group(self.group_id).get() delete = self._coord.delete_group(self.group_id) self.assertRaises(tooz.coordination.GroupNotEmpty, delete.get) self._coord.leave_group(self.group_id).get() self._coord.delete_group(self.group_id).get() def test_join_group(self): self._coord.create_group(self.group_id).get() self._coord.join_group(self.group_id).get() member_list = self._coord.get_members(self.group_id).get() self.assertIn(self.member_id, member_list) def test_join_nonexistent_group(self): join_group = self._coord.join_group(self.group_id) self.assertRaises(tooz.coordination.GroupNotCreated, join_group.get) def test_join_group_create(self): self._coord.join_group_create(self.group_id) member_list = self._coord.get_members(self.group_id).get() self.assertIn(self.member_id, member_list) def test_join_group_with_member_id_already_exists(self): self._coord.create_group(self.group_id).get() self._coord.join_group(self.group_id).get() client = tooz.coordination.get_coordinator(self.url, self.member_id) client.start() join_group = client.join_group(self.group_id) self.assertRaises(tooz.coordination.MemberAlreadyExist, join_group.get) def test_leave_group(self): self._coord.create_group(self.group_id).get() all_group_ids = self._coord.get_groups().get() self.assertIn(self.group_id, all_group_ids) self._coord.join_group(self.group_id).get() member_list = self._coord.get_members(self.group_id).get() self.assertIn(self.member_id, member_list) member_ids = self._coord.get_members(self.group_id).get() self.assertIn(self.member_id, member_ids) self._coord.leave_group(self.group_id).get() new_member_objects = self._coord.get_members(self.group_id).get() new_member_list = [member.member_id for member in new_member_objects] self.assertNotIn(self.member_id, new_member_list) def test_leave_nonexistent_group(self): all_group_ids = self._coord.get_groups().get() self.assertNotIn(self.group_id, all_group_ids) leave_group = self._coord.leave_group(self.group_id) # Drivers raise one of those depending on their capability self.assertRaisesAny([tooz.coordination.MemberNotJoined, tooz.coordination.GroupNotCreated], leave_group.get) def test_leave_group_not_joined_by_member(self): self._coord.create_group(self.group_id).get() all_group_ids = self._coord.get_groups().get() self.assertIn(self.group_id, all_group_ids) leave_group = self._coord.leave_group(self.group_id) self.assertRaises(tooz.coordination.MemberNotJoined, leave_group.get) def test_get_lock_twice_locked_one_released_two(self): name = tests.get_random_uuid() lock1 = self._coord.get_lock(name) lock2 = self._coord.get_lock(name) self.assertTrue(lock1.acquire()) self.assertFalse(lock2.acquire(blocking=False)) self.assertFalse(lock2.release()) self.assertTrue(lock1.release()) self.assertFalse(lock2.release()) def test_get_members(self): group_id_test2 = tests.get_random_uuid() member_id_test2 = tests.get_random_uuid() client2 = tooz.coordination.get_coordinator(self.url, member_id_test2) client2.start() self._coord.create_group(group_id_test2).get() self._coord.join_group(group_id_test2).get() client2.join_group(group_id_test2).get() members_ids = self._coord.get_members(group_id_test2).get() self.assertEqual({self.member_id, member_id_test2}, members_ids) def test_get_member_capabilities(self): self._coord.create_group(self.group_id).get() self._coord.join_group(self.group_id, b"test_capabilities") capa = self._coord.get_member_capabilities(self.group_id, self.member_id).get() self.assertEqual(capa, b"test_capabilities") def test_get_member_capabilities_complex(self): self._coord.create_group(self.group_id).get() caps = { 'type': 'warrior', 'abilities': ['fight', 'flight', 'double-hit-damage'], } self._coord.join_group(self.group_id, caps).get() capa = self._coord.get_member_capabilities(self.group_id, self.member_id).get() self.assertEqual(capa, caps) self.assertEqual(capa['type'], caps['type']) def test_get_member_capabilities_nonexistent_group(self): capa = self._coord.get_member_capabilities(self.group_id, self.member_id) # Drivers raise one of those depending on their capability self.assertRaisesAny([tooz.coordination.MemberNotJoined, tooz.coordination.GroupNotCreated], capa.get) def test_get_member_capabilities_nonjoined_member(self): self._coord.create_group(self.group_id).get() capa = self._coord.get_member_capabilities(self.group_id, self.member_id) self.assertRaises(tooz.coordination.MemberNotJoined, capa.get) def test_get_member_info(self): self._coord.create_group(self.group_id).get() self._coord.join_group(self.group_id, b"test_capabilities").get() member_info = self._coord.get_member_info(self.group_id, self.member_id).get() self.assertEqual(member_info['capabilities'], b"test_capabilities") def test_get_member_info_complex(self): self._coord.create_group(self.group_id).get() caps = { 'type': 'warrior', 'abilities': ['fight', 'flight', 'double-hit-damage'], } member_info = {'capabilities': 'caps', 'created_at': '0', 'updated_at': '0'} self._coord.join_group(self.group_id, caps).get() member_info = self._coord.get_member_info(self.group_id, self.member_id).get() self.assertEqual(member_info['capabilities'], caps) def test_get_member_info_nonexistent_group(self): member_info = self._coord.get_member_info(self.group_id, self.member_id) # Drivers raise one of those depending on their capability self.assertRaisesAny([tooz.coordination.MemberNotJoined, tooz.coordination.GroupNotCreated], member_info.get) def test_get_member_info_nonjoined_member(self): self._coord.create_group(self.group_id).get() member_id = tests.get_random_uuid() member_info = self._coord.get_member_info(self.group_id, member_id) self.assertRaises(tooz.coordination.MemberNotJoined, member_info.get) def test_update_capabilities(self): self._coord.create_group(self.group_id).get() self._coord.join_group(self.group_id, b"test_capabilities1").get() capa = self._coord.get_member_capabilities(self.group_id, self.member_id).get() self.assertEqual(capa, b"test_capabilities1") self._coord.update_capabilities(self.group_id, b"test_capabilities2").get() capa2 = self._coord.get_member_capabilities(self.group_id, self.member_id).get() self.assertEqual(capa2, b"test_capabilities2") def test_update_capabilities_with_group_id_nonexistent(self): update_cap = self._coord.update_capabilities(self.group_id, b'test_capabilities') # Drivers raise one of those depending on their capability self.assertRaisesAny([tooz.coordination.MemberNotJoined, tooz.coordination.GroupNotCreated], update_cap.get) def test_heartbeat(self): if not self._coord.requires_beating: raise testcase.TestSkipped("Test not applicable (heartbeating" " not required)") self._coord.heartbeat() def test_heartbeat_loop(self): if not self._coord.requires_beating: raise testcase.TestSkipped("Test not applicable (heartbeating" " not required)") heart = self._coord.heart self.assertFalse(heart.is_alive()) heart.start() # This will timeout if nothing ever is done... try: while not heart.beats: time.sleep(1) finally: heart.stop() heart.wait() def test_disconnect_leave_group(self): member_id_test2 = tests.get_random_uuid() client2 = tooz.coordination.get_coordinator(self.url, member_id_test2) client2.start() self._coord.create_group(self.group_id).get() self._coord.join_group(self.group_id).get() client2.join_group(self.group_id).get() members_ids = self._coord.get_members(self.group_id).get() self.assertIn(self.member_id, members_ids) self.assertIn(member_id_test2, members_ids) client2.stop() members_ids = self._coord.get_members(self.group_id).get() self.assertIn(self.member_id, members_ids) self.assertNotIn(member_id_test2, members_ids) def test_timeout(self): if (tooz.coordination.Characteristics.NON_TIMEOUT_BASED in self._coord.CHARACTERISTICS): self.skipTest("This driver is not based on timeout") self._coord.stop() if "?" in self.url: sep = "&" else: sep = "?" url = self.url + sep + "timeout=5" self._coord = tooz.coordination.get_coordinator(url, self.member_id) self._coord.start() member_id_test2 = tests.get_random_uuid() client2 = tooz.coordination.get_coordinator(url, member_id_test2) client2.start() self._coord.create_group(self.group_id).get() self._coord.join_group(self.group_id).get() client2.join_group(self.group_id).get() members_ids = self._coord.get_members(self.group_id).get() self.assertIn(self.member_id, members_ids) self.assertIn(member_id_test2, members_ids) # Watch the group, we want to be sure that when client2 is kicked out # we get an event. self._coord.watch_leave_group(self.group_id, self._set_event) # Run watchers to be sure we initialize the member cache and we *know* # client2 is a member now self._coord.run_watchers() time.sleep(3) self._coord.heartbeat() time.sleep(3) # Now client2 has timed out! members_ids = self._coord.get_members(self.group_id).get() while True: if self._coord.run_watchers(): break self.assertIn(self.member_id, members_ids) self.assertNotIn(member_id_test2, members_ids) # Check that the event has been triggered self.assertEqual(1, len(self.events)) event = self.events[0] self.assertIsInstance(event, tooz.coordination.MemberLeftGroup) self.assertEqual(member_id_test2, event.member_id) self.assertEqual(self.group_id, event.group_id) def _set_event(self, event): if not hasattr(self, "events"): self.events = [event] else: self.events.append(event) return 42 def test_watch_group_join(self): member_id_test2 = tests.get_random_uuid() client2 = tooz.coordination.get_coordinator(self.url, member_id_test2) client2.start() self._coord.create_group(self.group_id).get() # Watch the group self._coord.watch_join_group(self.group_id, self._set_event) # Join the group client2.join_group(self.group_id).get() members_ids = self._coord.get_members(self.group_id).get() self.assertIn(member_id_test2, members_ids) while True: if self._coord.run_watchers(): break self.assertEqual(1, len(self.events)) event = self.events[0] self.assertIsInstance(event, tooz.coordination.MemberJoinedGroup) self.assertEqual(member_id_test2, event.member_id) self.assertEqual(self.group_id, event.group_id) # Stop watching self._coord.unwatch_join_group(self.group_id, self._set_event) self.events = [] # Leave and rejoin group client2.leave_group(self.group_id).get() client2.join_group(self.group_id).get() self._coord.run_watchers() self.assertEqual([], self.events) def test_watch_leave_group(self): member_id_test2 = tests.get_random_uuid() client2 = tooz.coordination.get_coordinator(self.url, member_id_test2) client2.start() self._coord.create_group(self.group_id).get() # Watch the group: this can leads to race conditions in certain # driver that are not able to see all events, so we join, wait for # the join to be seen, and then we leave, and wait for the leave to # be seen. self._coord.watch_join_group(self.group_id, lambda children: True) self._coord.watch_leave_group(self.group_id, self._set_event) # Join and leave the group client2.join_group(self.group_id).get() # Consumes join event while True: if self._coord.run_watchers(): break client2.leave_group(self.group_id).get() # Consumes leave event while True: if self._coord.run_watchers(): break self.assertEqual(1, len(self.events)) event = self.events[0] self.assertIsInstance(event, tooz.coordination.MemberLeftGroup) self.assertEqual(member_id_test2, event.member_id) self.assertEqual(self.group_id, event.group_id) # Stop watching self._coord.unwatch_leave_group(self.group_id, self._set_event) self.events = [] # Rejoin and releave group client2.join_group(self.group_id).get() client2.leave_group(self.group_id).get() self._coord.run_watchers() self.assertEqual([], self.events) def test_watch_join_group_disappear(self): if not hasattr(self._coord, '_destroy_group'): self.skipTest("This test only works with coordinators" " that have the ability to destroy groups.") self._coord.create_group(self.group_id).get() self._coord.watch_join_group(self.group_id, self._set_event) self._coord.watch_leave_group(self.group_id, self._set_event) member_id_test2 = tests.get_random_uuid() client2 = tooz.coordination.get_coordinator(self.url, member_id_test2) client2.start() client2.join_group(self.group_id).get() while True: if self._coord.run_watchers(): break self.assertEqual(1, len(self.events)) event = self.events[0] self.assertIsInstance(event, tooz.coordination.MemberJoinedGroup) self.events = [] # Force the group to disappear... self._coord._destroy_group(self.group_id) while True: if self._coord.run_watchers(): break self.assertEqual(1, len(self.events)) event = self.events[0] self.assertIsInstance(event, tooz.coordination.MemberLeftGroup) def test_watch_join_group_non_existent(self): self.assertRaises(tooz.coordination.GroupNotCreated, self._coord.watch_join_group, self.group_id, lambda: None) self.assertEqual(0, len(self._coord._hooks_join_group[self.group_id])) def test_watch_join_group_booted_out(self): self._coord.create_group(self.group_id).get() self._coord.join_group(self.group_id).get() self._coord.watch_join_group(self.group_id, self._set_event) self._coord.watch_leave_group(self.group_id, self._set_event) member_id_test2 = tests.get_random_uuid() client2 = tooz.coordination.get_coordinator(self.url, member_id_test2) client2.start() client2.join_group(self.group_id).get() while True: if self._coord.run_watchers(): break client3 = tooz.coordination.get_coordinator(self.url, self.member_id) client3.start() client3.leave_group(self.group_id).get() # Only works for clients that have access to the groups they are part # of, to ensure that after we got booted out by client3 that this # client now no longer believes its part of the group. if (hasattr(self._coord, '_joined_groups') and (self._coord.run_watchers == tooz.coordination.CoordinationDriverCachedRunWatchers.run_watchers)): # noqa self.assertIn(self.group_id, self._coord._joined_groups) self._coord.run_watchers() self.assertNotIn(self.group_id, self._coord._joined_groups) def test_watch_leave_group_non_existent(self): self.assertRaises(tooz.coordination.GroupNotCreated, self._coord.watch_leave_group, self.group_id, lambda: None) self.assertEqual(0, len(self._coord._hooks_leave_group[self.group_id])) def test_run_for_election(self): self._coord.create_group(self.group_id).get() self._coord.watch_elected_as_leader(self.group_id, self._set_event) self._coord.run_watchers() self.assertEqual(1, len(self.events)) event = self.events[0] self.assertIsInstance(event, tooz.coordination.LeaderElected) self.assertEqual(self.member_id, event.member_id) self.assertEqual(self.group_id, event.group_id) def test_run_for_election_multiple_clients(self): self._coord.create_group(self.group_id).get() self._coord.watch_elected_as_leader(self.group_id, self._set_event) self._coord.run_watchers() member_id_test2 = tests.get_random_uuid() client2 = tooz.coordination.get_coordinator(self.url, member_id_test2) client2.start() client2.watch_elected_as_leader(self.group_id, self._set_event) client2.run_watchers() self.assertEqual(1, len(self.events)) event = self.events[0] self.assertIsInstance(event, tooz.coordination.LeaderElected) self.assertEqual(self.member_id, event.member_id) self.assertEqual(self.group_id, event.group_id) self.assertEqual(self._coord.get_leader(self.group_id).get(), self.member_id) self.events = [] self._coord.stop() client2.run_watchers() self.assertEqual(1, len(self.events)) event = self.events[0] self.assertIsInstance(event, tooz.coordination.LeaderElected) self.assertEqual(member_id_test2, event.member_id) self.assertEqual(self.group_id, event.group_id) self.assertEqual(client2.get_leader(self.group_id).get(), member_id_test2) # Restart the coord because tearDown stops it self._coord.start() def test_get_leader(self): self._coord.create_group(self.group_id).get() leader = self._coord.get_leader(self.group_id).get() self.assertIsNone(leader) self._coord.join_group(self.group_id).get() leader = self._coord.get_leader(self.group_id).get() self.assertIsNone(leader) # Let's get elected self._coord.watch_elected_as_leader(self.group_id, self._set_event) self._coord.run_watchers() leader = self._coord.get_leader(self.group_id).get() self.assertEqual(leader, self.member_id) def test_run_for_election_multiple_clients_stand_down(self): self._coord.create_group(self.group_id).get() self._coord.watch_elected_as_leader(self.group_id, self._set_event) self._coord.run_watchers() member_id_test2 = tests.get_random_uuid() client2 = tooz.coordination.get_coordinator(self.url, member_id_test2) client2.start() client2.watch_elected_as_leader(self.group_id, self._set_event) client2.run_watchers() self.assertEqual(1, len(self.events)) event = self.events[0] self.assertIsInstance(event, tooz.coordination.LeaderElected) self.assertEqual(self.member_id, event.member_id) self.assertEqual(self.group_id, event.group_id) self.events = [] self._coord.stand_down_group_leader(self.group_id) client2.run_watchers() self.assertEqual(1, len(self.events)) event = self.events[0] self.assertIsInstance(event, tooz.coordination.LeaderElected) self.assertEqual(member_id_test2, event.member_id) self.assertEqual(self.group_id, event.group_id) self.events = [] client2.stand_down_group_leader(self.group_id) self._coord.run_watchers() self.assertEqual(1, len(self.events)) event = self.events[0] self.assertIsInstance(event, tooz.coordination.LeaderElected) self.assertEqual(self.member_id, event.member_id) self.assertEqual(self.group_id, event.group_id) def test_unwatch_elected_as_leader(self): # Create a group and add a elected_as_leader callback self._coord.create_group(self.group_id).get() self._coord.watch_elected_as_leader(self.group_id, self._set_event) # Ensure exactly one leader election hook exists self.assertEqual(1, len(self._coord._hooks_elected_leader[self.group_id])) # Unwatch, and ensure no leader election hooks exist self._coord.unwatch_elected_as_leader(self.group_id, self._set_event) self.assertEqual(0, len(self._coord._hooks_elected_leader)) def test_unwatch_elected_as_leader_callback_not_found(self): self._coord.create_group(self.group_id).get() self.assertRaises(tooz.coordination.WatchCallbackNotFound, self._coord.unwatch_elected_as_leader, self.group_id, lambda x: None) def test_unwatch_join_group_callback_not_found(self): self._coord.create_group(self.group_id).get() self.assertRaises(tooz.coordination.WatchCallbackNotFound, self._coord.unwatch_join_group, self.group_id, lambda x: None) def test_unwatch_leave_group(self): # Create a group and add a leave_group callback self._coord.create_group(self.group_id).get() self.assertEqual(0, len(self._coord._hooks_leave_group)) self._coord.watch_leave_group(self.group_id, self._set_event) # Ensure exactly one leave group hook exists self.assertEqual(1, len(self._coord._hooks_leave_group[self.group_id])) # Unwatch, and ensure no leave group hooks exist self._coord.unwatch_leave_group(self.group_id, self._set_event) self.assertEqual(0, len(self._coord._hooks_leave_group)) def test_unwatch_leave_group_callback_not_found(self): self._coord.create_group(self.group_id).get() self.assertRaises(tooz.coordination.WatchCallbackNotFound, self._coord.unwatch_leave_group, self.group_id, lambda x: None) def test_get_lock(self): lock = self._coord.get_lock(tests.get_random_uuid()) self.assertTrue(lock.acquire()) self.assertTrue(lock.release()) with lock: pass def test_heartbeat_lock_not_acquired(self): lock = self._coord.get_lock(tests.get_random_uuid()) # Not all locks need heartbeat if hasattr(lock, "heartbeat"): self.assertFalse(lock.heartbeat()) def test_get_shared_lock(self): lock = self._coord.get_lock(tests.get_random_uuid()) self.assertTrue(lock.acquire(shared=True)) self.assertTrue(lock.release()) with lock(shared=True): pass def test_get_shared_lock_locking_same_lock_twice(self): lock = self._coord.get_lock(tests.get_random_uuid()) self.assertTrue(lock.acquire(shared=True)) self.assertTrue(lock.acquire(shared=True)) self.assertTrue(lock.release()) self.assertTrue(lock.release()) self.assertFalse(lock.release()) with lock(shared=True): pass def test_get_shared_lock_locking_two_lock(self): name = tests.get_random_uuid() lock1 = self._coord.get_lock(name) coord = tooz.coordination.get_coordinator( self.url, tests.get_random_uuid()) coord.start() lock2 = coord.get_lock(name) self.assertTrue(lock1.acquire(shared=True)) self.assertTrue(lock2.acquire(shared=True)) self.assertTrue(lock1.release()) self.assertTrue(lock2.release()) def test_get_lock_locking_shared_and_exclusive(self): name = tests.get_random_uuid() lock1 = self._coord.get_lock(name) coord = tooz.coordination.get_coordinator( self.url, tests.get_random_uuid()) coord.start() lock2 = coord.get_lock(name) self.assertTrue(lock1.acquire(shared=True)) self.assertFalse(lock2.acquire(blocking=False)) self.assertTrue(lock1.release()) self.assertFalse(lock2.release()) def test_get_lock_locking_exclusive_and_shared(self): name = tests.get_random_uuid() lock1 = self._coord.get_lock(name) coord = tooz.coordination.get_coordinator( self.url, tests.get_random_uuid()) coord.start() lock2 = coord.get_lock(name) self.assertTrue(lock1.acquire()) self.assertFalse(lock2.acquire(shared=True, blocking=False)) self.assertTrue(lock1.release()) self.assertFalse(lock2.release()) def test_get_lock_concurrency_locking_same_lock(self): lock = self._coord.get_lock(tests.get_random_uuid()) graceful_ending = threading.Event() def thread(): self.assertTrue(lock.acquire()) self.assertTrue(lock.release()) graceful_ending.set() t = threading.Thread(target=thread) t.daemon = True with lock: t.start() # Ensure the thread try to get the lock time.sleep(.1) t.join() graceful_ending.wait(.2) self.assertTrue(graceful_ending.is_set()) def _do_test_get_lock_concurrency_locking_two_lock(self, executor, use_same_coord): name = tests.get_random_uuid() lock1 = self._coord.get_lock(name) with lock1: with executor(max_workers=1) as e: coord = self._coord if use_same_coord else None f = e.submit(try_to_lock_job, name, coord, self.url, tests.get_random_uuid()) self.assertFalse(f.result()) def _do_test_get_lock_serial_locking_two_lock(self, executor, use_same_coord): name = tests.get_random_uuid() lock1 = self._coord.get_lock(name) lock1.acquire() lock1.release() with executor(max_workers=1) as e: coord = self._coord if use_same_coord else None f = e.submit(try_to_lock_job, name, coord, self.url, tests.get_random_uuid()) self.assertTrue(f.result()) def test_get_lock_concurrency_locking_two_lock_process(self): # NOTE(jd) Using gRPC and forking is not supported so this test might # very likely hang forever or crash. See # https://github.com/grpc/grpc/issues/10140#issuecomment-297548714 for # more info. if self.url.startswith("etcd3://"): self.skipTest("Unable to use etcd3 with fork()") self._do_test_get_lock_concurrency_locking_two_lock( futures.ProcessPoolExecutor, False) def test_get_lock_serial_locking_two_lock_process(self): # NOTE(jd) Using gRPC and forking is not supported so this test might # very likely hang forever or crash. See # https://github.com/grpc/grpc/issues/10140#issuecomment-297548714 for # more info. if self.url.startswith("etcd3://"): self.skipTest("Unable to use etcd3 with fork()") self._do_test_get_lock_serial_locking_two_lock( futures.ProcessPoolExecutor, False) def test_get_lock_concurrency_locking_two_lock_thread1(self): self._do_test_get_lock_concurrency_locking_two_lock( futures.ThreadPoolExecutor, False) def test_get_lock_concurrency_locking_two_lock_thread2(self): self._do_test_get_lock_concurrency_locking_two_lock( futures.ThreadPoolExecutor, True) def test_get_lock_concurrency_locking2(self): # NOTE(sileht): some database based lock can have only # one lock per connection, this test ensures acquiring a # second lock doesn't release the first one. lock1 = self._coord.get_lock(tests.get_random_uuid()) lock2 = self._coord.get_lock(tests.get_random_uuid()) graceful_ending = threading.Event() thread_locked = threading.Event() def thread(): with lock2: try: self.assertFalse(lock1.acquire(blocking=False)) except tooz.NotImplemented: pass thread_locked.set() graceful_ending.set() t = threading.Thread(target=thread) t.daemon = True with lock1: t.start() thread_locked.wait() self.assertTrue(thread_locked.is_set()) t.join() graceful_ending.wait() self.assertTrue(graceful_ending.is_set()) def test_get_lock_twice_locked_twice(self): name = tests.get_random_uuid() lock1 = self._coord.get_lock(name) lock2 = self._coord.get_lock(name) with lock1: self.assertFalse(lock2.acquire(blocking=False)) def test_get_lock_context_fails(self): name = tests.get_random_uuid() lock1 = self._coord.get_lock(name) lock2 = self._coord.get_lock(name) with mock.patch.object(lock2, 'acquire', return_value=False): with lock1: self.assertRaises( tooz.coordination.LockAcquireFailed, lock2.__enter__) def test_get_lock_context_check_value(self): name = tests.get_random_uuid() lock = self._coord.get_lock(name) with lock as returned_lock: self.assertEqual(lock, returned_lock) def test_lock_context_manager_acquire_no_argument(self): name = tests.get_random_uuid() lock1 = self._coord.get_lock(name) lock2 = self._coord.get_lock(name) with lock1(): self.assertFalse(lock2.acquire(blocking=False)) def test_lock_context_manager_acquire_argument_return_value(self): name = tests.get_random_uuid() blocking_value = 10.12 lock = self._coord.get_lock(name) with lock(blocking_value) as returned_lock: self.assertEqual(lock, returned_lock) def test_lock_context_manager_acquire_argument_release_within(self): name = tests.get_random_uuid() blocking_value = 10.12 lock = self._coord.get_lock(name) with lock(blocking_value) as returned_lock: self.assertTrue(returned_lock.release()) def test_lock_context_manager_acquire_argument(self): name = tests.get_random_uuid() blocking_value = 10.12 lock = self._coord.get_lock(name) with mock.patch.object(lock, 'acquire', wraps=True, autospec=True) as \ mock_acquire: with lock(blocking_value): mock_acquire.assert_called_once_with(blocking_value) def test_lock_context_manager_acquire_argument_timeout(self): name = tests.get_random_uuid() lock1 = self._coord.get_lock(name) lock2 = self._coord.get_lock(name) with lock1: try: with lock2(False): self.fail('Lock acquire should have failed') except tooz.coordination.LockAcquireFailed: pass def test_get_lock_locked_twice(self): name = tests.get_random_uuid() lock = self._coord.get_lock(name) with lock: self.assertFalse(lock.acquire(blocking=False)) def test_get_multiple_locks_with_same_coord(self): name = tests.get_random_uuid() lock1 = self._coord.get_lock(name) lock2 = self._coord.get_lock(name) self.assertTrue(lock1.acquire()) self.assertFalse(lock2.acquire(blocking=False)) self.assertFalse(self._coord.get_lock(name).acquire(blocking=False)) self.assertTrue(lock1.release()) def test_ensure_acquire_release_return(self): name = tests.get_random_uuid() lock1 = self._coord.get_lock(name) self.assertTrue(lock1.acquire()) self.assertTrue(lock1.release()) self.assertFalse(lock1.release()) def test_get_lock_multiple_coords(self): member_id2 = tests.get_random_uuid() client2 = tooz.coordination.get_coordinator(self.url, member_id2) client2.start() lock_name = tests.get_random_uuid() lock = self._coord.get_lock(lock_name) self.assertTrue(lock.acquire()) lock2 = client2.get_lock(lock_name) self.assertFalse(lock2.acquire(blocking=False)) self.assertTrue(lock.release()) self.assertTrue(lock2.acquire(blocking=True)) self.assertTrue(lock2.release()) def test_get_started_status(self): self.assertTrue(self._coord.is_started) self._coord.stop() self.assertFalse(self._coord.is_started) self._coord.start() def do_test_name_property(self): name = tests.get_random_uuid() lock = self._coord.get_lock(name) self.assertEqual(name, lock.name) def test_acquire_twice_no_deadlock_releasing(self): name = tests.get_random_uuid() lock = self._coord.get_lock(name) self.assertTrue(lock.acquire(blocking=False)) self.assertFalse(lock.acquire(blocking=False)) self.assertTrue(lock.release()) class TestHook(testcase.TestCase): def setUp(self): super(TestHook, self).setUp() self.hooks = tooz.coordination.Hooks() self.triggered = False def _trigger(self): self.triggered = True def test_register_hook(self): self.assertEqual(self.hooks.run(), []) self.assertFalse(self.triggered) self.hooks.append(self._trigger) self.assertEqual(self.hooks.run(), [None]) self.assertTrue(self.triggered) def test_unregister_hook(self): self.hooks.append(self._trigger) self.assertEqual(self.hooks.run(), [None]) self.assertTrue(self.triggered) self.triggered = False self.hooks.remove(self._trigger) self.assertEqual(self.hooks.run(), []) self.assertFalse(self.triggered) tooz-2.0.0/tooz/tests/test_postgresql.py0000664000175000017500000001004513616633007020504 0ustar zuulzuul00000000000000# -*- coding: utf-8 -*- # Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. try: # Added in python 3.3+ from unittest import mock except ImportError: import mock from oslo_utils import encodeutils import testtools from testtools import testcase import tooz from tooz import coordination from tooz import tests # Handle the case gracefully where the driver is not installed. try: import psycopg2 PGSQL_AVAILABLE = True except ImportError: PGSQL_AVAILABLE = False @testtools.skipUnless(PGSQL_AVAILABLE, 'psycopg2 is not available') class TestPostgreSQLFailures(testcase.TestCase): # Not actually used (but required none the less), since we mock out # the connect() method... FAKE_URL = "postgresql://localhost:1" def _create_coordinator(self): def _safe_stop(coord): try: coord.stop() except tooz.ToozError as e: # TODO(harlowja): make this better, so that we don't have to # do string checking... message = encodeutils.exception_to_unicode(e) if (message != 'Can not stop a driver which has not' ' been started'): raise coord = coordination.get_coordinator(self.FAKE_URL, tests.get_random_uuid()) self.addCleanup(_safe_stop, coord) return coord @mock.patch("tooz.drivers.pgsql.psycopg2.connect") def test_connect_failure(self, psycopg2_connector): psycopg2_connector.side_effect = psycopg2.Error("Broken") c = self._create_coordinator() self.assertRaises(coordination.ToozConnectionError, c.start) @mock.patch("tooz.drivers.pgsql.psycopg2.connect") def test_connect_failure_operational(self, psycopg2_connector): psycopg2_connector.side_effect = psycopg2.OperationalError("Broken") c = self._create_coordinator() self.assertRaises(coordination.ToozConnectionError, c.start) @mock.patch("tooz.drivers.pgsql.psycopg2.connect") def test_failure_acquire_lock(self, psycopg2_connector): execute_mock = mock.MagicMock() execute_mock.execute.side_effect = psycopg2.OperationalError("Broken") cursor_mock = mock.MagicMock() cursor_mock.__enter__ = mock.MagicMock(return_value=execute_mock) cursor_mock.__exit__ = mock.MagicMock(return_value=False) conn_mock = mock.MagicMock() conn_mock.cursor.return_value = cursor_mock psycopg2_connector.return_value = conn_mock c = self._create_coordinator() c.start() test_lock = c.get_lock(b'test-lock') self.assertRaises(tooz.ToozError, test_lock.acquire) @mock.patch("tooz.drivers.pgsql.psycopg2.connect") def test_failure_release_lock(self, psycopg2_connector): execute_mock = mock.MagicMock() execute_mock.execute.side_effect = [ True, psycopg2.OperationalError("Broken"), ] cursor_mock = mock.MagicMock() cursor_mock.__enter__ = mock.MagicMock(return_value=execute_mock) cursor_mock.__exit__ = mock.MagicMock(return_value=False) conn_mock = mock.MagicMock() conn_mock.cursor.return_value = cursor_mock psycopg2_connector.return_value = conn_mock c = self._create_coordinator() c.start() test_lock = c.get_lock(b'test-lock') self.assertTrue(test_lock.acquire()) self.assertRaises(tooz.ToozError, test_lock.release) tooz-2.0.0/tooz/tests/test_hashring.py0000664000175000017500000002274113616633007020112 0ustar zuulzuul00000000000000# Copyright 2013 Hewlett-Packard Development Company, L.P. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import hashlib import mock from testtools import matchers from testtools import testcase from tooz import hashring class HashRingTestCase(testcase.TestCase): # NOTE(deva): the mapping used in these tests is as follows: # if nodes = [foo, bar]: # fake -> foo, bar # if nodes = [foo, bar, baz]: # fake -> foo, bar, baz # fake-again -> bar, baz, foo @mock.patch.object(hashlib, 'md5', autospec=True) def test_hash2int_returns_int(self, mock_md5): r1 = 32 * 'a' r2 = 32 * 'b' # 2**PARTITION_EXPONENT calls to md5.update per node # PARTITION_EXPONENT is currently always 5, so 32 calls each here mock_md5.return_value.hexdigest.side_effect = [r1] * 32 + [r2] * 32 nodes = ['foo', 'bar'] ring = hashring.HashRing(nodes) self.assertIn(int(r1, 16), ring._ring) self.assertIn(int(r2, 16), ring._ring) def test_create_ring(self): nodes = {'foo', 'bar'} ring = hashring.HashRing(nodes) self.assertEqual(nodes, set(ring.nodes.keys())) self.assertEqual(2 ** 5 * 2, len(ring)) def test_add_node(self): nodes = {'foo', 'bar'} ring = hashring.HashRing(nodes) self.assertEqual(nodes, set(ring.nodes.keys())) self.assertEqual(2 ** 5 * len(nodes), len(ring)) nodes.add('baz') ring.add_node('baz') self.assertEqual(nodes, set(ring.nodes.keys())) self.assertEqual(2 ** 5 * len(nodes), len(ring)) def test_add_node_bytes(self): nodes = {'foo', 'bar'} ring = hashring.HashRing(nodes) self.assertEqual(nodes, set(ring.nodes.keys())) self.assertEqual(2 ** 5 * len(nodes), len(ring)) nodes.add(b'Z\xe2\xfa\x90\x17EC\xac\xae\x88\xa7[\xa1}:E') ring.add_node(b'Z\xe2\xfa\x90\x17EC\xac\xae\x88\xa7[\xa1}:E') self.assertEqual(nodes, set(ring.nodes.keys())) self.assertEqual(2 ** 5 * len(nodes), len(ring)) def test_add_node_unicode(self): nodes = {'foo', 'bar'} ring = hashring.HashRing(nodes) self.assertEqual(nodes, set(ring.nodes.keys())) self.assertEqual(2 ** 5 * len(nodes), len(ring)) nodes.add(u'\u0634\u0628\u06a9\u0647') ring.add_node(u'\u0634\u0628\u06a9\u0647') self.assertEqual(nodes, set(ring.nodes.keys())) self.assertEqual(2 ** 5 * len(nodes), len(ring)) def test_add_node_weight(self): nodes = {'foo', 'bar'} ring = hashring.HashRing(nodes) self.assertEqual(nodes, set(ring.nodes.keys())) self.assertEqual(2 ** 5 * len(nodes), len(ring)) nodes.add('baz') ring.add_node('baz', weight=10) self.assertEqual(nodes, set(ring.nodes.keys())) self.assertEqual(2 ** 5 * 12, len(ring)) def test_add_nodes_weight(self): nodes = {'foo', 'bar'} ring = hashring.HashRing(nodes) self.assertEqual(nodes, set(ring.nodes.keys())) self.assertEqual(2 ** 5 * len(nodes), len(ring)) nodes.add('baz') nodes.add('baz2') ring.add_nodes(set(['baz', 'baz2']), weight=10) self.assertEqual(nodes, set(ring.nodes.keys())) self.assertEqual(2 ** 5 * 22, len(ring)) def test_remove_node(self): nodes = {'foo', 'bar'} ring = hashring.HashRing(nodes) self.assertEqual(nodes, set(ring.nodes.keys())) self.assertEqual(2 ** 5 * len(nodes), len(ring)) nodes.discard('bar') ring.remove_node('bar') self.assertEqual(nodes, set(ring.nodes.keys())) self.assertEqual(2 ** 5 * len(nodes), len(ring)) def test_remove_node_bytes(self): nodes = {'foo', b'Z\xe2\xfa\x90\x17EC\xac\xae\x88\xa7[\xa1}:E'} ring = hashring.HashRing(nodes) self.assertEqual(nodes, set(ring.nodes.keys())) self.assertEqual(2 ** 5 * len(nodes), len(ring)) nodes.discard(b'Z\xe2\xfa\x90\x17EC\xac\xae\x88\xa7[\xa1}:E') ring.remove_node(b'Z\xe2\xfa\x90\x17EC\xac\xae\x88\xa7[\xa1}:E') self.assertEqual(nodes, set(ring.nodes.keys())) self.assertEqual(2 ** 5 * len(nodes), len(ring)) def test_remove_node_unknown(self): nodes = ['foo', 'bar'] ring = hashring.HashRing(nodes) self.assertRaises( hashring.UnknownNode, ring.remove_node, 'biz') def test_add_then_removenode(self): nodes = {'foo', 'bar'} ring = hashring.HashRing(nodes) self.assertEqual(nodes, set(ring.nodes.keys())) self.assertEqual(2 ** 5 * len(nodes), len(ring)) nodes.add('baz') ring.add_node('baz') self.assertEqual(nodes, set(ring.nodes.keys())) self.assertEqual(2 ** 5 * len(nodes), len(ring)) nodes.discard('bar') ring.remove_node('bar') self.assertEqual(nodes, set(ring.nodes.keys())) self.assertEqual(2 ** 5 * len(nodes), len(ring)) def test_distribution_one_replica(self): nodes = ['foo', 'bar', 'baz'] ring = hashring.HashRing(nodes) fake_1_nodes = ring.get_nodes(b'fake') fake_2_nodes = ring.get_nodes(b'fake-again') # We should have one nodes for each thing self.assertEqual(1, len(fake_1_nodes)) self.assertEqual(1, len(fake_2_nodes)) # And they must not be the same answers even on this simple data. self.assertNotEqual(fake_1_nodes, fake_2_nodes) def test_distribution_more_replica(self): nodes = ['foo', 'bar', 'baz'] ring = hashring.HashRing(nodes) fake_1_nodes = ring.get_nodes(b'fake', replicas=2) fake_2_nodes = ring.get_nodes(b'fake-again', replicas=2) # We should have one nodes for each thing self.assertEqual(2, len(fake_1_nodes)) self.assertEqual(2, len(fake_2_nodes)) fake_1_nodes = ring.get_nodes(b'fake', replicas=3) fake_2_nodes = ring.get_nodes(b'fake-again', replicas=3) # We should have one nodes for each thing self.assertEqual(3, len(fake_1_nodes)) self.assertEqual(3, len(fake_2_nodes)) self.assertEqual(fake_1_nodes, fake_2_nodes) def test_ignore_nodes(self): nodes = ['foo', 'bar', 'baz'] ring = hashring.HashRing(nodes) equals_bar_or_baz = matchers.MatchesAny( matchers.Equals({'bar'}), matchers.Equals({'baz'})) self.assertThat( ring.get_nodes(b'fake', ignore_nodes=['foo']), equals_bar_or_baz) self.assertThat( ring.get_nodes(b'fake', ignore_nodes=['foo', 'bar']), equals_bar_or_baz) self.assertEqual(set(), ring.get_nodes(b'fake', ignore_nodes=nodes)) @staticmethod def _compare_rings(nodes, conductors, ring, new_conductors, new_ring): delta = {} mapping = { 'node': list(ring.get_nodes(node.encode('ascii')))[0] for node in nodes } new_mapping = { 'node': list(new_ring.get_nodes(node.encode('ascii')))[0] for node in nodes } for key, old in mapping.items(): new = new_mapping.get(key, None) if new != old: delta[key] = (old, new) return delta def test_rebalance_stability_join(self): num_services = 10 num_nodes = 10000 # Adding 1 service to a set of N should move 1/(N+1) of all nodes # Eg, for a cluster of 10 nodes, adding one should move 1/11, or 9% # We allow for 1/N to allow for rounding in tests. redistribution_factor = 1.0 / num_services nodes = [str(x) for x in range(num_nodes)] services = [str(x) for x in range(num_services)] new_services = services + ['new'] delta = self._compare_rings( nodes, services, hashring.HashRing(services), new_services, hashring.HashRing(new_services)) self.assertLess(len(delta), num_nodes * redistribution_factor) def test_rebalance_stability_leave(self): num_services = 10 num_nodes = 10000 # Removing 1 service from a set of N should move 1/(N) of all nodes # Eg, for a cluster of 10 nodes, removing one should move 1/10, or 10% # We allow for 1/(N-1) to allow for rounding in tests. redistribution_factor = 1.0 / (num_services - 1) nodes = [str(x) for x in range(num_nodes)] services = [str(x) for x in range(num_services)] new_services = services[:] new_services.pop() delta = self._compare_rings( nodes, services, hashring.HashRing(services), new_services, hashring.HashRing(new_services)) self.assertLess(len(delta), num_nodes * redistribution_factor) def test_ignore_non_existent_node(self): nodes = ['foo', 'bar'] ring = hashring.HashRing(nodes) self.assertEqual({'foo'}, ring.get_nodes(b'fake', ignore_nodes=['baz'])) tooz-2.0.0/tooz/tests/test_mysql.py0000664000175000017500000000370713616633007017455 0ustar zuulzuul00000000000000# -*- coding: utf-8 -*- # Copyright (c) 2015 OpenStack Foundation # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. from oslo_utils import encodeutils from testtools import testcase import tooz from tooz import coordination from tooz import tests class TestMySQLDriver(testcase.TestCase): def _create_coordinator(self, url): def _safe_stop(coord): try: coord.stop() except tooz.ToozError as e: message = encodeutils.exception_to_unicode(e) if (message != 'Can not stop a driver which has not' ' been started'): raise coord = coordination.get_coordinator(url, tests.get_random_uuid()) self.addCleanup(_safe_stop, coord) return coord def test_connect_failure_invalid_hostname_provided(self): c = self._create_coordinator("mysql://invalidhost/test") self.assertRaises(coordination.ToozConnectionError, c.start) def test_connect_failure_invalid_port_provided(self): c = self._create_coordinator("mysql://localhost:54/test") self.assertRaises(coordination.ToozConnectionError, c.start) def test_connect_failure_invalid_hostname_and_port_provided(self): c = self._create_coordinator("mysql://invalidhost:54/test") self.assertRaises(coordination.ToozConnectionError, c.start) tooz-2.0.0/tooz/tests/test_etcd.py0000664000175000017500000000317713616633007017230 0ustar zuulzuul00000000000000# -*- coding: utf-8 -*- # # Copyright 2016 Red Hat, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import mock from testtools import testcase import tooz.coordination class TestEtcd(testcase.TestCase): FAKE_URL = "etcd://mocked-not-really-localhost:2379" FAKE_MEMBER_ID = "mocked-not-really-member" def setUp(self): super(TestEtcd, self).setUp() self._coord = tooz.coordination.get_coordinator(self.FAKE_URL, self.FAKE_MEMBER_ID) def test_multiple_locks_etcd_wait_index(self): lock = self._coord.get_lock('mocked-not-really-random') return_values = [ {'errorCode': {}, 'node': {}, 'index': 10}, {'errorCode': None, 'node': {}, 'index': 10} ] with mock.patch.object(lock.client, 'put', side_effect=return_values): with mock.patch.object(lock.client, 'get') as mocked_get: self.assertTrue(lock.acquire()) mocked_get.assert_called_once() call = str(mocked_get.call_args) self.assertIn("waitIndex=11", call) tooz-2.0.0/tooz/tests/test_memcache.py0000664000175000017500000000656713616633007020061 0ustar zuulzuul00000000000000# -*- coding: utf-8 -*- # Copyright (c) 2015 OpenStack Foundation # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import socket try: from unittest import mock except ImportError: import mock from testtools import testcase from tooz import coordination from tooz import tests class TestMemcacheDriverFailures(testcase.TestCase): FAKE_URL = "memcached://mocked-not-really-localhost" @mock.patch('pymemcache.client.PooledClient') def test_client_failure_start(self, mock_client_cls): mock_client_cls.side_effect = socket.timeout('timed-out') member_id = tests.get_random_uuid() coord = coordination.get_coordinator(self.FAKE_URL, member_id) self.assertRaises(coordination.ToozConnectionError, coord.start) @mock.patch('pymemcache.client.PooledClient') def test_client_failure_join(self, mock_client_cls): mock_client = mock.MagicMock() mock_client_cls.return_value = mock_client member_id = tests.get_random_uuid() coord = coordination.get_coordinator(self.FAKE_URL, member_id) coord.start() mock_client.gets.side_effect = socket.timeout('timed-out') fut = coord.join_group(tests.get_random_uuid()) self.assertRaises(coordination.ToozConnectionError, fut.get) @mock.patch('pymemcache.client.PooledClient') def test_client_failure_leave(self, mock_client_cls): mock_client = mock.MagicMock() mock_client_cls.return_value = mock_client member_id = tests.get_random_uuid() coord = coordination.get_coordinator(self.FAKE_URL, member_id) coord.start() mock_client.gets.side_effect = socket.timeout('timed-out') fut = coord.leave_group(tests.get_random_uuid()) self.assertRaises(coordination.ToozConnectionError, fut.get) @mock.patch('pymemcache.client.PooledClient') def test_client_failure_heartbeat(self, mock_client_cls): mock_client = mock.MagicMock() mock_client_cls.return_value = mock_client member_id = tests.get_random_uuid() coord = coordination.get_coordinator(self.FAKE_URL, member_id) coord.start() mock_client.set.side_effect = socket.timeout('timed-out') self.assertRaises(coordination.ToozConnectionError, coord.heartbeat) @mock.patch( 'tooz.coordination.CoordinationDriverCachedRunWatchers.run_watchers', autospec=True) @mock.patch('pymemcache.client.PooledClient') def test_client_run_watchers_mixin(self, mock_client_cls, mock_run_watchers): mock_client = mock.MagicMock() mock_client_cls.return_value = mock_client member_id = tests.get_random_uuid() coord = coordination.get_coordinator(self.FAKE_URL, member_id) coord.start() coord.run_watchers() self.assertTrue(mock_run_watchers.called) tooz-2.0.0/tooz/tests/test_utils.py0000664000175000017500000001031313616633007017437 0ustar zuulzuul00000000000000# -*- coding: utf-8 -*- # Copyright (c) 2015 OpenStack Foundation # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import os import tempfile import futurist import six from testtools import testcase import tooz from tooz import utils class TestProxyExecutor(testcase.TestCase): def test_fetch_check_executor(self): try_options = [ ({'executor': 'sync'}, futurist.SynchronousExecutor), ({'executor': 'thread'}, futurist.ThreadPoolExecutor), ] for options, expected_cls in try_options: executor = utils.ProxyExecutor.build("test", options) self.assertTrue(executor.internally_owned) executor.start() self.assertTrue(executor.started) self.assertIsInstance(executor.executor, expected_cls) executor.stop() self.assertFalse(executor.started) def test_fetch_default_executor(self): executor = utils.ProxyExecutor.build("test", {}) executor.start() try: self.assertIsInstance(executor.executor, futurist.ThreadPoolExecutor) finally: executor.stop() def test_fetch_unknown_executor(self): options = {'executor': 'huh'} self.assertRaises(tooz.ToozError, utils.ProxyExecutor.build, 'test', options) def test_no_submit_stopped(self): executor = utils.ProxyExecutor.build("test", {}) self.assertRaises(tooz.ToozError, executor.submit, lambda: None) class TestUtilsSafePath(testcase.TestCase): base = tempfile.gettempdir() def test_join(self): self.assertEqual(os.path.join(self.base, 'b'), utils.safe_abs_path(self.base, "b")) self.assertEqual(os.path.join(self.base, 'b', 'c'), utils.safe_abs_path(self.base, "b", 'c')) self.assertEqual(self.base, utils.safe_abs_path(self.base, "b", 'c', '../..')) def test_unsafe_join(self): self.assertRaises(ValueError, utils.safe_abs_path, self.base, "../b") self.assertRaises(ValueError, utils.safe_abs_path, self.base, "b", 'c', '../../../') class TestUtilsCollapse(testcase.TestCase): def test_bad_type(self): self.assertRaises(TypeError, utils.collapse, "") self.assertRaises(TypeError, utils.collapse, []) self.assertRaises(TypeError, utils.collapse, 2) def test_collapse_simple(self): ex = { 'a': [1], 'b': 2, 'c': (1, 2, 3), } c_ex = utils.collapse(ex) self.assertEqual({'a': 1, 'c': 3, 'b': 2}, c_ex) def test_collapse_exclusions(self): ex = { 'a': [1], 'b': 2, 'c': (1, 2, 3), } c_ex = utils.collapse(ex, exclude=['a']) self.assertEqual({'a': [1], 'c': 3, 'b': 2}, c_ex) def test_no_collapse(self): ex = { 'a': [1], 'b': [2], 'c': (1, 2, 3), } c_ex = utils.collapse(ex, exclude=set(six.iterkeys(ex))) self.assertEqual(ex, c_ex) def test_custom_selector(self): ex = { 'a': [1, 2, 3], } c_ex = utils.collapse(ex, item_selector=lambda items: items[0]) self.assertEqual({'a': 1}, c_ex) def test_empty_lists(self): ex = { 'a': [], 'b': (), 'c': [1], } c_ex = utils.collapse(ex) self.assertNotIn('b', c_ex) self.assertNotIn('a', c_ex) self.assertIn('c', c_ex) tooz-2.0.0/tooz/tests/drivers/0000775000175000017500000000000013616633265016354 5ustar zuulzuul00000000000000tooz-2.0.0/tooz/tests/drivers/test_file.py0000664000175000017500000000477513616633007020713 0ustar zuulzuul00000000000000# -*- coding: utf-8 -*- # Copyright 2016 Cloudbase Solutions Srl # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import os import fixtures import mock from testtools import testcase import tooz from tooz import coordination from tooz import tests class TestFileDriver(testcase.TestCase): _FAKE_MEMBER_ID = tests.get_random_uuid() def test_base_dir(self): file_path = '/fake/file/path' url = 'file://%s' % file_path coord = coordination.get_coordinator(url, self._FAKE_MEMBER_ID) self.assertEqual(file_path, coord._dir) def test_leftover_file(self): fixture = self.useFixture(fixtures.TempDir()) file_path = fixture.path url = 'file://%s' % file_path coord = coordination.get_coordinator(url, self._FAKE_MEMBER_ID) coord.start() self.addCleanup(coord.stop) coord.create_group(b"my_group").get() safe_group_id = coord._make_filesystem_safe(b"my_group") with open(os.path.join(file_path, 'groups', safe_group_id, "junk.txt"), "wb"): pass os.unlink(os.path.join(file_path, 'groups', safe_group_id, '.metadata')) self.assertRaises(tooz.ToozError, coord.delete_group(b"my_group").get) @mock.patch('os.path.normpath', lambda x: x.replace('/', '\\')) @mock.patch('sys.platform', 'win32') def test_base_dir_win32(self): coord = coordination.get_coordinator( 'file:///C:/path/', self._FAKE_MEMBER_ID) self.assertEqual('C:\\path\\', coord._dir) coord = coordination.get_coordinator( 'file:////share_addr/share_path/', self._FAKE_MEMBER_ID) self.assertEqual('\\\\share_addr\\share_path\\', coord._dir) # Administrative shares should be handled properly. coord = coordination.get_coordinator( 'file:////c$/path/', self._FAKE_MEMBER_ID) self.assertEqual('\\\\c$\\path\\', coord._dir) tooz-2.0.0/tooz/tests/drivers/__init__.py0000664000175000017500000000000013616633007020445 0ustar zuulzuul00000000000000tooz-2.0.0/tooz/tests/__init__.py0000664000175000017500000000455313616633007017010 0ustar zuulzuul00000000000000# -*- coding: utf-8 -*- # # Copyright (C) 2014 eNovance Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import functools import os import fixtures from oslo_utils import uuidutils import six from testtools import testcase import tooz def get_random_uuid(): return uuidutils.generate_uuid().encode('ascii') def _skip_decorator(func): @functools.wraps(func) def skip_if_not_implemented(*args, **kwargs): try: return func(*args, **kwargs) except tooz.NotImplemented as e: raise testcase.TestSkipped(str(e)) return skip_if_not_implemented class SkipNotImplementedMeta(type): def __new__(cls, name, bases, local): for attr in local: value = local[attr] if callable(value) and ( attr.startswith('test_') or attr == 'setUp'): local[attr] = _skip_decorator(value) return type.__new__(cls, name, bases, local) @six.add_metaclass(SkipNotImplementedMeta) class TestWithCoordinator(testcase.TestCase): url = os.getenv("TOOZ_TEST_URL") def setUp(self): super(TestWithCoordinator, self).setUp() if self.url is None: raise RuntimeError("No URL set for this driver") if os.getenv("TOOZ_TEST_ETCD3"): self.url = self.url.replace("etcd://", "etcd3://") if os.getenv("TOOZ_TEST_ETCD3GW"): self.url = self.url.replace("etcd://", "etcd3+http://") self.useFixture(fixtures.NestedTempfile()) self.group_id = get_random_uuid() self.member_id = get_random_uuid() self._coord = tooz.coordination.get_coordinator(self.url, self.member_id) self._coord.start() def tearDown(self): self._coord.stop() super(TestWithCoordinator, self).tearDown() tooz-2.0.0/tooz/tests/test_partitioner.py0000664000175000017500000000666613616633007020657 0ustar zuulzuul00000000000000# -*- coding: utf-8 -*- # # Copyright © 2016 Red Hat, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import six from tooz import coordination from tooz import tests class TestPartitioner(tests.TestWithCoordinator): def setUp(self): super(TestPartitioner, self).setUp() self._extra_coords = [] def tearDown(self): for c in self._extra_coords: c.stop() super(TestPartitioner, self).tearDown() def _add_members(self, number_of_members, weight=1): groups = [] for _ in six.moves.range(number_of_members): m = tests.get_random_uuid() coord = coordination.get_coordinator(self.url, m) coord.start() groups.append(coord.join_partitioned_group( self.group_id, weight=weight)) self._extra_coords.append(coord) self._coord.run_watchers() return groups def _remove_members(self, number_of_members): for _ in six.moves.range(number_of_members): c = self._extra_coords.pop() c.stop() self._coord.run_watchers() def test_join_partitioned_group(self): group_id = tests.get_random_uuid() self._coord.join_partitioned_group(group_id) def test_hashring_size(self): p = self._coord.join_partitioned_group(self.group_id) self.assertEqual(1, len(p.ring.nodes)) self._add_members(1) self.assertEqual(2, len(p.ring.nodes)) self._add_members(2) self.assertEqual(4, len(p.ring.nodes)) self._remove_members(3) self.assertEqual(1, len(p.ring.nodes)) p.stop() def test_hashring_weight(self): p = self._coord.join_partitioned_group(self.group_id, weight=5) self.assertEqual([5], list(p.ring.nodes.values())) p2 = self._add_members(1, weight=10)[0] self.assertEqual(set([5, 10]), set(p.ring.nodes.values())) self.assertEqual(set([5, 10]), set(p2.ring.nodes.values())) p.stop() def test_stop(self): p = self._coord.join_partitioned_group(self.group_id) p.stop() self.assertEqual(0, len(self._coord._hooks_join_group)) self.assertEqual(0, len(self._coord._hooks_leave_group)) def test_members_of_object_and_others(self): p = self._coord.join_partitioned_group(self.group_id) self._add_members(3) o = six.text_type(u"чупакабра") m = p.members_for_object(o) self.assertEqual(1, len(m)) m = m.pop() self.assertTrue(p.belongs_to_member(o, m)) self.assertFalse(p.belongs_to_member(o, b"chupacabra")) maybe = self.assertTrue if m == self.member_id else self.assertFalse maybe(p.belongs_to_self(o)) p.stop() class ZakeTestPartitioner(TestPartitioner): url = "zake://" class IPCTestPartitioner(TestPartitioner): url = "ipc://" class FileTestPartitioner(TestPartitioner): url = "file:///tmp" tooz-2.0.0/tooz/_retry.py0000664000175000017500000000174213616633007015410 0ustar zuulzuul00000000000000# -*- coding: utf-8 -*- # # Copyright © 2016 Red Hat, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import tenacity from tenacity import stop from tenacity import wait _default_wait = wait.wait_exponential(max=1) def retry(stop_max_delay=None, **kwargs): k = {"wait": _default_wait, "retry": lambda x: False} if stop_max_delay not in (True, False, None): k['stop'] = stop.stop_after_delay(stop_max_delay) return tenacity.retry(**k) TryAgain = tenacity.TryAgain tooz-2.0.0/tooz/drivers/0000775000175000017500000000000013616633265015212 5ustar zuulzuul00000000000000tooz-2.0.0/tooz/drivers/pgsql.py0000664000175000017500000001773013616633007016714 0ustar zuulzuul00000000000000# -*- coding: utf-8 -*- # # Copyright © 2014 eNovance # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import contextlib import hashlib import logging from oslo_utils import encodeutils import psycopg2 import six import tooz from tooz import _retry from tooz import coordination from tooz import locking from tooz import utils LOG = logging.getLogger(__name__) # See: psycopg/diagnostics_type.c for what kind of fields these # objects may have (things like 'schema_name', 'internal_query' # and so-on which are useful for figuring out what went wrong...) _DIAGNOSTICS_ATTRS = tuple([ 'column_name', 'constraint_name', 'context', 'datatype_name', 'internal_position', 'internal_query', 'message_detail', 'message_hint', 'message_primary', 'schema_name', 'severity', 'source_file', 'source_function', 'source_line', 'sqlstate', 'statement_position', 'table_name', ]) def _format_exception(e): lines = [ "%s: %s" % (type(e).__name__, encodeutils.exception_to_unicode(e).strip()), ] if hasattr(e, 'pgcode') and e.pgcode is not None: lines.append("Error code: %s" % e.pgcode) # The reason this hasattr check is done is that the 'diag' may not always # be present, depending on how new of a psycopg is installed... so better # to be safe than sorry... if hasattr(e, 'diag') and e.diag is not None: diagnostic_lines = [] for attr_name in _DIAGNOSTICS_ATTRS: if not hasattr(e.diag, attr_name): continue attr_value = getattr(e.diag, attr_name) if attr_value is None: continue diagnostic_lines.append(" %s = %s" (attr_name, attr_value)) if diagnostic_lines: lines.append('Diagnostics:') lines.extend(diagnostic_lines) return "\n".join(lines) @contextlib.contextmanager def _translating_cursor(conn): try: with conn.cursor() as cur: yield cur except psycopg2.Error as e: utils.raise_with_cause(tooz.ToozError, _format_exception(e), cause=e) class PostgresLock(locking.Lock): """A PostgreSQL based lock.""" def __init__(self, name, parsed_url, options): super(PostgresLock, self).__init__(name) self.acquired = False self._conn = None self._parsed_url = parsed_url self._options = options h = hashlib.md5() h.update(name) if six.PY2: self.key = list(map(ord, h.digest()[0:2])) else: self.key = h.digest()[0:2] def acquire(self, blocking=True, shared=False): if shared: raise tooz.NotImplemented @_retry.retry(stop_max_delay=blocking) def _lock(): # NOTE(sileht) One the same session the lock is not exclusive # so we track it internally if the process already has the lock. if self.acquired is True: if blocking: raise _retry.TryAgain return False if not self._conn or self._conn.closed: self._conn = PostgresDriver.get_connection(self._parsed_url, self._options) with _translating_cursor(self._conn) as cur: if blocking is True: cur.execute("SELECT pg_advisory_lock(%s, %s);", self.key) cur.fetchone() self.acquired = True return True else: cur.execute("SELECT pg_try_advisory_lock(%s, %s);", self.key) if cur.fetchone()[0] is True: self.acquired = True return True elif blocking is False: self._conn.close() return False else: raise _retry.TryAgain try: return _lock() except Exception: if self._conn: self._conn.close() raise def release(self): if not self.acquired: return False with _translating_cursor(self._conn) as cur: cur.execute("SELECT pg_advisory_unlock(%s, %s);", self.key) cur.fetchone() self.acquired = False self._conn.close() return True def __del__(self): if self.acquired: LOG.warning("unreleased lock %s garbage collected", self.name) class PostgresDriver(coordination.CoordinationDriver): """A `PostgreSQL`_ based driver. This driver users `PostgreSQL`_ database tables to provide the coordination driver semantics and required API(s). It **is** missing some functionality but in the future these not implemented API(s) will be filled in. The PostgreSQL driver connection URI should look like:: postgresql://[USERNAME[:PASSWORD]@]HOST:PORT?dbname=DBNAME .. _PostgreSQL: http://www.postgresql.org/ """ CHARACTERISTICS = ( coordination.Characteristics.NON_TIMEOUT_BASED, coordination.Characteristics.DISTRIBUTED_ACROSS_THREADS, coordination.Characteristics.DISTRIBUTED_ACROSS_PROCESSES, coordination.Characteristics.DISTRIBUTED_ACROSS_HOSTS, ) """ Tuple of :py:class:`~tooz.coordination.Characteristics` introspectable enum member(s) that can be used to interogate how this driver works. """ def __init__(self, member_id, parsed_url, options): """Initialize the PostgreSQL driver.""" super(PostgresDriver, self).__init__(member_id, parsed_url, options) self._parsed_url = parsed_url self._options = utils.collapse(options) def _start(self): self._conn = self.get_connection(self._parsed_url, self._options) def _stop(self): self._conn.close() def get_lock(self, name): return PostgresLock(name, self._parsed_url, self._options) @staticmethod def watch_join_group(group_id, callback): raise tooz.NotImplemented @staticmethod def unwatch_join_group(group_id, callback): raise tooz.NotImplemented @staticmethod def watch_leave_group(group_id, callback): raise tooz.NotImplemented @staticmethod def unwatch_leave_group(group_id, callback): raise tooz.NotImplemented @staticmethod def watch_elected_as_leader(group_id, callback): raise tooz.NotImplemented @staticmethod def unwatch_elected_as_leader(group_id, callback): raise tooz.NotImplemented @staticmethod def get_connection(parsed_url, options): host = options.get("host") or parsed_url.hostname port = options.get("port") or parsed_url.port dbname = options.get("dbname") or parsed_url.path[1:] kwargs = {} if parsed_url.username is not None: kwargs["user"] = parsed_url.username if parsed_url.password is not None: kwargs["password"] = parsed_url.password try: return psycopg2.connect(host=host, port=port, database=dbname, **kwargs) except psycopg2.Error as e: utils.raise_with_cause(coordination.ToozConnectionError, _format_exception(e), cause=e) tooz-2.0.0/tooz/drivers/etcd3gw.py0000664000175000017500000003435013616633007017123 0ustar zuulzuul00000000000000# -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. from __future__ import absolute_import import base64 import threading import uuid import etcd3gw from etcd3gw import exceptions as etcd3_exc from oslo_utils import encodeutils import six import tooz from tooz import _retry from tooz import coordination from tooz import locking from tooz import utils def _encode(data): """Safely encode data for consumption of the gateway.""" return base64.b64encode(data).decode("ascii") def _translate_failures(func): """Translates common requests exceptions into tooz exceptions.""" @six.wraps(func) def wrapper(*args, **kwargs): try: return func(*args, **kwargs) except etcd3_exc.ConnectionFailedError as e: utils.raise_with_cause(coordination.ToozConnectionError, encodeutils.exception_to_unicode(e), cause=e) except etcd3_exc.ConnectionTimeoutError as e: utils.raise_with_cause(coordination.OperationTimedOut, encodeutils.exception_to_unicode(e), cause=e) except etcd3_exc.Etcd3Exception as e: utils.raise_with_cause(coordination.ToozError, encodeutils.exception_to_unicode(e), cause=e) return wrapper class Etcd3Lock(locking.Lock): """An etcd3-specific lock. Thin wrapper over etcd3's lock object basically to provide the heartbeat() semantics for the coordination driver. """ LOCK_PREFIX = b"/tooz/locks" def __init__(self, coord, name, timeout): super(Etcd3Lock, self).__init__(name) self._timeout = timeout self._coord = coord self._key = self.LOCK_PREFIX + name self._key_b64 = _encode(self._key) self._uuid = _encode(uuid.uuid4().bytes) self._exclusive_access = threading.Lock() @_translate_failures def acquire(self, blocking=True, shared=False): if shared: raise tooz.NotImplemented @_retry.retry(stop_max_delay=blocking) def _acquire(): # TODO(jd): save the created revision so we can check it later to # make sure we still have the lock self._lease = self._coord.client.lease(self._timeout) txn = { 'compare': [{ 'key': self._key_b64, 'result': 'EQUAL', 'target': 'CREATE', 'create_revision': 0 }], 'success': [{ 'request_put': { 'key': self._key_b64, 'value': self._uuid, 'lease': self._lease.id } }], 'failure': [{ 'request_range': { 'key': self._key_b64 } }] } result = self._coord.client.transaction(txn) success = result.get('succeeded', False) if success is not True: if blocking is False: return False raise _retry.TryAgain self._coord._acquired_locks.add(self) return True return _acquire() @_translate_failures def release(self): txn = { 'compare': [{ 'key': self._key_b64, 'result': 'EQUAL', 'target': 'VALUE', 'value': self._uuid }], 'success': [{ 'request_delete_range': { 'key': self._key_b64 } }] } with self._exclusive_access: result = self._coord.client.transaction(txn) success = result.get('succeeded', False) if success: self._coord._acquired_locks.remove(self) return True return False @_translate_failures def break_(self): if self._coord.client.delete(self._key): self._coord._acquired_locks.discard(self) return True return False @property def acquired(self): return self in self._coord._acquired_locks @_translate_failures def heartbeat(self): with self._exclusive_access: if self.acquired: self._lease.refresh() return True return False class Etcd3Driver(coordination.CoordinationDriverWithExecutor): """An etcd based driver. This driver uses etcd provide the coordination driver semantics and required API(s). The Etcd driver connection URI should look like:: etcd3+http://[HOST[:PORT]][?OPTION1=VALUE1[&OPTION2=VALUE2[&...]]] If not specified, HOST defaults to localhost and PORT defaults to 2379. Available options are: ================== ======= Name Default ================== ======= protocol http timeout 30 lock_timeout 30 membership_timeout 30 ================== ======= """ #: Default socket/lock/member/leader timeout used when none is provided. DEFAULT_TIMEOUT = 30 #: Default hostname used when none is provided. DEFAULT_HOST = "localhost" #: Default port used if none provided (4001 or 2379 are the common ones). DEFAULT_PORT = 2379 GROUP_PREFIX = b"tooz/groups/" def __init__(self, member_id, parsed_url, options): super(Etcd3Driver, self).__init__(member_id, parsed_url, options) host = parsed_url.hostname or self.DEFAULT_HOST port = parsed_url.port or self.DEFAULT_PORT options = utils.collapse(options) timeout = int(options.get('timeout', self.DEFAULT_TIMEOUT)) self.client = etcd3gw.client(host=host, port=port, timeout=timeout) self.lock_timeout = int(options.get('lock_timeout', timeout)) self.membership_timeout = int(options.get( 'membership_timeout', timeout)) self._acquired_locks = set() def _start(self): super(Etcd3Driver, self)._start() self._membership_lease = self.client.lease(self.membership_timeout) def get_lock(self, name): return Etcd3Lock(self, name, self.lock_timeout) def heartbeat(self): # NOTE(jaypipes): Copying because set can mutate during iteration for lock in self._acquired_locks.copy(): lock.heartbeat() # TODO(kaifeng) use the same lease for locks? self._membership_lease.refresh() return min(self.lock_timeout, self.membership_timeout) def watch_join_group(self, group_id, callback): raise tooz.NotImplemented def unwatch_join_group(self, group_id, callback): raise tooz.NotImplemented def watch_leave_group(self, group_id, callback): raise tooz.NotImplemented def unwatch_leave_group(self, group_id, callback): raise tooz.NotImplemented def _encode_group_id(self, group_id): return _encode(self._prefix_group(group_id)) def _prefix_group(self, group_id): return b"%s%s/" % (self.GROUP_PREFIX, group_id) def create_group(self, group_id): @_translate_failures def _create_group(): encoded_group = self._encode_group_id(group_id) txn = { 'compare': [{ 'key': encoded_group, 'result': 'EQUAL', 'target': 'VERSION', 'version': 0 }], 'success': [{ 'request_put': { 'key': encoded_group, # We shouldn't need a value, but etcd3gw needs it for # now 'value': encoded_group } }], 'failure': [] } result = self.client.transaction(txn) if not result.get("succeeded"): raise coordination.GroupAlreadyExist(group_id) return coordination.CoordinatorResult( self._executor.submit(_create_group)) def _destroy_group(self, group_id): self.client.delete(group_id) def delete_group(self, group_id): @_translate_failures def _delete_group(): prefix_group = self._prefix_group(group_id) members = self.client.get_prefix(prefix_group) if len(members) > 1: raise coordination.GroupNotEmpty(group_id) encoded_group = self._encode_group_id(group_id) txn = { 'compare': [{ 'key': encoded_group, 'result': 'NOT_EQUAL', 'target': 'VERSION', 'version': 0 }], 'success': [{ 'request_delete_range': { 'key': encoded_group, } }], 'failure': [] } result = self.client.transaction(txn) if not result.get("succeeded"): raise coordination.GroupNotCreated(group_id) return coordination.CoordinatorResult( self._executor.submit(_delete_group)) def join_group(self, group_id, capabilities=b""): @_retry.retry() @_translate_failures def _join_group(): prefix_group = self._prefix_group(group_id) prefix_member = prefix_group + self._member_id members = self.client.get_prefix(prefix_group) encoded_member = _encode(prefix_member) group_metadata = None for cap, metadata in members: if metadata['key'] == prefix_member: raise coordination.MemberAlreadyExist(group_id, self._member_id) if metadata['key'] == prefix_group: group_metadata = metadata if group_metadata is None: raise coordination.GroupNotCreated(group_id) encoded_group = self._encode_group_id(group_id) txn = { 'compare': [{ 'key': encoded_group, 'result': 'EQUAL', 'target': 'VERSION', 'version': int(group_metadata['version']) }], 'success': [{ 'request_put': { 'key': encoded_member, 'value': _encode(utils.dumps(capabilities)), 'lease': self._membership_lease.id } }], 'failure': [] } result = self.client.transaction(txn) if not result.get('succeeded'): raise _retry.TryAgain else: self._joined_groups.add(group_id) return coordination.CoordinatorResult( self._executor.submit(_join_group)) def leave_group(self, group_id): @_translate_failures def _leave_group(): prefix_group = self._prefix_group(group_id) prefix_member = prefix_group + self._member_id members = self.client.get_prefix(prefix_group) for capabilities, metadata in members: if metadata['key'] == prefix_member: break else: raise coordination.MemberNotJoined(group_id, self._member_id) self.client.delete(prefix_member) self._joined_groups.discard(group_id) return coordination.CoordinatorResult( self._executor.submit(_leave_group)) def get_members(self, group_id): @_translate_failures def _get_members(): prefix_group = self._prefix_group(group_id) members = set() group_found = False for cap, metadata in self.client.get_prefix(prefix_group): if metadata['key'] == prefix_group: group_found = True else: members.add(metadata['key'][len(prefix_group):]) if not group_found: raise coordination.GroupNotCreated(group_id) return members return coordination.CoordinatorResult( self._executor.submit(_get_members)) def get_member_capabilities(self, group_id, member_id): @_translate_failures def _get_member_capabilities(): prefix_member = self._prefix_group(group_id) + member_id result = self.client.get(prefix_member) if not result: raise coordination.MemberNotJoined(group_id, member_id) return utils.loads(result[0]) return coordination.CoordinatorResult( self._executor.submit(_get_member_capabilities)) def update_capabilities(self, group_id, capabilities): @_translate_failures def _update_capabilities(): prefix_member = self._prefix_group(group_id) + self._member_id result = self.client.get(prefix_member) if not result: raise coordination.MemberNotJoined(group_id, self._member_id) self.client.put(prefix_member, utils.dumps(capabilities), lease=self._membership_lease) return coordination.CoordinatorResult( self._executor.submit(_update_capabilities)) def get_groups(self): @_translate_failures def _get_groups(): groups = self.client.get_prefix(self.GROUP_PREFIX) return [ group[1]['key'][len(self.GROUP_PREFIX):-1] for group in groups] return coordination.CoordinatorResult( self._executor.submit(_get_groups)) tooz-2.0.0/tooz/drivers/mysql.py0000664000175000017500000001547013616633007016732 0ustar zuulzuul00000000000000# -*- coding: utf-8 -*- # # Copyright © 2014 eNovance # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import logging from oslo_utils import encodeutils import pymysql import tooz from tooz import _retry from tooz import coordination from tooz import locking from tooz import utils LOG = logging.getLogger(__name__) class MySQLLock(locking.Lock): """A MySQL based lock.""" MYSQL_DEFAULT_PORT = 3306 def __init__(self, name, parsed_url, options): super(MySQLLock, self).__init__(name) self.acquired = False self._conn = MySQLDriver.get_connection(parsed_url, options, True) def acquire(self, blocking=True, shared=False): if shared: raise tooz.NotImplemented @_retry.retry(stop_max_delay=blocking) def _lock(): # NOTE(sileht): mysql-server (<5.7.5) allows only one lock per # connection at a time: # select GET_LOCK("a", 0); # select GET_LOCK("b", 0); <-- this release lock "a" ... # Or # select GET_LOCK("a", 0); # select GET_LOCK("a", 0); release and lock again "a" # # So, we track locally the lock status with self.acquired if self.acquired is True: if blocking: raise _retry.TryAgain return False try: if not self._conn.open: self._conn.connect() with self._conn as cur: cur.execute("SELECT GET_LOCK(%s, 0);", self.name) # Can return NULL on error if cur.fetchone()[0] is 1: self.acquired = True return True except pymysql.MySQLError as e: utils.raise_with_cause( tooz.ToozError, encodeutils.exception_to_unicode(e), cause=e) if blocking: raise _retry.TryAgain self._conn.close() return False try: return _lock() except Exception: # Close the connection if we tried too much and finally failed, or # anything else bad happened. self._conn.close() raise def release(self): if not self.acquired: return False try: with self._conn as cur: cur.execute("SELECT RELEASE_LOCK(%s);", self.name) cur.fetchone() self.acquired = False self._conn.close() return True except pymysql.MySQLError as e: utils.raise_with_cause(tooz.ToozError, encodeutils.exception_to_unicode(e), cause=e) def __del__(self): if self.acquired: LOG.warning("unreleased lock %s garbage collected", self.name) class MySQLDriver(coordination.CoordinationDriver): """A `MySQL`_ based driver. This driver users `MySQL`_ database tables to provide the coordination driver semantics and required API(s). It **is** missing some functionality but in the future these not implemented API(s) will be filled in. The MySQL driver connection URI should look like:: mysql://USERNAME:PASSWORD@HOST[:PORT]/DBNAME[?unix_socket=SOCKET_PATH] If not specified, PORT defaults to 3306. .. _MySQL: http://dev.mysql.com/ """ CHARACTERISTICS = ( coordination.Characteristics.NON_TIMEOUT_BASED, coordination.Characteristics.DISTRIBUTED_ACROSS_THREADS, coordination.Characteristics.DISTRIBUTED_ACROSS_PROCESSES, coordination.Characteristics.DISTRIBUTED_ACROSS_HOSTS, ) """ Tuple of :py:class:`~tooz.coordination.Characteristics` introspectable enum member(s) that can be used to interogate how this driver works. """ def __init__(self, member_id, parsed_url, options): """Initialize the MySQL driver.""" super(MySQLDriver, self).__init__(member_id, parsed_url, options) self._parsed_url = parsed_url self._options = utils.collapse(options) def _start(self): self._conn = MySQLDriver.get_connection(self._parsed_url, self._options) def _stop(self): self._conn.close() def get_lock(self, name): return MySQLLock(name, self._parsed_url, self._options) @staticmethod def watch_join_group(group_id, callback): raise tooz.NotImplemented @staticmethod def unwatch_join_group(group_id, callback): raise tooz.NotImplemented @staticmethod def watch_leave_group(group_id, callback): raise tooz.NotImplemented @staticmethod def unwatch_leave_group(group_id, callback): raise tooz.NotImplemented @staticmethod def watch_elected_as_leader(group_id, callback): raise tooz.NotImplemented @staticmethod def unwatch_elected_as_leader(group_id, callback): raise tooz.NotImplemented @staticmethod def get_connection(parsed_url, options, defer_connect=False): host = parsed_url.hostname port = parsed_url.port or MySQLLock.MYSQL_DEFAULT_PORT dbname = parsed_url.path[1:] username = parsed_url.username password = parsed_url.password unix_socket = options.get("unix_socket") try: if unix_socket: return pymysql.Connect(unix_socket=unix_socket, port=port, user=username, passwd=password, database=dbname, defer_connect=defer_connect) else: return pymysql.Connect(host=host, port=port, user=username, passwd=password, database=dbname, defer_connect=defer_connect) except (pymysql.err.OperationalError, pymysql.err.InternalError) as e: utils.raise_with_cause(coordination.ToozConnectionError, encodeutils.exception_to_unicode(e), cause=e) tooz-2.0.0/tooz/drivers/memcached.py0000664000175000017500000004701713616633007017475 0ustar zuulzuul00000000000000# -*- coding: utf-8 -*- # # Copyright © 2014 eNovance # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import contextlib import errno import functools import logging import socket from oslo_utils import encodeutils from pymemcache import client as pymemcache_client import six import tooz from tooz import _retry from tooz import coordination from tooz import locking from tooz import utils LOG = logging.getLogger(__name__) @contextlib.contextmanager def _failure_translator(): """Translates common pymemcache exceptions into tooz exceptions. https://github.com/pinterest/pymemcache/blob/d995/pymemcache/client.py#L202 """ try: yield except pymemcache_client.MemcacheUnexpectedCloseError as e: utils.raise_with_cause(coordination.ToozConnectionError, encodeutils.exception_to_unicode(e), cause=e) except (socket.timeout, socket.error, socket.gaierror, socket.herror) as e: # TODO(harlowja): get upstream pymemcache to produce a better # exception for these, using socket (vs. a memcache specific # error) seems sorta not right and/or the best approach... msg = encodeutils.exception_to_unicode(e) if e.errno is not None: msg += " (with errno %s [%s])" % (errno.errorcode[e.errno], e.errno) utils.raise_with_cause(coordination.ToozConnectionError, msg, cause=e) except pymemcache_client.MemcacheError as e: utils.raise_with_cause(tooz.ToozError, encodeutils.exception_to_unicode(e), cause=e) def _translate_failures(func): @six.wraps(func) def wrapper(*args, **kwargs): with _failure_translator(): return func(*args, **kwargs) return wrapper class MemcachedLock(locking.Lock): _LOCK_PREFIX = b'__TOOZ_LOCK_' def __init__(self, coord, name, timeout): super(MemcachedLock, self).__init__(self._LOCK_PREFIX + name) self.coord = coord self.timeout = timeout def is_still_owner(self): if not self.acquired: return False else: owner = self.get_owner() if owner is None: return False return owner == self.coord._member_id def acquire(self, blocking=True, shared=False): if shared: raise tooz.NotImplemented @_retry.retry(stop_max_delay=blocking) @_translate_failures def _acquire(): if self.coord.client.add( self.name, self.coord._member_id, expire=self.timeout, noreply=False): self.coord._acquired_locks.append(self) return True if blocking is False: return False raise _retry.TryAgain return _acquire() @_translate_failures def break_(self): return bool(self.coord.client.delete(self.name, noreply=False)) @_translate_failures def release(self): if not self.acquired: return False # NOTE(harlowja): this has the potential to delete others locks # especially if this key expired before the delete/release call is # triggered. # # For example: # # 1. App #1 with coordinator 'A' acquires lock "b" # 2. App #1 heartbeats every 10 seconds, expiry for lock let's # say is 11 seconds. # 3. App #2 with coordinator also named 'A' blocks trying to get # lock "b" (let's say it retries attempts every 0.5 seconds) # 4. App #1 is running behind a little bit, tries to heartbeat but # key has expired (log message is written); at this point app #1 # doesn't own the lock anymore but it doesn't know that. # 5. App #2 now retries and adds the key, and now it believes it # has the lock. # 6. App #1 (still believing it has the lock) calls release, and # deletes app #2 lock, app #2 now doesn't own the lock anymore # but it doesn't know that and now app #(X + 1) can get it. # 7. App #2 calls release (repeat #6 as many times as desired) # # Sadly I don't think memcache has the primitives to actually make # this work, redis does because it has lua which can check a session # id and then do the delete and bail out if the session id is not # as expected but memcache doesn't seem to have any equivalent # capability. if self not in self.coord._acquired_locks: return False # Do a ghetto test to see what the value is... (see above note), # and how this really can't be done safely with memcache due to # it being done in the client side (non-atomic). value = self.coord.client.get(self.name) if value != self.coord._member_id: return False else: was_deleted = self.coord.client.delete(self.name, noreply=False) if was_deleted: self.coord._acquired_locks.remove(self) return was_deleted @_translate_failures def heartbeat(self): """Keep the lock alive.""" if self.acquired: poked = self.coord.client.touch(self.name, expire=self.timeout, noreply=False) if poked: return True LOG.warning("Unable to heartbeat by updating key '%s' with " "extended expiry of %s seconds", self.name, self.timeout) return False @_translate_failures def get_owner(self): return self.coord.client.get(self.name) @property def acquired(self): return self in self.coord._acquired_locks class MemcachedDriver(coordination.CoordinationDriverCachedRunWatchers, coordination.CoordinationDriverWithExecutor): """A `memcached`_ based driver. This driver users `memcached`_ concepts to provide the coordination driver semantics and required API(s). It **is** fully functional and implements all of the coordination driver API(s). It stores data into memcache using expiries and `msgpack`_ encoded values. The Memcached driver connection URI should look like:: memcached://[HOST[:PORT]][?OPTION1=VALUE1[&OPTION2=VALUE2[&...]]] If not specified, HOST defaults to localhost and PORT defaults to 11211. Available options are: ================== ======= Name Default ================== ======= timeout 30 membership_timeout 30 lock_timeout 30 leader_timeout 30 max_pool_size None ================== ======= General recommendations/usage considerations: - Memcache (without different backend technology) is a **cache** enough said. .. _memcached: http://memcached.org/ .. _msgpack: http://msgpack.org/ """ CHARACTERISTICS = ( coordination.Characteristics.DISTRIBUTED_ACROSS_THREADS, coordination.Characteristics.DISTRIBUTED_ACROSS_PROCESSES, coordination.Characteristics.DISTRIBUTED_ACROSS_HOSTS, coordination.Characteristics.CAUSAL, ) """ Tuple of :py:class:`~tooz.coordination.Characteristics` introspectable enum member(s) that can be used to interogate how this driver works. """ #: Key prefix attached to groups (used in name-spacing keys) GROUP_PREFIX = b'_TOOZ_GROUP_' #: Key prefix attached to leaders of groups (used in name-spacing keys) GROUP_LEADER_PREFIX = b'_TOOZ_GROUP_LEADER_' #: Key prefix attached to members of groups (used in name-spacing keys) MEMBER_PREFIX = b'_TOOZ_MEMBER_' #: Key where all groups 'known' are stored. GROUP_LIST_KEY = b'_TOOZ_GROUP_LIST' #: Default socket/lock/member/leader timeout used when none is provided. DEFAULT_TIMEOUT = 30 #: String used to keep a key/member alive (until it next expires). STILL_ALIVE = b"It's alive!" def __init__(self, member_id, parsed_url, options): super(MemcachedDriver, self).__init__(member_id, parsed_url, options) self.host = (parsed_url.hostname or "localhost", parsed_url.port or 11211) default_timeout = self._options.get('timeout', self.DEFAULT_TIMEOUT) self.timeout = int(default_timeout) self.membership_timeout = int(self._options.get( 'membership_timeout', default_timeout)) self.lock_timeout = int(self._options.get( 'lock_timeout', default_timeout)) self.leader_timeout = int(self._options.get( 'leader_timeout', default_timeout)) max_pool_size = self._options.get('max_pool_size', None) if max_pool_size is not None: self.max_pool_size = int(max_pool_size) else: self.max_pool_size = None self._acquired_locks = [] @staticmethod def _msgpack_serializer(key, value): if isinstance(value, six.binary_type): return value, 1 return utils.dumps(value), 2 @staticmethod def _msgpack_deserializer(key, value, flags): if flags == 1: return value if flags == 2: return utils.loads(value) raise coordination.SerializationError("Unknown serialization" " format '%s'" % flags) @_translate_failures def _start(self): super(MemcachedDriver, self)._start() self.client = pymemcache_client.PooledClient( self.host, serializer=self._msgpack_serializer, deserializer=self._msgpack_deserializer, timeout=self.timeout, connect_timeout=self.timeout, max_pool_size=self.max_pool_size) # Run heartbeat here because pymemcache use a lazy connection # method and only connect once you do an operation. self.heartbeat() @_translate_failures def _stop(self): super(MemcachedDriver, self)._stop() for lock in list(self._acquired_locks): lock.release() self.client.delete(self._encode_member_id(self._member_id)) self.client.close() def _encode_group_id(self, group_id): return self.GROUP_PREFIX + utils.to_binary(group_id) def _encode_member_id(self, member_id): return self.MEMBER_PREFIX + utils.to_binary(member_id) def _encode_group_leader(self, group_id): return self.GROUP_LEADER_PREFIX + utils.to_binary(group_id) @_retry.retry() def _add_group_to_group_list(self, group_id): """Add group to the group list. :param group_id: The group id """ group_list, cas = self.client.gets(self.GROUP_LIST_KEY) if cas: group_list = set(group_list) group_list.add(group_id) if not self.client.cas(self.GROUP_LIST_KEY, list(group_list), cas): # Someone updated the group list before us, try again! raise _retry.TryAgain else: if not self.client.add(self.GROUP_LIST_KEY, [group_id], noreply=False): # Someone updated the group list before us, try again! raise _retry.TryAgain @_retry.retry() def _remove_from_group_list(self, group_id): """Remove group from the group list. :param group_id: The group id """ group_list, cas = self.client.gets(self.GROUP_LIST_KEY) group_list = set(group_list) group_list.remove(group_id) if not self.client.cas(self.GROUP_LIST_KEY, list(group_list), cas): # Someone updated the group list before us, try again! raise _retry.TryAgain def create_group(self, group_id): encoded_group = self._encode_group_id(group_id) @_translate_failures def _create_group(): if not self.client.add(encoded_group, {}, noreply=False): raise coordination.GroupAlreadyExist(group_id) self._add_group_to_group_list(group_id) return MemcachedFutureResult(self._executor.submit(_create_group)) def get_groups(self): @_translate_failures def _get_groups(): return self.client.get(self.GROUP_LIST_KEY) or [] return MemcachedFutureResult(self._executor.submit(_get_groups)) def join_group(self, group_id, capabilities=b""): encoded_group = self._encode_group_id(group_id) @_retry.retry() @_translate_failures def _join_group(): group_members, cas = self.client.gets(encoded_group) if group_members is None: raise coordination.GroupNotCreated(group_id) if self._member_id in group_members: raise coordination.MemberAlreadyExist(group_id, self._member_id) group_members[self._member_id] = { b"capabilities": capabilities, } if not self.client.cas(encoded_group, group_members, cas): # It changed, let's try again raise _retry.TryAgain self._joined_groups.add(group_id) return MemcachedFutureResult(self._executor.submit(_join_group)) def leave_group(self, group_id): encoded_group = self._encode_group_id(group_id) @_retry.retry() @_translate_failures def _leave_group(): group_members, cas = self.client.gets(encoded_group) if group_members is None: raise coordination.GroupNotCreated(group_id) if self._member_id not in group_members: raise coordination.MemberNotJoined(group_id, self._member_id) del group_members[self._member_id] if not self.client.cas(encoded_group, group_members, cas): # It changed, let's try again raise _retry.TryAgain self._joined_groups.discard(group_id) return MemcachedFutureResult(self._executor.submit(_leave_group)) def _destroy_group(self, group_id): self.client.delete(self._encode_group_id(group_id)) def delete_group(self, group_id): encoded_group = self._encode_group_id(group_id) @_retry.retry() @_translate_failures def _delete_group(): group_members, cas = self.client.gets(encoded_group) if group_members is None: raise coordination.GroupNotCreated(group_id) if group_members != {}: raise coordination.GroupNotEmpty(group_id) # Delete is not atomic, so we first set the group to # using CAS, and then we delete it, to avoid race conditions. if not self.client.cas(encoded_group, None, cas): raise _retry.TryAgain self.client.delete(encoded_group) self._remove_from_group_list(group_id) return MemcachedFutureResult(self._executor.submit(_delete_group)) @_retry.retry() @_translate_failures def _get_members(self, group_id): encoded_group = self._encode_group_id(group_id) group_members, cas = self.client.gets(encoded_group) if group_members is None: raise coordination.GroupNotCreated(group_id) actual_group_members = {} for m, v in six.iteritems(group_members): # Never kick self from the group, we know we're alive if (m == self._member_id or self.client.get(self._encode_member_id(m))): actual_group_members[m] = v if group_members != actual_group_members: # There are some dead members, update the group if not self.client.cas(encoded_group, actual_group_members, cas): # It changed, let's try again raise _retry.TryAgain return actual_group_members def get_members(self, group_id): def _get_members(): return set(self._get_members(group_id).keys()) return MemcachedFutureResult(self._executor.submit(_get_members)) def get_member_capabilities(self, group_id, member_id): def _get_member_capabilities(): group_members = self._get_members(group_id) if member_id not in group_members: raise coordination.MemberNotJoined(group_id, member_id) return group_members[member_id][b'capabilities'] return MemcachedFutureResult( self._executor.submit(_get_member_capabilities)) def update_capabilities(self, group_id, capabilities): encoded_group = self._encode_group_id(group_id) @_retry.retry() @_translate_failures def _update_capabilities(): group_members, cas = self.client.gets(encoded_group) if group_members is None: raise coordination.GroupNotCreated(group_id) if self._member_id not in group_members: raise coordination.MemberNotJoined(group_id, self._member_id) group_members[self._member_id][b'capabilities'] = capabilities if not self.client.cas(encoded_group, group_members, cas): # It changed, try again raise _retry.TryAgain return MemcachedFutureResult( self._executor.submit(_update_capabilities)) def get_leader(self, group_id): def _get_leader(): return self._get_leader_lock(group_id).get_owner() return MemcachedFutureResult(self._executor.submit(_get_leader)) @_translate_failures def heartbeat(self): self.client.set(self._encode_member_id(self._member_id), self.STILL_ALIVE, expire=self.membership_timeout) # Reset the acquired locks for lock in self._acquired_locks: lock.heartbeat() return min(self.membership_timeout, self.leader_timeout, self.lock_timeout) def get_lock(self, name): return MemcachedLock(self, name, self.lock_timeout) def _get_leader_lock(self, group_id): return MemcachedLock(self, self._encode_group_leader(group_id), self.leader_timeout) @_translate_failures def run_elect_coordinator(self): for group_id, hooks in six.iteritems(self._hooks_elected_leader): # Try to grab the lock, if that fails, that means someone has it # already. leader_lock = self._get_leader_lock(group_id) if leader_lock.acquire(blocking=False): # We got the lock hooks.run(coordination.LeaderElected( group_id, self._member_id)) def run_watchers(self, timeout=None): result = super(MemcachedDriver, self).run_watchers(timeout=timeout) self.run_elect_coordinator() return result MemcachedFutureResult = functools.partial( coordination.CoordinatorResult, failure_translator=_failure_translator) tooz-2.0.0/tooz/drivers/etcd3.py0000664000175000017500000003002013616633007016553 0ustar zuulzuul00000000000000# -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. from __future__ import absolute_import import contextlib import functools import threading import etcd3 from etcd3 import exceptions as etcd3_exc from oslo_utils import encodeutils import six import tooz from tooz import _retry from tooz import coordination from tooz import locking from tooz import utils @contextlib.contextmanager def _failure_translator(): """Translates common requests exceptions into tooz exceptions.""" try: yield except etcd3_exc.ConnectionFailedError as e: utils.raise_with_cause(coordination.ToozConnectionError, encodeutils.exception_to_unicode(e), cause=e) except etcd3_exc.ConnectionTimeoutError as e: utils.raise_with_cause(coordination.OperationTimedOut, encodeutils.exception_to_unicode(e), cause=e) except etcd3_exc.Etcd3Exception as e: utils.raise_with_cause(coordination.ToozError, encodeutils.exception_to_unicode(e), cause=e) def _translate_failures(func): @six.wraps(func) def wrapper(*args, **kwargs): with _failure_translator(): return func(*args, **kwargs) return wrapper class Etcd3Lock(locking.Lock): """An etcd3-specific lock. Thin wrapper over etcd3's lock object basically to provide the heartbeat() semantics for the coordination driver. """ LOCK_PREFIX = b"/tooz/locks" def __init__(self, coord, name, timeout): super(Etcd3Lock, self).__init__(name) self._coord = coord self._lock = coord.client.lock(name.decode(), timeout) self._exclusive_access = threading.Lock() @_translate_failures def acquire(self, blocking=True, shared=False): if shared: raise tooz.NotImplemented blocking, timeout = utils.convert_blocking(blocking) if blocking is False: timeout = 0 if self._lock.acquire(timeout): self._coord._acquired_locks.add(self) return True return False @property def acquired(self): return self in self._coord._acquired_locks @_translate_failures def release(self): with self._exclusive_access: if self.acquired and self._lock.release(): self._coord._acquired_locks.discard(self) return True return False @_translate_failures def heartbeat(self): with self._exclusive_access: if self.acquired: self._lock.refresh() return True return False class Etcd3Driver(coordination.CoordinationDriverCachedRunWatchers, coordination.CoordinationDriverWithExecutor): """An etcd based driver. This driver uses etcd provide the coordination driver semantics and required API(s). The Etcd driver connection URI should look like:: etcd3://[HOST[:PORT]][?OPTION1=VALUE1[&OPTION2=VALUE2[&...]]] If not specified, HOST defaults to localhost and PORT defaults to 2379. Available options are: ================== ======= Name Default ================== ======= protocol http timeout 30 lock_timeout 30 membership_timeout 30 ================== ======= """ #: Default socket/lock/member/leader timeout used when none is provided. DEFAULT_TIMEOUT = 30 #: Default hostname used when none is provided. DEFAULT_HOST = "localhost" #: Default port used if none provided (4001 or 2379 are the common ones). DEFAULT_PORT = 2379 def __init__(self, member_id, parsed_url, options): super(Etcd3Driver, self).__init__(member_id, parsed_url, options) host = parsed_url.hostname or self.DEFAULT_HOST port = parsed_url.port or self.DEFAULT_PORT options = utils.collapse(options) timeout = int(options.get('timeout', self.DEFAULT_TIMEOUT)) self.client = etcd3.client(host=host, port=port, timeout=timeout) self.lock_timeout = int(options.get('lock_timeout', timeout)) self.membership_timeout = int(options.get( 'membership_timeout', timeout)) self._acquired_locks = set() def _start(self): super(Etcd3Driver, self)._start() self._membership_lease = self.client.lease(self.membership_timeout) def _stop(self): super(Etcd3Driver, self)._stop() self._membership_lease.revoke() def get_lock(self, name): return Etcd3Lock(self, name, self.lock_timeout) def heartbeat(self): # NOTE(jaypipes): Copying because set can mutate during iteration for lock in self._acquired_locks.copy(): lock.heartbeat() # TODO(jd) use the same lease for locks? self._membership_lease.refresh() return min(self.lock_timeout, self.membership_timeout) GROUP_PREFIX = b"tooz/groups/" def _encode_group_id(self, group_id): return self.GROUP_PREFIX + utils.to_binary(group_id) + b"/" def _encode_group_member_id(self, group_id, member_id): return self._encode_group_id(group_id) + utils.to_binary(member_id) def create_group(self, group_id): encoded_group = self._encode_group_id(group_id) @_translate_failures def _create_group(): status, results = self.client.transaction( compare=[ self.client.transactions.version( encoded_group) == 0 ], success=[ self.client.transactions.put(encoded_group, b"") ], failure=[], ) if not status: raise coordination.GroupAlreadyExist(group_id) return EtcdFutureResult(self._executor.submit(_create_group)) def _destroy_group(self, group_id): self.client.delete(self._encode_group_id(group_id)) def delete_group(self, group_id): encoded_group = self._encode_group_id(group_id) @_translate_failures def _delete_group(): members = list(self.client.get_prefix(encoded_group)) if len(members) > 1: raise coordination.GroupNotEmpty(group_id) # Warning: as of this writing python-etcd3 does not support the # NOT_EQUAL operator so we use the EQUAL operator and will retry on # success, hihi status, results = self.client.transaction( compare=[ self.client.transactions.version(encoded_group) == 0 ], success=[], failure=[ self.client.transactions.delete(encoded_group) ], ) if status: raise coordination.GroupNotCreated(group_id) return EtcdFutureResult(self._executor.submit(_delete_group)) def join_group(self, group_id, capabilities=b""): encoded_group = self._encode_group_id(group_id) @_retry.retry() @_translate_failures def _join_group(): members = list(self.client.get_prefix(encoded_group)) encoded_member = self._encode_group_member_id( group_id, self._member_id) group_metadata = None for cap, metadata in members: if metadata.key == encoded_member: raise coordination.MemberAlreadyExist(group_id, self._member_id) if metadata.key == encoded_group: group_metadata = metadata if group_metadata is None: raise coordination.GroupNotCreated(group_id) status, results = self.client.transaction( # This comparison makes sure the group has not been deleted in # the mean time compare=[ self.client.transactions.version(encoded_group) == group_metadata.version ], success=[ self.client.transactions.put(encoded_member, utils.dumps(capabilities), lease=self._membership_lease) ], failure=[], ) if not status: # TODO(jd) There's a small optimization doable by getting the # current range on failure and passing it to this function as # the first arg when retrying to avoid redoing a get_prefix() raise _retry.TryAgain return EtcdFutureResult(self._executor.submit(_join_group)) def leave_group(self, group_id): encoded_group = self._encode_group_id(group_id) @_translate_failures def _leave_group(): members = list(self.client.get_prefix(encoded_group)) encoded_member = self._encode_group_member_id( group_id, self._member_id) for capabilities, metadata in members: if metadata.key == encoded_member: break else: raise coordination.MemberNotJoined(group_id, self._member_id) self.client.delete(encoded_member) return EtcdFutureResult(self._executor.submit(_leave_group)) def get_members(self, group_id): encoded_group = self._encode_group_id(group_id) @_translate_failures def _get_members(): members = set() group_found = False for cap, metadata in self.client.get_prefix(encoded_group): if metadata.key == encoded_group: group_found = True else: members.add(metadata.key[len(encoded_group):]) if not group_found: raise coordination.GroupNotCreated(group_id) return members return EtcdFutureResult(self._executor.submit(_get_members)) def get_member_capabilities(self, group_id, member_id): encoded_member = self._encode_group_member_id( group_id, member_id) @_translate_failures def _get_member_capabilities(): capabilities, metadata = self.client.get(encoded_member) if capabilities is None: raise coordination.MemberNotJoined(group_id, member_id) return utils.loads(capabilities) return EtcdFutureResult( self._executor.submit(_get_member_capabilities)) def update_capabilities(self, group_id, capabilities): encoded_member = self._encode_group_member_id( group_id, self._member_id) @_translate_failures def _update_capabilities(): cap, metadata = self.client.get(encoded_member) if cap is None: raise coordination.MemberNotJoined(group_id, self._member_id) self.client.put(encoded_member, utils.dumps(capabilities), lease=self._membership_lease) return EtcdFutureResult( self._executor.submit(_update_capabilities)) @staticmethod def watch_elected_as_leader(group_id, callback): raise tooz.NotImplemented @staticmethod def unwatch_elected_as_leader(group_id, callback): raise tooz.NotImplemented EtcdFutureResult = functools.partial(coordination.CoordinatorResult, failure_translator=_failure_translator) tooz-2.0.0/tooz/drivers/file.py0000664000175000017500000005040013616633007016474 0ustar zuulzuul00000000000000# -*- coding: utf-8 -*- # # Copyright © 2015 eNovance # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import contextlib import datetime import errno import functools import hashlib import logging import os import re import shutil import sys import tempfile import threading import weakref import fasteners from oslo_utils import encodeutils from oslo_utils import fileutils from oslo_utils import timeutils import six import voluptuous import tooz from tooz import coordination from tooz import locking from tooz import utils LOG = logging.getLogger(__name__) class _Barrier(object): def __init__(self): self.cond = threading.Condition() self.owner = None self.shared = False self.ref = 0 @contextlib.contextmanager def _translate_failures(): try: yield except (EnvironmentError, voluptuous.Invalid) as e: utils.raise_with_cause(tooz.ToozError, encodeutils.exception_to_unicode(e), cause=e) def _convert_from_old_format(data): # NOTE(sileht): previous version of the driver was storing str as-is # making impossible to read from python3 something written with python2 # version of the lib. # Now everything is stored with explicit type bytes or unicode. This # convert the old format to the new one to maintain compat of already # deployed file. # example of potential old python2 payload: # {b"member_id": b"member"} # {b"member_id": u"member"} # example of potential old python3 payload: # {u"member_id": b"member"} # {u"member_id": u"member"} if six.PY3 and b"member_id" in data or b"group_id" in data: data = dict((k.decode("utf8"), v) for k, v in data.items()) # About member_id and group_id valuse if the file have been written # with python2 and in the old format, we can't known with python3 # if we need to decode the value or not. Python3 see bytes blob # We keep it as-is and pray, this have a good change to break if # the application was using str in python2 and unicode in python3 # The member file is often overridden so it's should be fine # But the group file can be very old, so we # now have to update it each time create_group is called return data def _lock_me(lock): def wrapper(func): @six.wraps(func) def decorator(*args, **kwargs): with lock: return func(*args, **kwargs) return decorator return wrapper class FileLock(locking.Lock): """A file based lock.""" def __init__(self, path, barrier, member_id): super(FileLock, self).__init__(path) self.acquired = False self._lock = fasteners.InterProcessLock(path) self._barrier = barrier self._member_id = member_id self.ref = 0 def is_still_owner(self): return self.acquired def acquire(self, blocking=True, shared=False): blocking, timeout = utils.convert_blocking(blocking) watch = timeutils.StopWatch(duration=timeout) watch.start() # Make the shared barrier ours first. with self._barrier.cond: while self._barrier.owner is not None: if (shared and self._barrier.shared): break if not blocking or watch.expired(): return False self._barrier.cond.wait(watch.leftover(return_none=True)) self._barrier.owner = (threading.current_thread().ident, os.getpid(), self._member_id) self._barrier.shared = shared self._barrier.ref += 1 self.ref += 1 # Ok at this point we are now working in a thread safe manner, # and now we can try to get the actual lock... gotten = False try: gotten = self._lock.acquire( blocking=blocking, # Since the barrier waiting may have # taken a long time, we have to use # the leftover (and not the original). timeout=watch.leftover(return_none=True)) finally: # NOTE(harlowja): do this in a finally block to **ensure** that # we release the barrier if something bad happens... if not gotten: # Release the barrier to let someone else have a go at it... with self._barrier.cond: self._barrier.owner = None self._barrier.ref = 0 self._barrier.shared = False self._barrier.cond.notify_all() self.acquired = gotten return gotten def release(self): if not self.acquired: return False with self._barrier.cond: self._barrier.ref -= 1 self.ref -= 1 if not self.ref: self.acquired = False if not self._barrier.ref: self._barrier.owner = None self._lock.release() self._barrier.cond.notify_all() return True def __del__(self): if self.acquired: LOG.warning("Unreleased lock %s garbage collected", self.name) class FileDriver(coordination.CoordinationDriverCachedRunWatchers, coordination.CoordinationDriverWithExecutor): """A file based driver. This driver uses files and directories (and associated file locks) to provide the coordination driver semantics and required API(s). It **is** missing some functionality but in the future these not implemented API(s) will be filled in. The File driver connection URI should look like:: file://DIRECTORY[?timeout=TIMEOUT] DIRECTORY is the location that should be used to store lock files. TIMEOUT defaults to 10. General recommendations/usage considerations: - It does **not** automatically delete members from groups of processes that have died, manual cleanup will be needed for those types of failures. - It is **not** distributed (or recommended to be used in those situations, so the developer using this should really take that into account when applying this driver in there app). """ CHARACTERISTICS = ( coordination.Characteristics.NON_TIMEOUT_BASED, coordination.Characteristics.DISTRIBUTED_ACROSS_THREADS, coordination.Characteristics.DISTRIBUTED_ACROSS_PROCESSES, ) """ Tuple of :py:class:`~tooz.coordination.Characteristics` introspectable enum member(s) that can be used to interogate how this driver works. """ HASH_ROUTINE = 'sha1' """This routine is used to hash a member (or group) id into a filesystem safe name that can be used for member lookup and group joining.""" _barriers = weakref.WeakValueDictionary() """ Barriers shared among all file driver locks, this is required since interprocess locking is not thread aware, so we must add the thread awareness on-top of it instead. """ def __init__(self, member_id, parsed_url, options): """Initialize the file driver.""" super(FileDriver, self).__init__(member_id, parsed_url, options) self._dir = self._normalize_path(parsed_url.path) self._group_dir = os.path.join(self._dir, 'groups') self._tmpdir = os.path.join(self._dir, 'tmp') self._driver_lock_path = os.path.join(self._dir, '.driver_lock') self._driver_lock = self._get_raw_lock(self._driver_lock_path, self._member_id) self._reserved_dirs = [self._dir, self._group_dir, self._tmpdir] self._reserved_paths = list(self._reserved_dirs) self._reserved_paths.append(self._driver_lock_path) self._safe_member_id = self._make_filesystem_safe(member_id) self._timeout = int(self._options.get('timeout', 10)) @staticmethod def _normalize_path(path): if sys.platform == 'win32': # Replace slashes with backslashes and make sure we don't # have any at the beginning of paths that include drive letters. # # Expected url format: # file:////share_address/share_name # file:///C:/path return re.sub(r'\\(?=\w:\\)', '', os.path.normpath(path)) return path @classmethod def _get_raw_lock(cls, path, member_id): lock_barrier = cls._barriers.setdefault(path, _Barrier()) return FileLock(path, lock_barrier, member_id) def get_lock(self, name): path = utils.safe_abs_path(self._dir, name.decode()) if path in self._reserved_paths: raise ValueError("Unable to create a lock using" " reserved path '%s' for lock" " with name '%s'" % (path, name)) return self._get_raw_lock(path, self._member_id) @classmethod def _make_filesystem_safe(cls, item): item = utils.to_binary(item, encoding="utf8") return hashlib.new(cls.HASH_ROUTINE, item).hexdigest() def _start(self): super(FileDriver, self)._start() for a_dir in self._reserved_dirs: try: fileutils.ensure_tree(a_dir) except OSError as e: raise coordination.ToozConnectionError(e) def _update_group_metadata(self, path, group_id): details = { u'group_id': utils.to_binary(group_id, encoding="utf8") } details[u'encoded'] = details[u"group_id"] != group_id details_blob = utils.dumps(details) fd, name = tempfile.mkstemp("tooz", dir=self._tmpdir) with os.fdopen(fd, "wb") as fh: fh.write(details_blob) os.rename(name, path) def create_group(self, group_id): safe_group_id = self._make_filesystem_safe(group_id) group_dir = os.path.join(self._group_dir, safe_group_id) group_meta_path = os.path.join(group_dir, '.metadata') def _do_create_group(): if os.path.exists(os.path.join(group_dir, ".metadata")): # NOTE(sileht): We update the group metadata even # they are already good, so ensure dict key are convert # to unicode in case of the file have been written with # tooz < 1.36 self._update_group_metadata(group_meta_path, group_id) raise coordination.GroupAlreadyExist(group_id) else: fileutils.ensure_tree(group_dir) self._update_group_metadata(group_meta_path, group_id) fut = self._executor.submit(_do_create_group) return FileFutureResult(fut) def join_group(self, group_id, capabilities=b""): safe_group_id = self._make_filesystem_safe(group_id) group_dir = os.path.join(self._group_dir, safe_group_id) me_path = os.path.join(group_dir, "%s.raw" % self._safe_member_id) @_lock_me(self._driver_lock) def _do_join_group(): if not os.path.exists(os.path.join(group_dir, ".metadata")): raise coordination.GroupNotCreated(group_id) if os.path.isfile(me_path): raise coordination.MemberAlreadyExist(group_id, self._member_id) details = { u'capabilities': capabilities, u'joined_on': datetime.datetime.now(), u'member_id': utils.to_binary(self._member_id, encoding="utf-8") } details[u'encoded'] = details[u"member_id"] != self._member_id details_blob = utils.dumps(details) with open(me_path, "wb") as fh: fh.write(details_blob) self._joined_groups.add(group_id) fut = self._executor.submit(_do_join_group) return FileFutureResult(fut) def leave_group(self, group_id): safe_group_id = self._make_filesystem_safe(group_id) group_dir = os.path.join(self._group_dir, safe_group_id) me_path = os.path.join(group_dir, "%s.raw" % self._safe_member_id) @_lock_me(self._driver_lock) def _do_leave_group(): if not os.path.exists(os.path.join(group_dir, ".metadata")): raise coordination.GroupNotCreated(group_id) try: os.unlink(me_path) except EnvironmentError as e: if e.errno != errno.ENOENT: raise else: raise coordination.MemberNotJoined(group_id, self._member_id) else: self._joined_groups.discard(group_id) fut = self._executor.submit(_do_leave_group) return FileFutureResult(fut) _SCHEMAS = { 'group': voluptuous.Schema({ voluptuous.Required('group_id'): voluptuous.Any(six.text_type, six.binary_type), # NOTE(sileht): tooz <1.36 was creating file without this voluptuous.Optional('encoded'): bool, }), 'member': voluptuous.Schema({ voluptuous.Required('member_id'): voluptuous.Any(six.text_type, six.binary_type), voluptuous.Required('joined_on'): datetime.datetime, # NOTE(sileht): tooz <1.36 was creating file without this voluptuous.Optional('encoded'): bool, }, extra=voluptuous.ALLOW_EXTRA), } def _load_and_validate(self, blob, schema_key): data = utils.loads(blob) data = _convert_from_old_format(data) schema = self._SCHEMAS[schema_key] return schema(data) def _read_member_id(self, path): with open(path, 'rb') as fh: details = self._load_and_validate(fh.read(), 'member') if details.get("encoded"): return details[u'member_id'].decode("utf-8") return details[u'member_id'] def get_members(self, group_id): safe_group_id = self._make_filesystem_safe(group_id) group_dir = os.path.join(self._group_dir, safe_group_id) @_lock_me(self._driver_lock) def _do_get_members(): if not os.path.isdir(group_dir): raise coordination.GroupNotCreated(group_id) members = set() try: entries = os.listdir(group_dir) except EnvironmentError as e: # Did someone manage to remove it before we got here... if e.errno != errno.ENOENT: raise else: for entry in entries: if not entry.endswith('.raw'): continue entry_path = os.path.join(group_dir, entry) try: m_time = datetime.datetime.fromtimestamp( os.stat(entry_path).st_mtime) current_time = datetime.datetime.now() delta_time = timeutils.delta_seconds(m_time, current_time) if delta_time >= 0 and delta_time <= self._timeout: member_id = self._read_member_id(entry_path) else: continue except EnvironmentError as e: if e.errno != errno.ENOENT: raise else: members.add(member_id) return members fut = self._executor.submit(_do_get_members) return FileFutureResult(fut) def get_member_capabilities(self, group_id, member_id): safe_group_id = self._make_filesystem_safe(group_id) group_dir = os.path.join(self._group_dir, safe_group_id) safe_member_id = self._make_filesystem_safe(member_id) member_path = os.path.join(group_dir, "%s.raw" % safe_member_id) @_lock_me(self._driver_lock) def _do_get_member_capabilities(): try: with open(member_path, "rb") as fh: contents = fh.read() except EnvironmentError as e: if e.errno == errno.ENOENT: if not os.path.isdir(group_dir): raise coordination.GroupNotCreated(group_id) else: raise coordination.MemberNotJoined(group_id, member_id) else: raise else: details = self._load_and_validate(contents, 'member') return details.get(u"capabilities") fut = self._executor.submit(_do_get_member_capabilities) return FileFutureResult(fut) def delete_group(self, group_id): safe_group_id = self._make_filesystem_safe(group_id) group_dir = os.path.join(self._group_dir, safe_group_id) @_lock_me(self._driver_lock) def _do_delete_group(): try: entries = os.listdir(group_dir) except EnvironmentError as e: if e.errno == errno.ENOENT: raise coordination.GroupNotCreated(group_id) else: raise else: if len(entries) > 1: raise coordination.GroupNotEmpty(group_id) elif len(entries) == 1 and entries != ['.metadata']: raise tooz.ToozError( "Unexpected path '%s' found in" " group directory '%s' (expected to only find" " a '.metadata' path)" % (entries[0], group_dir)) else: try: shutil.rmtree(group_dir) except EnvironmentError as e: if e.errno != errno.ENOENT: raise fut = self._executor.submit(_do_delete_group) return FileFutureResult(fut) def _read_group_id(self, path): with open(path, 'rb') as fh: details = self._load_and_validate(fh.read(), 'group') if details.get("encoded"): return details[u'group_id'].decode("utf-8") return details[u'group_id'] def get_groups(self): def _do_get_groups(): groups = [] for entry in os.listdir(self._group_dir): path = os.path.join(self._group_dir, entry, '.metadata') try: groups.append(self._read_group_id(path)) except EnvironmentError as e: if e.errno != errno.ENOENT: raise return groups fut = self._executor.submit(_do_get_groups) return FileFutureResult(fut) def heartbeat(self): for group_id in self._joined_groups: safe_group_id = self._make_filesystem_safe(group_id) group_dir = os.path.join(self._group_dir, safe_group_id) member_path = os.path.join(group_dir, "%s.raw" % self._safe_member_id) @_lock_me(self._driver_lock) def _do_heartbeat(): try: os.utime(member_path, None) except EnvironmentError as err: if err.errno != errno.ENOENT: raise _do_heartbeat() return self._timeout @staticmethod def watch_elected_as_leader(group_id, callback): raise tooz.NotImplemented @staticmethod def unwatch_elected_as_leader(group_id, callback): raise tooz.NotImplemented FileFutureResult = functools.partial(coordination.CoordinatorResult, failure_translator=_translate_failures) tooz-2.0.0/tooz/drivers/zake.py0000664000175000017500000000421613616633007016513 0ustar zuulzuul00000000000000# Copyright (c) 2013-2014 Mirantis Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. from __future__ import absolute_import from zake import fake_client from zake import fake_storage from tooz import coordination from tooz.drivers import zookeeper class ZakeDriver(zookeeper.KazooDriver): """This driver uses the `zake`_ client to mimic real `zookeeper`_ servers. It **should** be mainly used (and **is** really only intended to be used in this manner) for testing and integration (where real `zookeeper`_ servers are typically not available). .. _zake: https://pypi.org/project/zake .. _zookeeper: http://zookeeper.apache.org/ """ CHARACTERISTICS = ( coordination.Characteristics.NON_TIMEOUT_BASED, coordination.Characteristics.DISTRIBUTED_ACROSS_THREADS, ) """ Tuple of :py:class:`~tooz.coordination.Characteristics` introspectable enum member(s) that can be used to interogate how this driver works. """ # NOTE(harlowja): this creates a shared backend 'storage' layer that # would typically exist inside a zookeeper server, but since zake has # no concept of a 'real' zookeeper server we create a fake one and share # it among active clients to simulate zookeeper's consistent storage in # a thread-safe manner. fake_storage = fake_storage.FakeStorage( fake_client.k_threading.SequentialThreadingHandler()) @classmethod def _make_client(cls, parsed_url, options): if 'storage' in options: storage = options['storage'] else: storage = cls.fake_storage return fake_client.FakeClient(storage=storage) tooz-2.0.0/tooz/drivers/__init__.py0000664000175000017500000000000013616633007017303 0ustar zuulzuul00000000000000tooz-2.0.0/tooz/drivers/etcd.py0000664000175000017500000002224413616633007016501 0ustar zuulzuul00000000000000# -*- coding: utf-8 -*- # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import logging import threading import fasteners from oslo_utils import encodeutils from oslo_utils import timeutils import requests import six import tooz from tooz import coordination from tooz import locking from tooz import utils LOG = logging.getLogger(__name__) def _translate_failures(func): """Translates common requests exceptions into tooz exceptions.""" @six.wraps(func) def wrapper(*args, **kwargs): try: return func(*args, **kwargs) except ValueError as e: # Typically json decoding failed for some reason. utils.raise_with_cause(tooz.ToozError, encodeutils.exception_to_unicode(e), cause=e) except requests.exceptions.RequestException as e: utils.raise_with_cause(coordination.ToozConnectionError, encodeutils.exception_to_unicode(e), cause=e) return wrapper class _Client(object): def __init__(self, host, port, protocol): self.host = host self.port = port self.protocol = protocol self.session = requests.Session() @property def base_url(self): return self.protocol + '://' + self.host + ':' + str(self.port) def get_url(self, path): return self.base_url + '/v2/' + path.lstrip("/") def get(self, url, **kwargs): if kwargs.pop('make_url', True): url = self.get_url(url) return self.session.get(url, **kwargs).json() def put(self, url, **kwargs): if kwargs.pop('make_url', True): url = self.get_url(url) return self.session.put(url, **kwargs).json() def delete(self, url, **kwargs): if kwargs.pop('make_url', True): url = self.get_url(url) return self.session.delete(url, **kwargs).json() def self_stats(self): return self.session.get(self.get_url("/stats/self")) class EtcdLock(locking.Lock): _TOOZ_LOCK_PREFIX = "tooz_locks" def __init__(self, lock_url, name, coord, client, ttl=60): super(EtcdLock, self).__init__(name) self.client = client self.coord = coord self.ttl = ttl self._lock_url = lock_url self._node = None # NOTE(jschwarz): this lock is mainly used to prevent concurrent runs # of hearthbeat() with another function. For more details, see # https://bugs.launchpad.net/python-tooz/+bug/1603005. self._lock = threading.Lock() @_translate_failures @fasteners.locked def break_(self): reply = self.client.delete(self._lock_url, make_url=False) return reply.get('errorCode') is None def acquire(self, blocking=True, shared=False): if shared: raise tooz.NotImplemented blocking, timeout = utils.convert_blocking(blocking) if timeout is not None: watch = timeutils.StopWatch(duration=timeout) watch.start() else: watch = None while True: if self.acquired: # We already acquired the lock. Just go ahead and wait for ever # if blocking != False using the last index. lastindex = self._node['modifiedIndex'] else: try: reply = self.client.put( self._lock_url, make_url=False, timeout=watch.leftover() if watch else None, data={"ttl": self.ttl, "prevExist": "false"}) except requests.exceptions.RequestException: if not watch or watch.leftover() == 0: return False # We got the lock! if reply.get("errorCode") is None: with self._lock: self._node = reply['node'] self.coord._acquired_locks.add(self) return True # No lock, somebody got it, wait for it to be released lastindex = reply['index'] + 1 # We didn't get the lock and we don't want to wait if not blocking: return False # Ok, so let's wait a bit (or forever!) try: reply = self.client.get( self._lock_url + "?wait=true&waitIndex=%d" % lastindex, make_url=False, timeout=watch.leftover() if watch else None) except requests.exceptions.RequestException: if not watch or watch.expired(): return False @_translate_failures @fasteners.locked def release(self): if self.acquired: lock_url = self._lock_url lock_url += "?prevIndex=%s" % self._node['modifiedIndex'] reply = self.client.delete(lock_url, make_url=False) errorcode = reply.get("errorCode") if errorcode is None: self.coord._acquired_locks.discard(self) self._node = None return True else: LOG.warning("Unable to release '%s' due to %d, %s", self.name, errorcode, reply.get('message')) return False @property def acquired(self): return self in self.coord._acquired_locks @_translate_failures @fasteners.locked def heartbeat(self): """Keep the lock alive.""" if self.acquired: poked = self.client.put(self._lock_url, data={"ttl": self.ttl, "prevExist": "true"}, make_url=False) self._node = poked['node'] errorcode = poked.get("errorCode") if not errorcode: return True LOG.warning("Unable to heartbeat by updating key '%s' with " "extended expiry of %s seconds: %d, %s", self.name, self.ttl, errorcode, poked.get("message")) return False class EtcdDriver(coordination.CoordinationDriver): """An etcd based driver. This driver uses etcd provide the coordination driver semantics and required API(s). The Etcd driver connection URI should look like:: etcd://[HOST[:PORT]][?OPTION1=VALUE1[&OPTION2=VALUE2[&...]]] If not specified, HOST defaults to localhost and PORT defaults to 2379. Available options are: ================== ======= Name Default ================== ======= protocol http timeout 30 lock_timeout 30 ================== ======= """ #: Default socket/lock/member/leader timeout used when none is provided. DEFAULT_TIMEOUT = 30 #: Default hostname used when none is provided. DEFAULT_HOST = "localhost" #: Default port used if none provided (4001 or 2379 are the common ones). DEFAULT_PORT = 2379 #: Class that will be used to encode lock names into a valid etcd url. lock_encoder_cls = utils.Base64LockEncoder def __init__(self, member_id, parsed_url, options): super(EtcdDriver, self).__init__(member_id, parsed_url, options) host = parsed_url.hostname or self.DEFAULT_HOST port = parsed_url.port or self.DEFAULT_PORT options = utils.collapse(options) self.client = _Client(host=host, port=port, protocol=options.get('protocol', 'http')) default_timeout = options.get('timeout', self.DEFAULT_TIMEOUT) self.lock_encoder = self.lock_encoder_cls(self.client.get_url("keys")) self.lock_timeout = int(options.get('lock_timeout', default_timeout)) self._acquired_locks = set() def _start(self): try: self.client.self_stats() except requests.exceptions.ConnectionError as e: raise coordination.ToozConnectionError( encodeutils.exception_to_unicode(e)) def get_lock(self, name): return EtcdLock(self.lock_encoder.check_and_encode(name), name, self, self.client, self.lock_timeout) def heartbeat(self): for lock in self._acquired_locks.copy(): lock.heartbeat() return self.lock_timeout def watch_join_group(self, group_id, callback): raise tooz.NotImplemented def unwatch_join_group(self, group_id, callback): raise tooz.NotImplemented def watch_leave_group(self, group_id, callback): raise tooz.NotImplemented def unwatch_leave_group(self, group_id, callback): raise tooz.NotImplemented tooz-2.0.0/tooz/drivers/zookeeper.py0000664000175000017500000005667213616633007017601 0ustar zuulzuul00000000000000# -*- coding: utf-8 -*- # # Copyright (C) 2013-2014 eNovance Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. from kazoo import client from kazoo import exceptions from kazoo import security try: from kazoo.handlers import eventlet as eventlet_handler except ImportError: eventlet_handler = None from kazoo.handlers import threading as threading_handler from kazoo.protocol import paths from oslo_utils import encodeutils from oslo_utils import strutils import six from six.moves import filter as compat_filter import tooz from tooz import coordination from tooz import locking from tooz import utils class ZooKeeperLock(locking.Lock): def __init__(self, name, lock): super(ZooKeeperLock, self).__init__(name) self._lock = lock self._client = lock.client def is_still_owner(self): if not self.acquired: return False try: data, _znode = self._client.get( paths.join(self._lock.path, self._lock.node)) return data == self._lock.data except (self._client.handler.timeout_exception, exceptions.ConnectionLoss, exceptions.ConnectionDropped, exceptions.NoNodeError): return False except exceptions.KazooException as e: utils.raise_with_cause(tooz.ToozError, "operation error: %s" % (e), cause=e) def acquire(self, blocking=True, shared=False): if shared: raise tooz.NotImplemented blocking, timeout = utils.convert_blocking(blocking) return self._lock.acquire(blocking=blocking, timeout=timeout) def release(self): if self.acquired: self._lock.release() return True else: return False @property def acquired(self): return self._lock.is_acquired class KazooDriver(coordination.CoordinationDriverCachedRunWatchers): """This driver uses the `kazoo`_ client against real `zookeeper`_ servers. It **is** fully functional and implements all of the coordination driver API(s). It stores data into `zookeeper`_ using znodes and `msgpack`_ encoded values. To configure the client to your liking a subset of the options defined at http://kazoo.readthedocs.org/en/latest/api/client.html will be extracted from the coordinator url (or any provided options), so that a specific coordinator can be created that will work for you. The Zookeeper coordinator url should look like:: zookeeper://[USERNAME:PASSWORD@][HOST[:PORT]][?OPTION1=VALUE1[&OPTION2=VALUE2[&...]]] Currently the following options will be proxied to the contained client: ================ =============================== ==================== Name Source Default ================ =============================== ==================== hosts url netloc + 'hosts' option key localhost:2181 timeout 'timeout' options key 10.0 (kazoo default) connection_retry 'connection_retry' options key None command_retry 'command_retry' options key None randomize_hosts 'randomize_hosts' options key True ================ =============================== ==================== .. _kazoo: http://kazoo.readthedocs.org/ .. _zookeeper: http://zookeeper.apache.org/ .. _msgpack: http://msgpack.org/ """ #: Default namespace when none is provided. TOOZ_NAMESPACE = b"tooz" HANDLERS = { 'threading': threading_handler.SequentialThreadingHandler, } if eventlet_handler: HANDLERS['eventlet'] = eventlet_handler.SequentialEventletHandler """ Restricted immutable dict of handler 'kinds' -> handler classes that this driver can accept via 'handler' option key (the expected value for this option is one of the keys in this dictionary). """ CHARACTERISTICS = ( coordination.Characteristics.NON_TIMEOUT_BASED, coordination.Characteristics.DISTRIBUTED_ACROSS_THREADS, coordination.Characteristics.DISTRIBUTED_ACROSS_PROCESSES, coordination.Characteristics.DISTRIBUTED_ACROSS_HOSTS, # Writes *always* go through a single leader process, but it may # take a while for those writes to propagate to followers (and = # during this time clients can read older values)... coordination.Characteristics.SEQUENTIAL, ) """ Tuple of :py:class:`~tooz.coordination.Characteristics` introspectable enum member(s) that can be used to interogate how this driver works. """ def __init__(self, member_id, parsed_url, options): super(KazooDriver, self).__init__(member_id, parsed_url, options) options = utils.collapse(options, exclude=['hosts']) self.timeout = int(options.get('timeout', '10')) self._namespace = options.get('namespace', self.TOOZ_NAMESPACE) self._coord = self._make_client(parsed_url, options) self._timeout_exception = self._coord.handler.timeout_exception def _start(self): try: self._coord.start(timeout=self.timeout) except self._coord.handler.timeout_exception as e: e_msg = encodeutils.exception_to_unicode(e) utils.raise_with_cause(coordination.ToozConnectionError, "Operational error: %s" % e_msg, cause=e) try: self._coord.ensure_path(self._paths_join("/", self._namespace)) except exceptions.KazooException as e: e_msg = encodeutils.exception_to_unicode(e) utils.raise_with_cause(tooz.ToozError, "Operational error: %s" % e_msg, cause=e) self._leader_locks = {} def _stop(self): self._coord.stop() @staticmethod def _dumps(data): return utils.dumps(data) @staticmethod def _loads(blob): return utils.loads(blob) def _create_group_handler(self, async_result, timeout, timeout_exception, group_id): try: async_result.get(block=True, timeout=timeout) except timeout_exception as e: utils.raise_with_cause(coordination.OperationTimedOut, encodeutils.exception_to_unicode(e), cause=e) except exceptions.NodeExistsError: raise coordination.GroupAlreadyExist(group_id) except exceptions.NoNodeError as e: utils.raise_with_cause(tooz.ToozError, "Tooz namespace '%s' has not" " been created" % self._namespace, cause=e) except exceptions.ZookeeperError as e: utils.raise_with_cause(tooz.ToozError, encodeutils.exception_to_unicode(e), cause=e) def create_group(self, group_id): group_path = self._path_group(group_id) async_result = self._coord.create_async(group_path) return ZooAsyncResult(async_result, self._create_group_handler, timeout_exception=self._timeout_exception, group_id=group_id) @staticmethod def _delete_group_handler(async_result, timeout, timeout_exception, group_id): try: async_result.get(block=True, timeout=timeout) except timeout_exception as e: utils.raise_with_cause(coordination.OperationTimedOut, encodeutils.exception_to_unicode(e), cause=e) except exceptions.NoNodeError: raise coordination.GroupNotCreated(group_id) except exceptions.NotEmptyError: raise coordination.GroupNotEmpty(group_id) except exceptions.ZookeeperError as e: utils.raise_with_cause(tooz.ToozError, encodeutils.exception_to_unicode(e), cause=e) def delete_group(self, group_id): group_path = self._path_group(group_id) async_result = self._coord.delete_async(group_path) return ZooAsyncResult(async_result, self._delete_group_handler, timeout_exception=self._timeout_exception, group_id=group_id) @staticmethod def _join_group_handler(async_result, timeout, timeout_exception, group_id, member_id): try: async_result.get(block=True, timeout=timeout) except timeout_exception as e: utils.raise_with_cause(coordination.OperationTimedOut, encodeutils.exception_to_unicode(e), cause=e) except exceptions.NodeExistsError: raise coordination.MemberAlreadyExist(group_id, member_id) except exceptions.NoNodeError: raise coordination.GroupNotCreated(group_id) except exceptions.ZookeeperError as e: utils.raise_with_cause(tooz.ToozError, encodeutils.exception_to_unicode(e), cause=e) def join_group(self, group_id, capabilities=b""): member_path = self._path_member(group_id, self._member_id) capabilities = self._dumps(capabilities) async_result = self._coord.create_async(member_path, value=capabilities, ephemeral=True) return ZooAsyncResult(async_result, self._join_group_handler, timeout_exception=self._timeout_exception, group_id=group_id, member_id=self._member_id) @staticmethod def _leave_group_handler(async_result, timeout, timeout_exception, group_id, member_id): try: async_result.get(block=True, timeout=timeout) except timeout_exception as e: utils.raise_with_cause(coordination.OperationTimedOut, encodeutils.exception_to_unicode(e), cause=e) except exceptions.NoNodeError: raise coordination.MemberNotJoined(group_id, member_id) except exceptions.ZookeeperError as e: utils.raise_with_cause(tooz.ToozError, encodeutils.exception_to_unicode(e), cause=e) def heartbeat(self): # Just fetch the base path (and do nothing with it); this will # force any waiting heartbeat responses to be flushed, and also # ensures that the connection still works as expected... base_path = self._paths_join("/", self._namespace) try: self._coord.get(base_path) except self._timeout_exception as e: utils.raise_with_cause(coordination.OperationTimedOut, encodeutils.exception_to_unicode(e), cause=e) except exceptions.NoNodeError: pass except exceptions.ZookeeperError as e: utils.raise_with_cause(tooz.ToozError, encodeutils.exception_to_unicode(e), cause=e) return self.timeout def leave_group(self, group_id): member_path = self._path_member(group_id, self._member_id) async_result = self._coord.delete_async(member_path) return ZooAsyncResult(async_result, self._leave_group_handler, timeout_exception=self._timeout_exception, group_id=group_id, member_id=self._member_id) @staticmethod def _get_members_handler(async_result, timeout, timeout_exception, group_id): try: members_ids = async_result.get(block=True, timeout=timeout) except timeout_exception as e: utils.raise_with_cause(coordination.OperationTimedOut, encodeutils.exception_to_unicode(e), cause=e) except exceptions.NoNodeError: raise coordination.GroupNotCreated(group_id) except exceptions.ZookeeperError as e: utils.raise_with_cause(tooz.ToozError, encodeutils.exception_to_unicode(e), cause=e) else: return set(m.encode('ascii') for m in members_ids) def get_members(self, group_id): group_path = self._paths_join("/", self._namespace, group_id) async_result = self._coord.get_children_async(group_path) return ZooAsyncResult(async_result, self._get_members_handler, timeout_exception=self._timeout_exception, group_id=group_id) @staticmethod def _update_capabilities_handler(async_result, timeout, timeout_exception, group_id, member_id): try: async_result.get(block=True, timeout=timeout) except timeout_exception as e: utils.raise_with_cause(coordination.OperationTimedOut, encodeutils.exception_to_unicode(e), cause=e) except exceptions.NoNodeError: raise coordination.MemberNotJoined(group_id, member_id) except exceptions.ZookeeperError as e: utils.raise_with_cause(tooz.ToozError, encodeutils.exception_to_unicode(e), cause=e) def update_capabilities(self, group_id, capabilities): member_path = self._path_member(group_id, self._member_id) capabilities = self._dumps(capabilities) async_result = self._coord.set_async(member_path, capabilities) return ZooAsyncResult(async_result, self._update_capabilities_handler, timeout_exception=self._timeout_exception, group_id=group_id, member_id=self._member_id) @classmethod def _get_member_capabilities_handler(cls, async_result, timeout, timeout_exception, group_id, member_id): try: capabilities = async_result.get(block=True, timeout=timeout)[0] except timeout_exception as e: utils.raise_with_cause(coordination.OperationTimedOut, encodeutils.exception_to_unicode(e), cause=e) except exceptions.NoNodeError: raise coordination.MemberNotJoined(group_id, member_id) except exceptions.ZookeeperError as e: utils.raise_with_cause(tooz.ToozError, encodeutils.exception_to_unicode(e), cause=e) else: return cls._loads(capabilities) def get_member_capabilities(self, group_id, member_id): member_path = self._path_member(group_id, member_id) async_result = self._coord.get_async(member_path) return ZooAsyncResult(async_result, self._get_member_capabilities_handler, timeout_exception=self._timeout_exception, group_id=group_id, member_id=self._member_id) @classmethod def _get_member_info_handler(cls, async_result, timeout, timeout_exception, group_id, member_id): try: capabilities, znode_stats = async_result.get(block=True, timeout=timeout) except timeout_exception as e: utils.raise_with_cause(coordination.OperationTimedOut, encodeutils.exception_to_unicode(e), cause=e) except exceptions.NoNodeError: raise coordination.MemberNotJoined(group_id, member_id) except exceptions.ZookeeperError as e: utils.raise_with_cause(tooz.ToozError, encodeutils.exception_to_unicode(e), cause=e) else: member_info = { 'capabilities': cls._loads(capabilities), 'created_at': utils.millis_to_datetime(znode_stats.ctime), 'updated_at': utils.millis_to_datetime(znode_stats.mtime) } return member_info def get_member_info(self, group_id, member_id): member_path = self._path_member(group_id, member_id) async_result = self._coord.get_async(member_path) return ZooAsyncResult(async_result, self._get_member_info_handler, timeout_exception=self._timeout_exception, group_id=group_id, member_id=self._member_id) def _get_groups_handler(self, async_result, timeout, timeout_exception): try: group_ids = async_result.get(block=True, timeout=timeout) except timeout_exception as e: utils.raise_with_cause(coordination.OperationTimedOut, encodeutils.exception_to_unicode(e), cause=e) except exceptions.NoNodeError as e: utils.raise_with_cause(tooz.ToozError, "Tooz namespace '%s' has not" " been created" % self._namespace, cause=e) except exceptions.ZookeeperError as e: utils.raise_with_cause(tooz.ToozError, encodeutils.exception_to_unicode(e), cause=e) else: return set(g.encode('ascii') for g in group_ids) def get_groups(self): tooz_namespace = self._paths_join("/", self._namespace) async_result = self._coord.get_children_async(tooz_namespace) return ZooAsyncResult(async_result, self._get_groups_handler, timeout_exception=self._timeout_exception) def _path_group(self, group_id): return self._paths_join("/", self._namespace, group_id) def _path_member(self, group_id, member_id): return self._paths_join("/", self._namespace, group_id, member_id) @staticmethod def _paths_join(arg, *more_args): """Converts paths into a string (unicode).""" args = [arg] args.extend(more_args) cleaned_args = [] for arg in args: if isinstance(arg, six.binary_type): cleaned_args.append(arg.decode('ascii')) else: cleaned_args.append(arg) return paths.join(*cleaned_args) def _make_client(self, parsed_url, options): # Creates a kazoo client, # See: https://github.com/python-zk/kazoo/blob/2.2.1/kazoo/client.py # for what options a client takes... if parsed_url.username and parsed_url.password: username = parsed_url.username password = parsed_url.password digest_auth = "%s:%s" % (username, password) digest_acl = security.make_digest_acl(username, password, all=True) default_acl = (digest_acl,) auth_data = [('digest', digest_auth)] else: default_acl = None auth_data = None maybe_hosts = [parsed_url.netloc] + list(options.get('hosts', [])) hosts = list(compat_filter(None, maybe_hosts)) if not hosts: hosts = ['localhost:2181'] randomize_hosts = options.get('randomize_hosts', True) client_kwargs = { 'hosts': ",".join(hosts), 'timeout': float(options.get('timeout', self.timeout)), 'connection_retry': options.get('connection_retry'), 'command_retry': options.get('command_retry'), 'randomize_hosts': strutils.bool_from_string(randomize_hosts), 'auth_data': auth_data, 'default_acl': default_acl, } handler_kind = options.get('handler') if handler_kind: try: handler_cls = self.HANDLERS[handler_kind] except KeyError: raise ValueError("Unknown handler '%s' requested" " valid handlers are %s" % (handler_kind, sorted(self.HANDLERS.keys()))) client_kwargs['handler'] = handler_cls() return client.KazooClient(**client_kwargs) def stand_down_group_leader(self, group_id): if group_id in self._leader_locks: self._leader_locks[group_id].release() return True return False def _get_group_leader_lock(self, group_id): if group_id not in self._leader_locks: self._leader_locks[group_id] = self._coord.Lock( self._path_group(group_id) + "/leader", self._member_id.decode('ascii')) return self._leader_locks[group_id] def get_leader(self, group_id): contenders = self._get_group_leader_lock(group_id).contenders() if contenders and contenders[0]: leader = contenders[0].encode('ascii') else: leader = None return ZooAsyncResult(None, lambda *args: leader) def get_lock(self, name): z_lock = self._coord.Lock( self._paths_join(b"/", self._namespace, b"locks", name), self._member_id.decode('ascii')) return ZooKeeperLock(name, z_lock) def run_elect_coordinator(self): for group_id in six.iterkeys(self._hooks_elected_leader): leader_lock = self._get_group_leader_lock(group_id) if leader_lock.is_acquired: # Previously acquired/still leader, leave it be... continue if leader_lock.acquire(blocking=False): # We are now leader for this group self._hooks_elected_leader[group_id].run( coordination.LeaderElected( group_id, self._member_id)) def run_watchers(self, timeout=None): results = super(KazooDriver, self).run_watchers(timeout) self.run_elect_coordinator() return results class ZooAsyncResult(coordination.CoordAsyncResult): def __init__(self, kazoo_async_result, handler, **kwargs): self._kazoo_async_result = kazoo_async_result self._handler = handler self._kwargs = kwargs def get(self, timeout=None): return self._handler(self._kazoo_async_result, timeout, **self._kwargs) def done(self): return self._kazoo_async_result.ready() tooz-2.0.0/tooz/drivers/redis.py0000664000175000017500000007220613616633007016673 0ustar zuulzuul00000000000000# -*- coding: utf-8 -*- # Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. from __future__ import absolute_import import contextlib from distutils import version import functools import logging import string import threading from oslo_utils import encodeutils from oslo_utils import strutils import redis from redis import exceptions from redis import sentinel import six from six.moves import map as compat_map from six.moves import zip as compat_zip import tooz from tooz import coordination from tooz import locking from tooz import utils LOG = logging.getLogger(__name__) @contextlib.contextmanager def _translate_failures(): """Translates common redis exceptions into tooz exceptions.""" try: yield except (exceptions.ConnectionError, exceptions.TimeoutError) as e: utils.raise_with_cause(coordination.ToozConnectionError, encodeutils.exception_to_unicode(e), cause=e) except exceptions.RedisError as e: utils.raise_with_cause(tooz.ToozError, encodeutils.exception_to_unicode(e), cause=e) class RedisLock(locking.Lock): def __init__(self, coord, client, name, timeout): name = "%s_%s_lock" % (coord.namespace, six.text_type(name)) super(RedisLock, self).__init__(name) # NOTE(jd) Make sure we don't release and heartbeat at the same time by # using a exclusive access lock (LP#1557593) self._exclusive_access = threading.Lock() self._lock = client.lock(name, timeout=timeout, thread_local=False) self._coord = coord self._client = client def is_still_owner(self): with _translate_failures(): lock_tok = self._lock.local.token if not lock_tok: return False owner_tok = self._client.get(self.name) return owner_tok == lock_tok def break_(self): with _translate_failures(): return bool(self._client.delete(self.name)) def acquire(self, blocking=True, shared=False): if shared: raise tooz.NotImplemented blocking, timeout = utils.convert_blocking(blocking) with _translate_failures(): acquired = self._lock.acquire( blocking=blocking, blocking_timeout=timeout) if acquired: with self._exclusive_access: self._coord._acquired_locks.add(self) return acquired def release(self): with self._exclusive_access: with _translate_failures(): try: self._lock.release() except exceptions.LockError as e: LOG.error("Unable to release lock '%r': %s", self, e) return False finally: self._coord._acquired_locks.discard(self) return True def heartbeat(self): with self._exclusive_access: if self.acquired: with _translate_failures(): self._lock.extend(self._lock.timeout) return True return False @property def acquired(self): return self in self._coord._acquired_locks class RedisDriver(coordination.CoordinationDriverCachedRunWatchers, coordination.CoordinationDriverWithExecutor): """Redis provides a few nice benefits that act as a poormans zookeeper. It **is** fully functional and implements all of the coordination driver API(s). It stores data into `redis`_ using the provided `redis`_ API(s) using `msgpack`_ encoded values as needed. - Durability (when setup with `AOF`_ mode). - Consistent, note that this is still restricted to only one redis server, without the recently released redis (alpha) clustering > 1 server will not be consistent when partitions or failures occur (even redis clustering docs state it is not a fully AP or CP solution, which means even with it there will still be *potential* inconsistencies). - Master/slave failover (when setup with redis `sentinel`_), giving some notion of HA (values *can* be lost when a failover transition occurs). The Redis driver connection URI should look like:: redis://[:PASSWORD@]HOST:PORT[?OPTION=VALUE[&OPTION2=VALUE2[&...]]] For a list of options recognized by this driver, see the documentation for the member CLIENT_ARGS, and to determine the expected types of those options see CLIENT_BOOL_ARGS, CLIENT_INT_ARGS, and CLIENT_LIST_ARGS. To use a `sentinel`_ the connection URI must point to the sentinel server. At connection time the sentinel will be asked for the current IP and port of the master and then connect there. The connection URI for sentinel should be written as follows:: redis://:?sentinel= Additional sentinel hosts are listed with multiple ``sentinel_fallback`` parameters as follows:: redis://:?sentinel=& sentinel_fallback=:& sentinel_fallback=:& sentinel_fallback=: Further resources/links: - http://redis.io/ - http://redis.io/topics/sentinel - http://redis.io/topics/cluster-spec Note that this client will itself retry on transaction failure (when they keys being watched have changed underneath the current transaction). Currently the number of attempts that are tried is infinite (this might be addressed in https://github.com/andymccurdy/redis-py/issues/566 when that gets worked on). See http://redis.io/topics/transactions for more information on this topic. General recommendations/usage considerations: - When used for locks, run in AOF mode and think carefully about how your redis deployment handles losing a server (the clustering support is supposed to aid in losing servers, but it is also of unknown reliablity and is relatively new, so use at your own risk). .. _redis: http://redis.io/ .. _msgpack: http://msgpack.org/ .. _sentinel: http://redis.io/topics/sentinel .. _AOF: http://redis.io/topics/persistence """ CHARACTERISTICS = ( coordination.Characteristics.DISTRIBUTED_ACROSS_THREADS, coordination.Characteristics.DISTRIBUTED_ACROSS_PROCESSES, coordination.Characteristics.DISTRIBUTED_ACROSS_HOSTS, coordination.Characteristics.CAUSAL, ) """ Tuple of :py:class:`~tooz.coordination.Characteristics` introspectable enum member(s) that can be used to interogate how this driver works. """ MIN_VERSION = version.LooseVersion("2.6.0") """ The min redis version that this driver requires to operate with... """ GROUP_EXISTS = b'__created__' """ Redis deletes dictionaries that have no keys in them, which means the key will disappear which means we can't tell the difference between a group not existing and a group being empty without this key being saved... """ #: Value used (with group exists key) to keep a group from disappearing. GROUP_EXISTS_VALUE = b'1' #: Default namespace for keys when none is provided. DEFAULT_NAMESPACE = b'_tooz' NAMESPACE_SEP = b':' """ Separator that is used to combine a key with the namespace (to get the **actual** key that will be used). """ DEFAULT_ENCODING = 'utf8' """ This is for python3.x; which will behave differently when returned binary types or unicode types (redis uses binary internally it appears), so to just stick with a common way of doing this, make all the things binary (with this default encoding if one is not given and a unicode string is provided). """ CLIENT_ARGS = frozenset([ 'db', 'encoding', 'retry_on_timeout', 'socket_keepalive', 'socket_timeout', 'ssl', 'ssl_certfile', 'ssl_keyfile', 'sentinel', 'sentinel_fallback', ]) """ Keys that we allow to proxy from the coordinator configuration into the redis client (used to configure the redis client internals so that it works as you expect/want it to). See: http://redis-py.readthedocs.org/en/latest/#redis.Redis See: https://github.com/andymccurdy/redis-py/blob/2.10.3/redis/client.py """ #: Client arguments that are expected/allowed to be lists. CLIENT_LIST_ARGS = frozenset([ 'sentinel_fallback', ]) #: Client arguments that are expected to be boolean convertible. CLIENT_BOOL_ARGS = frozenset([ 'retry_on_timeout', 'ssl', ]) #: Client arguments that are expected to be int convertible. CLIENT_INT_ARGS = frozenset([ 'db', 'socket_keepalive', 'socket_timeout', ]) #: Default socket timeout to use when none is provided. CLIENT_DEFAULT_SOCKET_TO = 30 #: String used to keep a key/member alive (until it next expires). STILL_ALIVE = b"Not dead!" SCRIPTS = { 'create_group': """ -- Extract *all* the variables (so we can easily know what they are)... local namespaced_group_key = KEYS[1] local all_groups_key = KEYS[2] local no_namespaced_group_key = ARGV[1] if redis.call("exists", namespaced_group_key) == 1 then return 0 end redis.call("sadd", all_groups_key, no_namespaced_group_key) redis.call("hset", namespaced_group_key, "${group_existence_key}", "${group_existence_value}") return 1 """, 'delete_group': """ -- Extract *all* the variables (so we can easily know what they are)... local namespaced_group_key = KEYS[1] local all_groups_key = KEYS[2] local no_namespaced_group_key = ARGV[1] if redis.call("exists", namespaced_group_key) == 0 then return -1 end if redis.call("sismember", all_groups_key, no_namespaced_group_key) == 0 then return -2 end if redis.call("hlen", namespaced_group_key) > 1 then return -3 end -- First remove from the set (then delete the group); if the set removal -- fails, at least the group will still exist (and can be fixed manually)... if redis.call("srem", all_groups_key, no_namespaced_group_key) == 0 then return -4 end redis.call("del", namespaced_group_key) return 1 """, 'update_capabilities': """ -- Extract *all* the variables (so we can easily know what they are)... local group_key = KEYS[1] local member_id = ARGV[1] local caps = ARGV[2] if redis.call("exists", group_key) == 0 then return -1 end if redis.call("hexists", group_key, member_id) == 0 then return -2 end redis.call("hset", group_key, member_id, caps) return 1 """, } """`Lua`_ **template** scripts that will be used by various methods (they are turned into real scripts and loaded on call into the :func:`.start` method). .. _Lua: http://www.lua.org """ EXCLUDE_OPTIONS = CLIENT_LIST_ARGS def __init__(self, member_id, parsed_url, options): super(RedisDriver, self).__init__(member_id, parsed_url, options) self._parsed_url = parsed_url self._encoding = self._options.get('encoding', self.DEFAULT_ENCODING) timeout = self._options.get('timeout', self.CLIENT_DEFAULT_SOCKET_TO) self.timeout = int(timeout) self.membership_timeout = float(self._options.get( 'membership_timeout', timeout)) lock_timeout = self._options.get('lock_timeout', self.timeout) self.lock_timeout = int(lock_timeout) namespace = self._options.get('namespace', self.DEFAULT_NAMESPACE) self._namespace = utils.to_binary(namespace, encoding=self._encoding) self._group_prefix = self._namespace + b"_group" self._beat_prefix = self._namespace + b"_beats" self._groups = self._namespace + b"_groups" self._client = None self._acquired_locks = set() self._started = False self._server_info = {} self._scripts = {} def _check_fetch_redis_version(self, geq_version, not_existent=True): if isinstance(geq_version, six.string_types): desired_version = version.LooseVersion(geq_version) elif isinstance(geq_version, version.LooseVersion): desired_version = geq_version else: raise TypeError("Version check expects a string/version type") try: redis_version = version.LooseVersion( self._server_info['redis_version']) except KeyError: return (not_existent, None) else: if redis_version < desired_version: return (False, redis_version) else: return (True, redis_version) @property def namespace(self): return self._namespace @property def running(self): return self._started def get_lock(self, name): return RedisLock(self, self._client, name, self.lock_timeout) _dumps = staticmethod(utils.dumps) _loads = staticmethod(utils.loads) @classmethod def _make_client(cls, parsed_url, options, default_socket_timeout): kwargs = {} if parsed_url.hostname: kwargs['host'] = parsed_url.hostname if parsed_url.port: kwargs['port'] = parsed_url.port else: if not parsed_url.path: raise ValueError("Expected socket path in parsed urls path") kwargs['unix_socket_path'] = parsed_url.path if parsed_url.password: kwargs['password'] = parsed_url.password for a in cls.CLIENT_ARGS: if a not in options: continue if a in cls.CLIENT_BOOL_ARGS: v = strutils.bool_from_string(options[a]) elif a in cls.CLIENT_LIST_ARGS: v = options[a] elif a in cls.CLIENT_INT_ARGS: v = int(options[a]) else: v = options[a] kwargs[a] = v if 'socket_timeout' not in kwargs: kwargs['socket_timeout'] = default_socket_timeout # Ask the sentinel for the current master if there is a # sentinel arg. if 'sentinel' in kwargs: sentinel_hosts = [ tuple(fallback.split(':')) for fallback in kwargs.get('sentinel_fallback', []) ] sentinel_hosts.insert(0, (kwargs['host'], kwargs['port'])) sentinel_server = sentinel.Sentinel( sentinel_hosts, socket_timeout=kwargs['socket_timeout']) sentinel_name = kwargs['sentinel'] del kwargs['sentinel'] if 'sentinel_fallback' in kwargs: del kwargs['sentinel_fallback'] master_client = sentinel_server.master_for(sentinel_name, **kwargs) # The master_client is a redis.StrictRedis using a # Sentinel managed connection pool. return master_client return redis.StrictRedis(**kwargs) def _start(self): super(RedisDriver, self)._start() try: self._client = self._make_client(self._parsed_url, self._options, self.timeout) except exceptions.RedisError as e: utils.raise_with_cause(coordination.ToozConnectionError, encodeutils.exception_to_unicode(e), cause=e) else: # Ensure that the server is alive and not dead, this does not # ensure the server will always be alive, but does insure that it # at least is alive once... with _translate_failures(): self._server_info = self._client.info() # Validate we have a good enough redis version we are connected # to so that the basic set of features we support will actually # work (instead of blowing up). new_enough, redis_version = self._check_fetch_redis_version( self.MIN_VERSION) if not new_enough: raise tooz.NotImplemented("Redis version greater than or" " equal to '%s' is required" " to use this driver; '%s' is" " being used which is not new" " enough" % (self.MIN_VERSION, redis_version)) tpl_params = { 'group_existence_value': self.GROUP_EXISTS_VALUE, 'group_existence_key': self.GROUP_EXISTS, } # For py3.x ensure these are unicode since the string template # replacement will expect unicode (and we don't want b'' as a # prefix which will happen in py3.x if this is not done). for (k, v) in six.iteritems(tpl_params.copy()): if isinstance(v, six.binary_type): v = v.decode('ascii') tpl_params[k] = v prepared_scripts = {} for name, raw_script_tpl in six.iteritems(self.SCRIPTS): script_tpl = string.Template(raw_script_tpl) script = script_tpl.substitute(**tpl_params) prepared_scripts[name] = self._client.register_script(script) self._scripts = prepared_scripts self.heartbeat() self._started = True def _encode_beat_id(self, member_id): member_id = utils.to_binary(member_id, encoding=self._encoding) return self.NAMESPACE_SEP.join([self._beat_prefix, member_id]) def _encode_member_id(self, member_id): member_id = utils.to_binary(member_id, encoding=self._encoding) if member_id == self.GROUP_EXISTS: raise ValueError("Not allowed to use private keys as a member id") return member_id def _decode_member_id(self, member_id): return utils.to_binary(member_id, encoding=self._encoding) def _encode_group_leader(self, group_id): group_id = utils.to_binary(group_id, encoding=self._encoding) return b"leader_of_" + group_id def _encode_group_id(self, group_id, apply_namespace=True): group_id = utils.to_binary(group_id, encoding=self._encoding) if not apply_namespace: return group_id return self.NAMESPACE_SEP.join([self._group_prefix, group_id]) def _decode_group_id(self, group_id): return utils.to_binary(group_id, encoding=self._encoding) def heartbeat(self): with _translate_failures(): beat_id = self._encode_beat_id(self._member_id) expiry_ms = max(0, int(self.membership_timeout * 1000.0)) self._client.psetex(beat_id, time_ms=expiry_ms, value=self.STILL_ALIVE) for lock in self._acquired_locks.copy(): try: lock.heartbeat() except tooz.ToozError: LOG.warning("Unable to heartbeat lock '%s'", lock, exc_info=True) return min(self.lock_timeout, self.membership_timeout) def _stop(self): while self._acquired_locks: lock = self._acquired_locks.pop() try: lock.release() except tooz.ToozError: LOG.warning("Unable to release lock '%s'", lock, exc_info=True) super(RedisDriver, self)._stop() if self._client is not None: # Make sure we no longer exist... beat_id = self._encode_beat_id(self._member_id) try: # NOTE(harlowja): this will delete nothing if the key doesn't # exist in the first place, which is fine/expected/desired... with _translate_failures(): self._client.delete(beat_id) except tooz.ToozError: LOG.warning("Unable to delete heartbeat key '%s'", beat_id, exc_info=True) self._client = None self._server_info = {} self._scripts.clear() self._started = False def _submit(self, cb, *args, **kwargs): if not self._started: raise tooz.ToozError("Redis driver has not been started") return self._executor.submit(cb, *args, **kwargs) def _get_script(self, script_key): try: return self._scripts[script_key] except KeyError: raise tooz.ToozError("Redis driver has not been started") def create_group(self, group_id): script = self._get_script('create_group') def _create_group(script): encoded_group = self._encode_group_id(group_id) keys = [ encoded_group, self._groups, ] args = [ self._encode_group_id(group_id, apply_namespace=False), ] result = script(keys=keys, args=args) result = strutils.bool_from_string(result) if not result: raise coordination.GroupAlreadyExist(group_id) return RedisFutureResult(self._submit(_create_group, script)) def update_capabilities(self, group_id, capabilities): script = self._get_script('update_capabilities') def _update_capabilities(script): keys = [ self._encode_group_id(group_id), ] args = [ self._encode_member_id(self._member_id), self._dumps(capabilities), ] result = int(script(keys=keys, args=args)) if result == -1: raise coordination.GroupNotCreated(group_id) if result == -2: raise coordination.MemberNotJoined(group_id, self._member_id) return RedisFutureResult(self._submit(_update_capabilities, script)) def leave_group(self, group_id): encoded_group = self._encode_group_id(group_id) encoded_member_id = self._encode_member_id(self._member_id) def _leave_group(p): if not p.exists(encoded_group): raise coordination.GroupNotCreated(group_id) p.multi() p.hdel(encoded_group, encoded_member_id) c = p.execute()[0] if c == 0: raise coordination.MemberNotJoined(group_id, self._member_id) else: self._joined_groups.discard(group_id) return RedisFutureResult(self._submit(self._client.transaction, _leave_group, encoded_group, value_from_callable=True)) def get_members(self, group_id): encoded_group = self._encode_group_id(group_id) def _get_members(p): if not p.exists(encoded_group): raise coordination.GroupNotCreated(group_id) potential_members = set() for m in p.hkeys(encoded_group): m = self._decode_member_id(m) if m != self.GROUP_EXISTS: potential_members.add(m) if not potential_members: return set() # Ok now we need to see which members have passed away... gone_members = set() member_values = p.mget(compat_map(self._encode_beat_id, potential_members)) for (potential_member, value) in compat_zip(potential_members, member_values): # Always preserve self (just incase we haven't heartbeated # while this call/s was being made...), this does *not* prevent # another client from removing this though... if potential_member == self._member_id: continue if not value: gone_members.add(potential_member) # Trash all the members that no longer are with us... RIP... if gone_members: p.multi() encoded_gone_members = list(self._encode_member_id(m) for m in gone_members) p.hdel(encoded_group, *encoded_gone_members) p.execute() return set(m for m in potential_members if m not in gone_members) return potential_members return RedisFutureResult(self._submit(self._client.transaction, _get_members, encoded_group, value_from_callable=True)) def get_member_capabilities(self, group_id, member_id): encoded_group = self._encode_group_id(group_id) encoded_member_id = self._encode_member_id(member_id) def _get_member_capabilities(p): if not p.exists(encoded_group): raise coordination.GroupNotCreated(group_id) capabilities = p.hget(encoded_group, encoded_member_id) if capabilities is None: raise coordination.MemberNotJoined(group_id, member_id) return self._loads(capabilities) return RedisFutureResult(self._submit(self._client.transaction, _get_member_capabilities, encoded_group, value_from_callable=True)) def join_group(self, group_id, capabilities=b""): encoded_group = self._encode_group_id(group_id) encoded_member_id = self._encode_member_id(self._member_id) def _join_group(p): if not p.exists(encoded_group): raise coordination.GroupNotCreated(group_id) p.multi() p.hset(encoded_group, encoded_member_id, self._dumps(capabilities)) c = p.execute()[0] if c == 0: # Field already exists... raise coordination.MemberAlreadyExist(group_id, self._member_id) else: self._joined_groups.add(group_id) return RedisFutureResult(self._submit(self._client.transaction, _join_group, encoded_group, value_from_callable=True)) def delete_group(self, group_id): script = self._get_script('delete_group') def _delete_group(script): keys = [ self._encode_group_id(group_id), self._groups, ] args = [ self._encode_group_id(group_id, apply_namespace=False), ] result = int(script(keys=keys, args=args)) if result in (-1, -2): raise coordination.GroupNotCreated(group_id) if result == -3: raise coordination.GroupNotEmpty(group_id) if result == -4: raise tooz.ToozError("Unable to remove '%s' key" " from set located at '%s'" % (args[0], keys[-1])) if result != 1: raise tooz.ToozError("Internal error, unable" " to complete group '%s' removal" % (group_id)) return RedisFutureResult(self._submit(_delete_group, script)) def _destroy_group(self, group_id): """Should only be used in tests...""" self._client.delete(self._encode_group_id(group_id)) def get_groups(self): def _get_groups(): results = [] for g in self._client.smembers(self._groups): results.append(self._decode_group_id(g)) return results return RedisFutureResult(self._submit(_get_groups)) def _get_leader_lock(self, group_id): name = self._encode_group_leader(group_id) return self.get_lock(name) def run_elect_coordinator(self): for group_id, hooks in six.iteritems(self._hooks_elected_leader): leader_lock = self._get_leader_lock(group_id) if leader_lock.acquire(blocking=False): # We got the lock hooks.run(coordination.LeaderElected(group_id, self._member_id)) def run_watchers(self, timeout=None): result = super(RedisDriver, self).run_watchers(timeout=timeout) self.run_elect_coordinator() return result RedisFutureResult = functools.partial(coordination.CoordinatorResult, failure_translator=_translate_failures) tooz-2.0.0/tooz/drivers/consul.py0000664000175000017500000001430313616633007017062 0ustar zuulzuul00000000000000# -*- coding: utf-8 -*- # # Copyright © 2015 Yahoo! Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. from __future__ import absolute_import import consul from oslo_utils import encodeutils import tooz from tooz import _retry from tooz import coordination from tooz import locking from tooz import utils class ConsulLock(locking.Lock): def __init__(self, name, node, address, session_id, client): super(ConsulLock, self).__init__(name) self._name = name self._node = node self._address = address self._session_id = session_id self._client = client self.acquired = False def acquire(self, blocking=True, shared=False): if shared: raise tooz.NotImplemented @_retry.retry(stop_max_delay=blocking) def _acquire(): # Check if we are the owner and if we are simulate # blocking (because consul will not block a second # acquisition attempt by the same owner). _index, value = self._client.kv.get(key=self._name) if value and value.get('Session') == self._session_id: if blocking is False: return False else: raise _retry.TryAgain else: # The value can be anything. gotten = self._client.kv.put(key=self._name, value=u"I got it!", acquire=self._session_id) if gotten: self.acquired = True return True if blocking is False: return False else: raise _retry.TryAgain return _acquire() def release(self): if not self.acquired: return False # Get the lock to verify the session ID's are same _index, contents = self._client.kv.get(key=self._name) if not contents: return False owner = contents.get('Session') if owner == self._session_id: removed = self._client.kv.put(key=self._name, value=self._session_id, release=self._session_id) if removed: self.acquired = False return True return False class ConsulDriver(coordination.CoordinationDriver): """This driver uses `python-consul`_ client against `consul`_ servers. The ConsulDriver implements a minimal set of coordination driver API(s) needed to make Consul being used as an option for Distributed Locking. The data is stored in Consul's key-value store. The Consul driver connection URI should look like:: consul://HOST[:PORT][?OPTION1=VALUE1[&OPTION2=VALUE2[&...]]] If not specified, PORT defaults to 8500. Available options are: ================== ======= Name Default ================== ======= ttl 15 namespace tooz ================== ======= For details on the available options, refer to http://python-consul.readthedocs.org/en/latest/. .. _python-consul: http://python-consul.readthedocs.org/ .. _consul: https://consul.io/ """ #: Default namespace when none is provided TOOZ_NAMESPACE = u"tooz" #: Default TTL DEFAULT_TTL = 15 #: Default consul port if not provided. DEFAULT_PORT = 8500 def __init__(self, member_id, parsed_url, options): super(ConsulDriver, self).__init__(member_id, parsed_url, options) options = utils.collapse(options) self._host = parsed_url.hostname self._port = parsed_url.port or self.DEFAULT_PORT self._session_id = None self._session_name = encodeutils.safe_decode(member_id) self._ttl = int(options.get('ttl', self.DEFAULT_TTL)) namespace = options.get('namespace', self.TOOZ_NAMESPACE) self._namespace = encodeutils.safe_decode(namespace) self._client = None def _start(self): """Create a client, register a node and create a session.""" # Create a consul client if self._client is None: self._client = consul.Consul(host=self._host, port=self._port) local_agent = self._client.agent.self() self._node = local_agent['Member']['Name'] self._address = local_agent['Member']['Addr'] # Register a Node self._client.catalog.register(node=self._node, address=self._address) # Create a session self._session_id = self._client.session.create( name=self._session_name, node=self._node, ttl=self._ttl) def _stop(self): if self._client is not None: if self._session_id is not None: self._client.session.destroy(self._session_id) self._session_id = None self._client = None def get_lock(self, name): real_name = self._paths_join(self._namespace, u"locks", name) return ConsulLock(real_name, self._node, self._address, session_id=self._session_id, client=self._client) @staticmethod def _paths_join(*args): pieces = [] for arg in args: pieces.append(encodeutils.safe_decode(arg)) return u"/".join(pieces) def watch_join_group(self, group_id, callback): raise tooz.NotImplemented def unwatch_join_group(self, group_id, callback): raise tooz.NotImplemented def watch_leave_group(self, group_id, callback): raise tooz.NotImplemented def unwatch_leave_group(self, group_id, callback): raise tooz.NotImplemented tooz-2.0.0/tooz/drivers/ipc.py0000664000175000017500000002043313616633007016333 0ustar zuulzuul00000000000000# -*- coding: utf-8 -*- # # Copyright © 2014 eNovance # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import hashlib import struct import time import msgpack import six import sysv_ipc import tooz from tooz import coordination from tooz import locking from tooz import utils if sysv_ipc.KEY_MIN <= 0: _KEY_RANGE = abs(sysv_ipc.KEY_MIN) + sysv_ipc.KEY_MAX else: _KEY_RANGE = sysv_ipc.KEY_MAX - sysv_ipc.KEY_MIN def ftok(name, project): # Similar to ftok & http://semanchuk.com/philip/sysv_ipc/#ftok_weakness # but hopefully without as many weaknesses... h = hashlib.md5() if not isinstance(project, six.binary_type): project = project.encode('ascii') h.update(project) if not isinstance(name, six.binary_type): name = name.encode('ascii') h.update(name) return (int(h.hexdigest(), 16) % _KEY_RANGE) + sysv_ipc.KEY_MIN class IPCLock(locking.Lock): """A sysv IPC based lock. Please ensure you have read over (and understand) the limitations of sysv IPC locks, and especially have tried and used $ ipcs -l (note the maximum number of semaphores system wide field that command outputs). To ensure that you do not reach that limit it is recommended to use destroy() at the correct program exit/entry points. """ _LOCK_PROJECT = b'__TOOZ_LOCK_' def __init__(self, name): super(IPCLock, self).__init__(name) self.key = ftok(name, self._LOCK_PROJECT) self._lock = None def break_(self): try: lock = sysv_ipc.Semaphore(key=self.key) lock.remove() except sysv_ipc.ExistentialError: return False else: return True def acquire(self, blocking=True, shared=False): if shared: raise tooz.NotImplemented if (blocking is not True and sysv_ipc.SEMAPHORE_TIMEOUT_SUPPORTED is False): raise tooz.NotImplemented("This system does not support" " semaphore timeouts") blocking, timeout = utils.convert_blocking(blocking) start_time = None if not blocking: timeout = 0 elif blocking and timeout is not None: start_time = time.time() while True: tmplock = None try: tmplock = sysv_ipc.Semaphore(self.key, flags=sysv_ipc.IPC_CREX, initial_value=1) tmplock.undo = True except sysv_ipc.ExistentialError: # We failed to create it because it already exists, then try to # grab the existing one. try: tmplock = sysv_ipc.Semaphore(self.key) tmplock.undo = True except sysv_ipc.ExistentialError: # Semaphore has been deleted in the mean time, retry from # the beginning! continue if start_time is not None: elapsed = max(0.0, time.time() - start_time) if elapsed >= timeout: # Ran out of time... return False adjusted_timeout = timeout - elapsed else: adjusted_timeout = timeout try: tmplock.acquire(timeout=adjusted_timeout) except sysv_ipc.BusyError: tmplock = None return False except sysv_ipc.ExistentialError: # Likely the lock has been deleted in the meantime, retry continue else: self._lock = tmplock return True def release(self): if self._lock is not None: try: self._lock.remove() self._lock = None except sysv_ipc.ExistentialError: return False return True return False class IPCDriver(coordination.CoordinationDriverWithExecutor): """A `IPC`_ based driver. This driver uses `IPC`_ concepts to provide the coordination driver semantics and required API(s). It **is** missing some functionality but in the future these not implemented API(s) will be filled in. The IPC driver connection URI should look like:: ipc:// General recommendations/usage considerations: - It is **not** distributed (or recommended to be used in those situations, so the developer using this should really take that into account when applying this driver in there app). .. _IPC: http://en.wikipedia.org/wiki/Inter-process_communication """ CHARACTERISTICS = ( coordination.Characteristics.NON_TIMEOUT_BASED, coordination.Characteristics.DISTRIBUTED_ACROSS_THREADS, coordination.Characteristics.DISTRIBUTED_ACROSS_PROCESSES, ) """ Tuple of :py:class:`~tooz.coordination.Characteristics` introspectable enum member(s) that can be used to interogate how this driver works. """ _SEGMENT_SIZE = 1024 _GROUP_LIST_KEY = "GROUP_LIST" _GROUP_PROJECT = "_TOOZ_INTERNAL" _INTERNAL_LOCK_NAME = "TOOZ_INTERNAL_LOCK" def _start(self): super(IPCDriver, self)._start() self._group_list = sysv_ipc.SharedMemory( ftok(self._GROUP_LIST_KEY, self._GROUP_PROJECT), sysv_ipc.IPC_CREAT, size=self._SEGMENT_SIZE) self._lock = self.get_lock(self._INTERNAL_LOCK_NAME) def _stop(self): super(IPCDriver, self)._stop() try: self._group_list.detach() self._group_list.remove() except sysv_ipc.ExistentialError: pass def _read_group_list(self): data = self._group_list.read(byte_count=2) length = struct.unpack("H", data)[0] if length == 0: return set() data = self._group_list.read(byte_count=length, offset=2) return set(msgpack.loads(data)) def _write_group_list(self, group_list): data = msgpack.dumps(list(group_list)) if len(data) >= self._SEGMENT_SIZE - 2: raise tooz.ToozError("Group list is too big") self._group_list.write(struct.pack('H', len(data))) self._group_list.write(data, offset=2) def create_group(self, group_id): def _create_group(): with self._lock: group_list = self._read_group_list() if group_id in group_list: raise coordination.GroupAlreadyExist(group_id) group_list.add(group_id) self._write_group_list(group_list) return coordination.CoordinatorResult( self._executor.submit(_create_group)) def delete_group(self, group_id): def _delete_group(): with self._lock: group_list = self._read_group_list() if group_id not in group_list: raise coordination.GroupNotCreated(group_id) group_list.remove(group_id) self._write_group_list(group_list) return coordination.CoordinatorResult( self._executor.submit(_delete_group)) def watch_join_group(self, group_id, callback): # Check the group exist self.get_members(group_id).get() super(IPCDriver, self).watch_join_group(group_id, callback) def watch_leave_group(self, group_id, callback): # Check the group exist self.get_members(group_id).get() super(IPCDriver, self).watch_leave_group(group_id, callback) def _get_groups_handler(self): with self._lock: return self._read_group_list() def get_groups(self): return coordination.CoordinatorResult(self._executor.submit( self._get_groups_handler)) @staticmethod def get_lock(name): return IPCLock(name) tooz-2.0.0/tooz/__init__.py0000664000175000017500000000241513616633007015641 0ustar zuulzuul00000000000000# -*- coding: utf-8 -*- # # Copyright (C) 2014 eNovance Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. class ToozError(Exception): """Exception raised when an internal error occurs. Raised for instance in case of server internal error. :ivar cause: the cause of the exception being raised, when not none this will itself be an exception instance, this is useful for creating a chain of exceptions for versions of python where this is not yet implemented/supported natively. """ def __init__(self, message, cause=None): super(ToozError, self).__init__(message) self.cause = cause class NotImplemented(NotImplementedError, ToozError): pass tooz-2.0.0/tooz/utils.py0000664000175000017500000001746113616633007015251 0ustar zuulzuul00000000000000# -*- coding: utf-8 -*- # Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import base64 import datetime import operator import os import futurist import msgpack from oslo_serialization import msgpackutils from oslo_utils import encodeutils from oslo_utils import excutils import six import tooz class Base64LockEncoder(object): def __init__(self, keyspace_url, prefix=''): self.keyspace_url = keyspace_url if prefix: self.keyspace_url += prefix def check_and_encode(self, name): if not isinstance(name, (six.text_type, six.binary_type)): raise TypeError("Provided lock name is expected to be a string" " or binary type and not %s" % type(name)) try: return self.encode(name) except (UnicodeDecodeError, UnicodeEncodeError) as e: raise ValueError("Invalid lock name due to encoding/decoding " " issue: %s" % encodeutils.exception_to_unicode(e)) def encode(self, name): if isinstance(name, six.text_type): name = name.encode("ascii") enc_name = base64.urlsafe_b64encode(name) return self.keyspace_url + "/" + enc_name.decode("ascii") class ProxyExecutor(object): KIND_TO_FACTORY = { 'threaded': (lambda: futurist.ThreadPoolExecutor(max_workers=1)), 'synchronous': lambda: futurist.SynchronousExecutor(), } # Provide a few common aliases... KIND_TO_FACTORY['thread'] = KIND_TO_FACTORY['threaded'] KIND_TO_FACTORY['threading'] = KIND_TO_FACTORY['threaded'] KIND_TO_FACTORY['sync'] = KIND_TO_FACTORY['synchronous'] DEFAULT_KIND = 'threaded' def __init__(self, driver_name, default_executor_factory): self.default_executor_factory = default_executor_factory self.driver_name = driver_name self.started = False self.executor = None self.internally_owned = True @classmethod def build(cls, driver_name, options): default_executor_fact = cls.KIND_TO_FACTORY[cls.DEFAULT_KIND] if 'executor' in options: executor_kind = options['executor'] try: default_executor_fact = cls.KIND_TO_FACTORY[executor_kind] except KeyError: executors_known = sorted(list(cls.KIND_TO_FACTORY)) raise tooz.ToozError("Unknown executor" " '%s' provided, accepted values" " are %s" % (executor_kind, executors_known)) return cls(driver_name, default_executor_fact) def start(self): if self.started: return self.executor = self.default_executor_factory() self.started = True def stop(self): executor = self.executor self.executor = None if executor is not None: executor.shutdown() self.started = False def submit(self, cb, *args, **kwargs): if not self.started: raise tooz.ToozError("%s driver asynchronous executor" " has not been started" % self.driver_name) try: return self.executor.submit(cb, *args, **kwargs) except RuntimeError: raise tooz.ToozError("%s driver asynchronous executor has" " been shutdown" % self.driver_name) def safe_abs_path(rooted_at, *pieces): # Avoids the following junk... # # >>> import os # >>> os.path.join("/b", "..") # '/b/..' # >>> os.path.abspath(os.path.join("/b", "..")) # '/' path = os.path.abspath(os.path.join(rooted_at, *pieces)) if not path.startswith(rooted_at): raise ValueError("Unable to create path that is outside of" " parent directory '%s' using segments %s" % (rooted_at, list(pieces))) return path def convert_blocking(blocking): """Converts a multi-type blocking variable into its derivatives.""" timeout = None if not isinstance(blocking, bool): timeout = float(blocking) blocking = True return blocking, timeout def collapse(config, exclude=None, item_selector=operator.itemgetter(-1)): """Collapses config with keys and **list/tuple** values. NOTE(harlowja): The last item/index from the list/tuple value is selected be default as the new value (values that are not lists/tuples are left alone). If the list/tuple value is empty (zero length), then no value is set. """ if not isinstance(config, dict): raise TypeError("Unexpected config type, dict expected") if not config: return {} if exclude is None: exclude = set() collapsed = {} for (k, v) in six.iteritems(config): if isinstance(v, (tuple, list)): if k in exclude: collapsed[k] = v else: if len(v): collapsed[k] = item_selector(v) else: collapsed[k] = v return collapsed def to_binary(text, encoding='ascii'): """Return the binary representation of string (if not already binary).""" if not isinstance(text, six.binary_type): text = text.encode(encoding) return text class SerializationError(tooz.ToozError): "Exception raised when serialization or deserialization breaks." def dumps(data, excp_cls=SerializationError): """Serializes provided data using msgpack into a byte string.""" try: return msgpackutils.dumps(data) except (msgpack.PackException, ValueError) as e: raise_with_cause(excp_cls, encodeutils.exception_to_unicode(e), cause=e) def loads(blob, excp_cls=SerializationError): """Deserializes provided data using msgpack (from a prior byte string).""" try: return msgpackutils.loads(blob) except (msgpack.UnpackException, ValueError) as e: raise_with_cause(excp_cls, encodeutils.exception_to_unicode(e), cause=e) def millis_to_datetime(milliseconds): """Converts number of milliseconds (from epoch) into a datetime object.""" return datetime.datetime.fromtimestamp(float(milliseconds) / 1000) def raise_with_cause(exc_cls, message, *args, **kwargs): """Helper to raise + chain exceptions (when able) and associate a *cause*. **For internal usage only.** NOTE(harlowja): Since in py3.x exceptions can be chained (due to :pep:`3134`) we should try to raise the desired exception with the given *cause*. :param exc_cls: the :py:class:`~tooz.ToozError` class to raise. :param message: the text/str message that will be passed to the exceptions constructor as its first positional argument. :param args: any additional positional arguments to pass to the exceptions constructor. :param kwargs: any additional keyword arguments to pass to the exceptions constructor. """ if not issubclass(exc_cls, tooz.ToozError): raise ValueError("Subclass of tooz error is required") excutils.raise_with_cause(exc_cls, message, *args, **kwargs) tooz-2.0.0/examples/0000775000175000017500000000000013616633265014357 5ustar zuulzuul00000000000000tooz-2.0.0/examples/group_membership_watch.py0000664000175000017500000000075313616633007021465 0ustar zuulzuul00000000000000import uuid import six from tooz import coordination coordinator = coordination.get_coordinator('zake://', b'host-1') coordinator.start() # Create a group group = six.binary_type(six.text_type(uuid.uuid4()).encode('ascii')) request = coordinator.create_group(group) request.get() def group_joined(event): # Event is an instance of tooz.coordination.MemberJoinedGroup print(event.group_id, event.member_id) coordinator.watch_join_group(group, group_joined) coordinator.stop() tooz-2.0.0/examples/coordinator.py0000664000175000017500000000020713616633007017245 0ustar zuulzuul00000000000000from tooz import coordination coordinator = coordination.get_coordinator('zake://', b'host-1') coordinator.start() coordinator.stop() tooz-2.0.0/examples/hashring.py0000664000175000017500000000101313616633007016521 0ustar zuulzuul00000000000000from tooz import hashring hashring = hashring.HashRing({'node1', 'node2', 'node3'}) # Returns set(['node2']) nodes_for_foo = hashring[b'foo'] # Returns set(['node2', 'node3']) nodes_for_foo_with_replicas = hashring.get_nodes(b'foo', replicas=2) # Returns set(['node1', 'node3']) nodes_for_foo_with_replicas = hashring.get_nodes(b'foo', replicas=2, ignore_nodes={'node2'}) tooz-2.0.0/examples/leader_election.py0000664000175000017500000000140213616633007020036 0ustar zuulzuul00000000000000import time import uuid import six from tooz import coordination ALIVE_TIME = 1 coordinator = coordination.get_coordinator('zake://', b'host-1') coordinator.start() # Create a group group = six.binary_type(six.text_type(uuid.uuid4()).encode('ascii')) request = coordinator.create_group(group) request.get() # Join a group request = coordinator.join_group(group) request.get() def when_i_am_elected_leader(event): # event is a LeaderElected event print(event.group_id, event.member_id) # Propose to be a leader for the group coordinator.watch_elected_as_leader(group, when_i_am_elected_leader) start = time.time() while time.time() - start < ALIVE_TIME: coordinator.heartbeat() coordinator.run_watchers() time.sleep(0.1) coordinator.stop() tooz-2.0.0/examples/partitioner.py0000664000175000017500000000047413616633007017270 0ustar zuulzuul00000000000000from tooz import coordination coordinator = coordination.get_coordinator('zake://', b'host-1') coordinator.start() partitioner = coordinator.join_partitioned_group("group1") # Returns {'host-1'} member = partitioner.members_for_object(object()) coordinator.leave_partitioned_group(partitioner) coordinator.stop() tooz-2.0.0/examples/lock.py0000664000175000017500000000037013616633007015653 0ustar zuulzuul00000000000000from tooz import coordination coordinator = coordination.get_coordinator('zake://', b'host-1') coordinator.start() # Create a lock lock = coordinator.get_lock("foobar") with lock: print("Do something that is distributed") coordinator.stop() tooz-2.0.0/examples/group_membership.py0000664000175000017500000000056613616633007020301 0ustar zuulzuul00000000000000import uuid import six from tooz import coordination coordinator = coordination.get_coordinator('zake://', b'host-1') coordinator.start() # Create a group group = six.binary_type(six.text_type(uuid.uuid4()).encode('ascii')) request = coordinator.create_group(group) request.get() # Join a group request = coordinator.join_group(group) request.get() coordinator.stop() tooz-2.0.0/examples/coordinator_heartbeat.py0000664000175000017500000000022713616633007021266 0ustar zuulzuul00000000000000from tooz import coordination coordinator = coordination.get_coordinator('zake://', b'host-1') coordinator.start(start_heart=True) coordinator.stop() tooz-2.0.0/tooz.egg-info/0000775000175000017500000000000013616633265015226 5ustar zuulzuul00000000000000tooz-2.0.0/tooz.egg-info/dependency_links.txt0000664000175000017500000000000113616633265021274 0ustar zuulzuul00000000000000 tooz-2.0.0/tooz.egg-info/PKG-INFO0000664000175000017500000000417613616633265016333 0ustar zuulzuul00000000000000Metadata-Version: 2.1 Name: tooz Version: 2.0.0 Summary: Coordination library for distributed systems. Home-page: https://docs.openstack.org/tooz/latest/ Author: OpenStack Author-email: openstack-discuss@lists.openstack.org License: Apache-2 Description: Tooz ==== .. image:: https://img.shields.io/pypi/v/tooz.svg :target: https://pypi.org/project/tooz/ :alt: Latest Version .. image:: https://img.shields.io/pypi/dm/tooz.svg :target: https://pypi.org/project/tooz/ :alt: Downloads The Tooz project aims at centralizing the most common distributed primitives like group membership protocol, lock service and leader election by providing a coordination API helping developers to build distributed applications. * Free software: Apache license * Documentation: https://docs.openstack.org/tooz/latest/ * Source: https://opendev.org/openstack/tooz * Bugs: https://bugs.launchpad.net/python-tooz/ * Release notes: https://docs.openstack.org/releasenotes/tooz Join us ------- - https://launchpad.net/python-tooz Platform: UNKNOWN Classifier: Environment :: OpenStack Classifier: Intended Audience :: Developers Classifier: Intended Audience :: Information Technology Classifier: License :: OSI Approved :: Apache Software License Classifier: Operating System :: POSIX :: Linux Classifier: Programming Language :: Python Classifier: Programming Language :: Python :: 3 Classifier: Programming Language :: Python :: 3.6 Classifier: Programming Language :: Python :: 3.7 Classifier: Programming Language :: Python :: 3 :: Only Classifier: Programming Language :: Python :: Implementation :: CPython Classifier: Topic :: System :: Distributed Computing Requires-Python: >=3.6 Provides-Extra: consul Provides-Extra: etcd Provides-Extra: etcd3 Provides-Extra: etcd3gw Provides-Extra: zake Provides-Extra: redis Provides-Extra: postgresql Provides-Extra: mysql Provides-Extra: zookeeper Provides-Extra: memcached Provides-Extra: ipc Provides-Extra: test tooz-2.0.0/tooz.egg-info/top_level.txt0000664000175000017500000000000513616633265017753 0ustar zuulzuul00000000000000tooz tooz-2.0.0/tooz.egg-info/entry_points.txt0000664000175000017500000000104713616633265020526 0ustar zuulzuul00000000000000[tooz.backends] consul = tooz.drivers.consul:ConsulDriver etcd = tooz.drivers.etcd:EtcdDriver etcd3 = tooz.drivers.etcd3:Etcd3Driver etcd3+http = tooz.drivers.etcd3gw:Etcd3Driver file = tooz.drivers.file:FileDriver ipc = tooz.drivers.ipc:IPCDriver kazoo = tooz.drivers.zookeeper:KazooDriver memcached = tooz.drivers.memcached:MemcachedDriver mysql = tooz.drivers.mysql:MySQLDriver postgresql = tooz.drivers.pgsql:PostgresDriver redis = tooz.drivers.redis:RedisDriver zake = tooz.drivers.zake:ZakeDriver zookeeper = tooz.drivers.zookeeper:KazooDriver tooz-2.0.0/tooz.egg-info/pbr.json0000664000175000017500000000005613616633265016705 0ustar zuulzuul00000000000000{"git_version": "ba27954", "is_release": true}tooz-2.0.0/tooz.egg-info/requires.txt0000664000175000017500000000110513616633265017623 0ustar zuulzuul00000000000000pbr>=1.6 stevedore>=1.16.0 six>=1.9.0 voluptuous>=0.8.9 msgpack>=0.4.0 fasteners>=0.7 tenacity>=3.2.1 futurist>=1.2.0 oslo.utils>=3.15.0 oslo.serialization>=1.10.0 [consul] python-consul>=0.4.7 [etcd] requests>=2.10.0 [etcd3] etcd3>=0.6.2 grpcio>=1.18.0 [etcd3gw] etcd3gw>=0.1.0 [ipc] sysv-ipc>=0.6.8 [memcached] pymemcache!=1.3.0,>=1.2.9 [mysql] PyMySQL>=0.6.2 [postgresql] psycopg2>=2.5 [redis] redis>=2.10.0 [test] mock>=2.0 python-subunit>=0.0.18 testtools>=1.4.0 coverage>=3.6 fixtures>=3.0.0 pifpaf>=0.10.0 stestr>=2.0.0 [zake] zake>=0.1.6 [zookeeper] kazoo>=2.2 tooz-2.0.0/tooz.egg-info/not-zip-safe0000664000175000017500000000000113616633265017454 0ustar zuulzuul00000000000000 tooz-2.0.0/tooz.egg-info/SOURCES.txt0000664000175000017500000000522213616633265017113 0ustar zuulzuul00000000000000.coveragerc .stestr.conf .zuul.yaml AUTHORS CONTRIBUTING.rst ChangeLog LICENSE README.rst bindep.txt requirements.txt run-examples.sh run-tests.sh setup-consul-env.sh setup-etcd-env.sh setup.cfg setup.py test-requirements.txt tox.ini doc/requirements.txt doc/source/conf.py doc/source/index.rst doc/source/install/index.rst doc/source/reference/index.rst doc/source/user/compatibility.rst doc/source/user/drivers.rst doc/source/user/history.rst doc/source/user/index.rst doc/source/user/tutorial/coordinator.rst doc/source/user/tutorial/group_membership.rst doc/source/user/tutorial/hashring.rst doc/source/user/tutorial/index.rst doc/source/user/tutorial/leader_election.rst doc/source/user/tutorial/lock.rst doc/source/user/tutorial/partitioner.rst examples/coordinator.py examples/coordinator_heartbeat.py examples/group_membership.py examples/group_membership_watch.py examples/hashring.py examples/leader_election.py examples/lock.py examples/partitioner.py playbooks/stop-redis.yaml releasenotes/notes/add-reno-996dd44974d53238.yaml releasenotes/notes/drop-python-2-7-73d3113c69d724d6.yaml releasenotes/notes/etcd3-group-support-b039cf19f4a268a3.yaml releasenotes/notes/etcd3gw-group-support-598832a8764a8aa6.yaml releasenotes/notes/hashring-0470f9119ef63d49.yaml releasenotes/notes/join_group_create-5095ec02e20c7242.yaml releasenotes/notes/partitioner-4005767d287dc7c9.yaml releasenotes/source/conf.py releasenotes/source/index.rst releasenotes/source/ocata.rst releasenotes/source/pike.rst releasenotes/source/queens.rst releasenotes/source/rocky.rst releasenotes/source/stein.rst releasenotes/source/train.rst releasenotes/source/unreleased.rst releasenotes/source/_static/.placeholder releasenotes/source/_templates/.placeholder tools/compat-matrix.py tooz/__init__.py tooz/_retry.py tooz/coordination.py tooz/hashring.py tooz/locking.py tooz/partitioner.py tooz/utils.py tooz.egg-info/PKG-INFO tooz.egg-info/SOURCES.txt tooz.egg-info/dependency_links.txt tooz.egg-info/entry_points.txt tooz.egg-info/not-zip-safe tooz.egg-info/pbr.json tooz.egg-info/requires.txt tooz.egg-info/top_level.txt tooz/drivers/__init__.py tooz/drivers/consul.py tooz/drivers/etcd.py tooz/drivers/etcd3.py tooz/drivers/etcd3gw.py tooz/drivers/file.py tooz/drivers/ipc.py tooz/drivers/memcached.py tooz/drivers/mysql.py tooz/drivers/pgsql.py tooz/drivers/redis.py tooz/drivers/zake.py tooz/drivers/zookeeper.py tooz/tests/__init__.py tooz/tests/test_coordination.py tooz/tests/test_etcd.py tooz/tests/test_hashring.py tooz/tests/test_memcache.py tooz/tests/test_mysql.py tooz/tests/test_partitioner.py tooz/tests/test_postgresql.py tooz/tests/test_utils.py tooz/tests/drivers/__init__.py tooz/tests/drivers/test_file.pytooz-2.0.0/bindep.txt0000664000175000017500000000051313616633007014534 0ustar zuulzuul00000000000000redis-sentinel [platform:ubuntu !platform:ubuntu-trusty] redis-server [platform:dpkg] libpq-dev [platform:dpkg] postgresql [platform:dpkg] mysql-client [platform:dpkg] mysql-server [platform:dpkg] build-essential [platform:dpkg] libffi-dev [platform:dpkg] zookeeperd [platform:dpkg] memcached [platform:dpkg] unzip [platform:dpkg] tooz-2.0.0/ChangeLog0000664000175000017500000005377113616633265014330 0ustar zuulzuul00000000000000CHANGES ======= 2.0.0 ----- * [ussuri][goal] Drop python 2.7 support and testing 1.67.2 ------ * RedisLock release() should not check if the lock has been acquired 1.67.1 ------ 1.67.0 ------ * Drop os-testr test-requirement and pretty\_tox.sh * Update master for stable/train * Add shared arg in metaclass Lock 1.66.2 ------ * Fix membership lease issue on the etcd3gw driver 1.66.1 ------ * Fix wrong log level during heartbeat 1.66.0 ------ * Add Python 3 Train unit tests * Move grpcio from requirements.txt to extras * Blacklist sphinx 2.1.0 1.65.0 ------ * Move test deps to test-requirements.txt * Remove unused requirements * Update Sphinx requirement and uncap grpcio * Referencing testenv deps now works * Update master for stable/stein * Replace git.openstack.org URLs with opendev.org URLs * Remove py35, add py37 classifiers * Unblock tooz gate * OpenDev Migration Patch * add python 3.7 unit test job 1.64.2 ------ 1.64.1 ------ * Fixed UnicodeEncodeError for Python2 unicode objects 1.64.0 ------ * More explicitly document driver connection strings * Unblock tooz gate * Change openstack-dev to openstack-discuss 1.63.1 ------ 1.63.0 ------ * coordination: do not retry the whole heartbeat on fail * Use templates for cover * Migrate to stestr * Fix coverage tests * Switch to autodoc\_default\_options * Ensure consistent encoding of strings for ID * add lib-forward-testing-python3 test job * add python 3.6 unit test job * import zuul job settings from project-config * Update reno for stable/rocky * Add release note link in README * fix tox python3 overrides 1.62.0 ------ * Trivial: Update pypi url to new url * set default python to python3 * Implement group support for etcd3gw 1.61.0 ------ * Zuul: Remove project name * Zuul: Remove project name * Update reno for stable/queens * partitioner: do not use hash() to determine object identity * msgpack-python has been renamed to msgpack * Follow the new PTI for document build * Remove tox\_install.sh * Use native Zuul v3 tox jobs * Add doc/requirements.txt * Remove setting of version/release from releasenotes * Zuul: add file extension to playbook path * Move legacy jobs to project 1.59.0 ------ * Acquire fails with "ToozError: Not found" * redis: always remove lock from acquired lock when release()ing * redis: log an error on release failure * Use the same default timeout for async result * Update reno for stable/pike 1.58.0 ------ * Update URLs in documents according to document migration * doc: use list-table for driver support tables * rearrange existing documentation to fit the new standard layout 1.57.4 ------ * Switch from oslosphinx to openstackdocstheme * Turn on warning-is-error in doc build * Add etcd3 group support 1.57.3 ------ * Make sure Lock.heartbeat() returns True/False * etcd3: skip ProcessPool based test * etcd3: replace custom lock code by more recent etcd3 lock code * pgsql: fix self.\_conn.close() being called without connection * test: leverage existing helper method in test\_partitioner * coordination: remove double serialization of capabilities * tests: fix missing .get() on some group operations * Mutualize executor code in a mixin class * coordination: fix reversed fiels for \_\_repr\_\_ for events * Fix docstring for group and member id 1.57.2 ------ * {my,pg}sql: close connections when out of retry * Disable test\_get\_lock\_serial\_locking\_two\_lock\_process for etcd3 * Simplify env list and test running * Factorize tox envlist for better readability 1.57.1 ------ * consul: remove unused executor 1.57.0 ------ * Separate etcd3gw driver that uses the etcd3 grpc gateway * etcd3: use discard() rather than remove() * etcd: fix blocking argument * etcd: fix acquire(blocking=True) on request exception * etcd3: fix test run * coordination: factorize common async result futures code 1.56.1 ------ * Fix psycopg2 connection argument 1.56.0 ------ * doc: update heartbeat doc to use start\_heart=True * sql: close connection for lock if not used * http->https for security * etcd3: add etcd3 coordination driver 1.55.0 ------ * Implement heartbeat for FileDriver * simplify hashring node lookup 1.54.0 ------ * redis: fix concurrent access on acquire() * tests: tests fail if no URL is set + run partitioner tests on basic drivers * tests: fix etcd and consul test run * add weight tests for add\_nodes * get weight of existing members * coordination: do not get member list if not needed * Add shared filelock 1.53.0 ------ * Enhance heartbeat sleep timer 1.52.0 ------ * FileDriver:Support multiple processes * Switch tests to use latest etcd - 3.1.3 1.51.0 ------ * pass on partitions * hashring: allow to use bytes as node name 1.50.0 ------ * postgresql: only pass username and password if they are set * Rewrite heartbeat runner with event * Adds authentication support for zookeeperDriver 1.49.0 ------ * support unicode node name * Update reno for stable/ocata 1.48.0 ------ * Fix test function name with two underscores to have only one 1.47.0 ------ * Add partitioner implementation * Stop making tooz.utils depending on tooz.coordination * [doc] Note lack of constraints is a choice * The 'moves.moved\_class' function creates a new class * Add weight support to the hashring * coordination: allow to pass capabilities in join\_group\_create() * coordination: fix moved\_class usage for ToozError * zookeeper: switch to standard group membership watching * Add a hashring implementation * Move ToozError to root module * Fixup concurrent modification * Replaces uuid.uuid4 with uuidutils.generate\_uuid() 1.46.0 ------ * Do not re-set the members cache for watchers by default * coordination: add \_\_repr\_\_ for join/leave events * coordination: renforce event based testing * Factorize member\_id in the base coordinator class * Use the internal group of list rather than listing the groups * Move the cached-based watcher implementation into its own class 1.45.0 ------ * Factorize group quit on stop() * coordination: make get\_members() return a set * Replace 'assertTrue(a (not)in b)' with 'assert(Not)In(a, b)' * Changed author and author-email * redis: make sure we don't release and heartbeat a lock at the same time * coordinator: add join\_group\_create * Replace retrying with tenacity * Add CONTRIBUTING.rst * Replace 'assertTrue(a in b)' with 'assertIn(a, b)' and 'assertFalse(a in b)' with 'assertNotIn(a, b)' * Using assertIsNone() instead of assertEqual(None, ...) * tox: use pretty tox output * tox: install docs dependency in docs target and reno * Bump hacking to 0.12 * Add reno for release notes management 1.44.0 ------ * Changed the home-page link * file: update .metadata atomically * file: return converted voluptuous data * file: move \_load\_and\_validate to a method * file: move \_read\_{group,member}\_id to staticmethod-s * etcd: run tests in clustering mode too * Use method ensure\_tree from oslo.utils * Switch from Python 3.4 to Python 3.5 * Install only needed packages * Fix a typo in file.py * Update etcd version in tests 1.43.0 ------ * Makedirs only throws oserror, so only catch that 1.42.0 ------ * etcd: don't run heartbeat() concurrently * Raise tooz error when unexpected last entries found * etcd: properly block when using 'wait' * Share \_get\_random\_uuid() among all tests * Updated from global requirements * Clean leave group hooks when unwatching * Fix the test test\_unwatch\_elected\_as\_leader * Updated from global requirements 1.41.0 ------ * File driver: properly handle Windows paths * Updated from global requirements * Updated from global requirements 1.40.0 ------ * Add docs for new consul driver * Change dependency to use flavors * Run doc8 only in pep8 target * Move pep8 requirements in their own target * zookeeper: do not hard depend on eventlet * Remove unused iso8601 dependency * tests: remove testscenario usage * file: set no timeout by default * tests: move bad\_url from scenarios to static test * Expose timeout capabilities and use them for tests * Use pifpaf to setup daemons 1.39.0 ------ * Updated from global requirements 1.38.0 ------ * Using LOG.warning instead of LOG.warn * Updated from global requirements * redis: do not force LuaLock * Fix coordinator typo * Updated from global requirements * Ensure etcd is in developer and driver docs 1.37.0 ------ * Remove unused consul future result * Updated from global requirements * Add a consul based driver * file: make python2 payload readable from python3 1.36.0 ------ * Updated from global requirements 1.35.0 ------ * Drop babel as requirement since its not used * Updated from global requirements * Updated from global requirements * Updated from global requirements * Updated from global requirements * coordination: expose a heartbeat loop method 1.34.0 ------ 1.33.0 ------ * Updated from global requirements * Compute requires\_beating * Fix calling acquire(blocking=False) twice leads to a deadlock 1.32.0 ------ * Raises proper error when unwatching a group 1.31.0 ------ * Updated from global requirements * Updated from global requirements * Add .tox, \*.pyo and \*.egg to .gitignore * Enable OS\_LOG\_CAPTURE so that logs can be seen (on error) 1.30.0 ------ * Updated from global requirements * Add lock breaking * pgsql: fix hostname parsing * Updated from global requirements * Updated from global requirements * Update voluptuous requirement * Updated from global requirements * Updated from global requirements * Have zookeeper heartbeat perform basic get * Add desired characteristics strict subset validation * Add base64 key encoder (and validations) * Use voluptuous instead of jsonschema * Add programatic introspection of drivers characteristic(s) * Updated from global requirements * pep8: fix remaining errors and enable all checks * Use utils.convert\_blocking to convert blocking argument * Adjust some of the zookeeper exception message * Fix etcd env setup * tests: do not hardcode /tmp * utils: replace exception\_message by exception\_to\_unicode * Add a default port and default host * etcd: driver with lock support * Use utils.to\_binary instead of using redis module equivalent * Remove tested under 2.6 from docs 1.29.0 ------ * Updated from global requirements * Add basic file content schema validation * Spice up the driver summary/info page * Make all locks operate the same when fetched from different coordinators * Add noted driver weaknesses onto the drivers docs * Updated from global requirements * File: read member id from file with suffix ".raw" * Reduce duplication of code in handling multi-type blocking argument * Updated from global requirements * Add comment in memcache explaining the current situation with lock release 1.28.0 ------ * Add 'requires\_beating' property to coordination driver * {pg,my}sql: fix AttributeError on connection failure * tests: allow ipc to bypass blocking=False test * pgsql: remove unused left-over code * Add 'is\_still\_owner' lock test function 1.27.0 ------ * Updated from global requirements * Updated from global requirements * Remove python 2.6 and cleanup tox.ini 1.26.0 ------ * Updated from global requirements * Allow specifying a kazoo async handler 'kind' * Updated from global requirements 1.25.0 ------ * Updated from global requirements * Add standard code coverage configuration file * docs - Set pbr 'warnerrors' option for doc build * Include changelog/history in docs * Updated from global requirements * Expose Znode Stats and Capabilities * Allow more kazoo specific client options to be proxied through 1.24.0 ------ * Updated from global requirements 1.23.0 ------ * Changes to add driver list to the documentation * Updated from global requirements 1.22.0 ------ * Updated from global requirements * Accept blocking argument in lock's context manager * Make RedisLock's init consistent with other locks * Updated from global requirements 1.21.0 ------ * Raise exception on failed lock's CM acquire fail * Be more restrictive on the executors users can provide 1.20.0 ------ * Updated from global requirements * Updated from global requirements * Use futurist to allow for executor providing and unifying * Use a lua script(s) instead of transactions 1.19.0 ------ * Updated from global requirements * Change Lock.name to a property * Update .gitignore * Updated from global requirements * Fixup dependencies * Expose started state of coordinator to external * Updated from global requirements * Updated from global requirements 1.18.0 ------ * Remove tooz/openstack as it is empty and not used * Fix sp 'seonds' -> 'seconds' * Ensure run\_watchers called from mixin, not base class * Updated from global requirements * Update compatibility matrix due to file drivers new abilities 0.17.0 ------ * No longer need kazoo lock custom retry code * Ensure unwatch\_elected\_as\_leader correctly clears hooks 0.16.0 ------ * Updated from global requirements * Updated from global requirements * Ensure lock(s) acquire/release returns boolean values * Expose 'run\_elect\_coordinator' and call it from 'run\_watchers' * Share most of the \`run\_watchers\` code via a common mixin * Remove 2.6 classifier * Remove file-driver special no-async abilities * Delay interpolating the LOG string * Use \`encodeutils.exception\_to\_unicode\` for exception -> string function * Use the \`excutils.raise\_with\_cause\` after doing our type check * Updated from global requirements * Use the 'driver\_lock' around read operations * Updated from global requirements * Switch badges from 'pypip.in' to 'shields.io' * Updated from global requirements * Add watch file driver support * Make the file driver more capable (with regard to groups) * Ensure locks can not be created outside of the root file driver directory * Updated from global requirements * Use MySQL default port when not set explicitly * Use fasteners library for interprocess locks * Implement watch/unwatch elected\_as\_leader for redis driver * Updated from global requirements * Use lua locks instead of pipeline locks * Move more string constants to class constants with docstrings * Updated from global requirements * Updated from global requirements * Remove support for redis < 2.6.0 * Expose Zookeeper client class constants * Expose redis client class constants * Use a serialization/deserialization specific exception * Expose memcache coord. class constants * Explicitly start and execute most transactions * Provide and use a options collapsing function * Add zookeeper tag in setup.cfg * Use pymemcache pooled client * Use oslo.serialization msgpackutils * Provide ability for namespace customization for Zookeeper and Zake drivers * Typo in Locking doc * Move optional driver requirements to test-requirements.txt * Have run\_watchers take a timeout and respect it * Heartbeat on acquired locks copy * Avoid using a thread local token storage 0.15.0 ------ * Fix param name to be its right name * Replace more instance(s) of exception chaining with helper * Just use staticmethod functions to create \_dumps/\_loads * Uncap library requirements for liberty * Link AOF to redis persistence docs * Add exception docs to developer docs * Add + use helper to raise + chain exceptions * Allow the acquired file to be closed manually * Updated from global requirements * Silence logs + errors when stopping and group membership lost * Make and use a thread safe pymemcache client subclass * Handle errors that come out of pymemcache better * Use rst inline code structure + link to sentinel 0.14.0 ------ * Beef up the docstrings on the various drivers * fix lock concurrency issues with certain drivers * Add pypi download + version badges * Denote that 2.6 testing is still happening * Updated from global requirements * Use a sentinel connection pool to manage failover * fix mysql driver url parsing 0.13.1 ------ * Switch to non-namespaced module imports * Add a driver feature compatibility matrix * Remove support for 3.3 0.13.0 ------ * Two locks acquired from one coord must works * Updated from global requirements * Releases locks in tests * Allow coordinator non-string options and use them * Since we use msgpack this can be more than a str * Updated from global requirements * Avoid re-using the same timeout for further watcher ops 0.12 ---- * retry: fix decorator * file: fix typo in errno.EACCES 0.11 ---- * Add a file based driver * Upgrade to hacking 0.10 * Update sentinel support to allow multiple sentinel hosts * Allow to pass arguments to retry() * IPC simplification 0.10 ---- * Add support for an optional redis-sentinel * README.rst tweaks * A few more documentation tweaks * Sync requirements to global requirements * Add create/join/leave group support in IPC driver * Add driver autogenerated docs * Update links + python version supported * zookeeper: add support for delete group * redis: add support for group deletion * tests: minor code simplification * memcached: add support for group deletion * memcached: add support for \_destroy\_group * Switch to using oslosphinx * Add doc on how transaction is itself retrying internally * Fix .gitreview after rename/transfer * tests: use scenarios attributes for timeout capability * tests: check for leave group events on dead members cleanup * memcached: delete stale/dead group members on get\_members() * tests: remove check\_port * tests: do not skip test on connection error 0.9 --- * doc: add missing new drivers * doc: switch examples to Zake * doc: add locking * Fix tox envlist * Drop Python 3.3 tests in tox * Allow tox with py34 and MySQL * Test connection error scenarios on more drivers * Translate psycopg2 errors/exceptions into tooz exceptions * Ensure 'leave\_group' result gotten before further work * watch\_leave\_group not triggering callback on expired members * Add MySQL driver * Discard 'self' from '\_joined\_groups' if we got booted out * Implement non-blocking locks with PostgreSQL * More retry code out of memcached * Add a PostgreSQL driver * Fix gate * Handle when a group used to exist but no longer does * tox: split redis/memcached env * Fix memcached heartbeat on start() * tox: splits test scenarios * Add a minimum redis version check while starting * Make requirement on redis 2.10.x explicit * Try to use PSETEX when possible * Use hdel with many keys where supported * Avoid logging warnings when group deleted or member gone * Ensure that we correctly expire (and cleanup) redis members * Various fixes for locks and version compatibility * Move sysv\_ipc deps to test-requirements 0.8 --- * test: try to stop() first * Convert the rest of memcached driver functions to futures * Add a assertRaisesAny helper method * Allow zake to be tested * Add a redis driver * Ensure groups leaving returns are gotten * Raise the new OperationTimedOut when futures don't finish * Start to add a catch and reraise of timed out exceptions * Adjust the timeout to reflect the repeated retries * ipc: do not delete the lock if we never acquired it * Add home-page field 0.7 --- * Split up the requirements for py2.x and py3.x * ipc: Fix acquire lock loop logic 0.6 --- * Make lock blocking with no time out by default 0.5 --- * coordination: remove destroy() from the lock protocol * IPC: fix a potential race condition at init * Fix IPC driver on OS X * Switch to oslo.utils * Blacklist retrying 1.3.0 * Use futures to make parts of the memcached driver async * Have examples run in the py27 environment and make them work 0.4 --- * Standardize the async result subclasses * Fix the comment which was borrowed from the IPC driver * Be more tolerant of unicode exceptions * Standardize on the same lock acquire method definition * Standardize on hiding the lock implementation * On lock removal validate that they key was actually deleted * Use a thread safe deque instead of a queue * Change inline docs about class fake storage variable * LOG a warning if the heartbeat can not be validated * Add doc8 to the py27 test running * Use the more reliable sysv\_ipc instead of posix\_ipc+lockutils * Only start zookeeper/memcached when not already running * Let zake act as a in-memory fully functional driver * Switch to a custom NotImplemented error * Ensure lock list isn't mutated while iterating * Move Zake driver code to separated Python module * Work toward Python 3.4 support and testing * Unlock the kazoo version * Bump up zake to be using the newer 0.1 or greater * Fix zake driver with latest release * memcached: switch leader election implementation to a lock * Add the generation of the documentation in tox.ini * Add coverage report 0.3 --- * Switch to URL for loading backends * Import network\_utils from Oslo * coordination: add IPC driver * coordination: raise NotImplementedError as default * Add documentation * Upgrade hacking requirement * memcached: use retrying rather than sleeping * Use retrying instead of our custom code * Update requirements file matching global requ 0.2 --- * memcached: implement leader election * Fix a race condition in one of the test 0.1 --- * memcached: add locking * coordination: implement lock mechanism in ZK * coordination, zookeeper: add get\_leader() * coordination, zookeeper: implement leader election * coordination: remove wrong comment in tests * memcached: add support for leave events * memcached: implement {un,}watch\_join\_group() * coordination: raise GroupNotCreated when watching uncreated group * coordination, zookeeper: add {un,}watch\_leave\_group * coordination, zookeeper: add watch\_join\_group * tests: skip test if function is not implemented * coordination: add hooks system * Add memcached driver * zookeeper: use bytes as input/output type * tests: test client disconnection * coordination: add heartbeat method * Add pbr generated and testr files to gitignore * coordination: enhance MemberAlreadyExist exception * coordination: enhance GroupNotCreated exception * coordination: enhance MemberNotJoined * coordination: enhance GroupAlreadyExist exception * tests: test capabilities on non existent group/member * tests: add a test for group already existing * tests: fix variable name * Fix the default prototype for join\_group * Adds basic tests which deals with exceptions * Fixes TypeError in \_leave\_group\_handler * Remove \_wrap\_call\_kazoo * Add asynchronous API * Delete models.py and clean get\_members() * Add a fake ZooKeeper driver * Allow passing in a handler * First commit of Tooz * Added .gitreview tooz-2.0.0/AUTHORS0000664000175000017500000000537713616633265013625 0ustar zuulzuul00000000000000Abhijeet Malawade Adam Gandelman Akihiro Motoki Andreas Jaeger Ben Nemec Bob.Haddleton Cao Xuan Hoang ChangBo Guo(gcb) Chris Dent Corey Bryant Davanum Srinivas Davanum Srinivas Dina Belova Dirk Mueller Dmitriy Rabotjagov Doug Hellmann Duong Ha-Quang Eoghan Glynn Gary Kotton Ghanshyam Mann Gorka Eguileor Hervé Beraud Hoang Trung Hieu Imran Ansari James E. Blair James Page Jay Clark Jay Pipes Jeremy Stanley John Schwarz Joshua Harlow Joshua Harlow Joshua Harlow Julien Danjou Kaifeng Wang Kiall Mac Innes Kobi Samoray Lahoucine BENLAHMR Longgeek Lucas Alvares Gomes Lucian Petrut Mehdi Abaakouk Mehdi Abaakouk Mehdi Abaakouk Monty Taylor OpenStack Release Bot Robert Collins Ronald Bradford Sahid Orentino Ferdjaoui Sean McGinnis Sergey Lukjanov Thomas Bechtold Thomas Herve Tony Breeds Victor Morales Vilobh Meshram XiaojueGuan Yassine Lamgarchal Yassine Lamgarchal caoyuan fuzk garenchan gord chung gordon chung howardlee lahoucine BENLAHMR liu-sheng malei melissaml mengalong ushen yenai zhangguoqing zhangsong zhangyanxian tooz-2.0.0/.coveragerc0000664000175000017500000000014313616633007014652 0ustar zuulzuul00000000000000[run] branch = True source = tooz omit = tooz/tests/* [report] ignore_errors = True precision = 2 tooz-2.0.0/test-requirements.txt0000664000175000017500000000061013616633007016771 0ustar zuulzuul00000000000000 # The order of packages is significant, because pip processes them in the order # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. mock>=2.0 # BSD python-subunit>=0.0.18 # Apache-2.0/BSD testtools>=1.4.0 # MIT coverage>=3.6 # Apache-2.0 fixtures>=3.0.0 # Apache-2.0/BSD pifpaf>=0.10.0 # Apache-2.0 stestr>=2.0.0 tooz-2.0.0/tools/0000775000175000017500000000000013616633265013701 5ustar zuulzuul00000000000000tooz-2.0.0/tools/compat-matrix.py0000664000175000017500000000654513616633007017044 0ustar zuulzuul00000000000000# -*- coding: utf-8 -*- # Copyright (C) 2015 Yahoo! Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. from tabulate import tabulate def print_header(txt, delim="="): print(txt) print(delim * len(txt)) def print_methods(methods): driver_tpl = ":py:meth:`~tooz.coordination.CoordinationDriver.%s`" for api_name in methods: method_name = driver_tpl % api_name print("* %s" % method_name) if methods: print("") driver_tpl = ":py:class:`~tooz.drivers.%s`" driver_class_names = [ "consul.ConsulDriver", "etcd.EtcdDriver", "file.FileDriver", "ipc.IPCDriver", "memcached.MemcachedDriver", "mysql.MySQLDriver", "pgsql.PostgresDriver", "redis.RedisDriver", "zake.ZakeDriver", "zookeeper.KazooDriver", ] driver_headers = [] for n in driver_class_names: driver_headers.append(driver_tpl % (n)) print_header("Grouping") print("") print_header("APIs", delim="-") print("") grouping_methods = [ 'watch_join_group', 'unwatch_join_group', 'watch_leave_group', 'unwatch_leave_group', 'create_group', 'get_groups', 'join_group', 'leave_group', 'delete_group', 'get_members', 'get_member_capabilities', 'update_capabilities', ] print_methods(grouping_methods) print_header("Driver support", delim="-") print("") grouping_table = [ [ "No", # Consul "No", # Etcd "Yes", # File "No", # IPC "Yes", # Memcached "No", # MySQL "No", # PostgreSQL "Yes", # Redis "Yes", # Zake "Yes", # Zookeeper ], ] print(tabulate(grouping_table, driver_headers, tablefmt="rst")) print("") print_header("Leaders") print("") print_header("APIs", delim="-") print("") leader_methods = [ 'watch_elected_as_leader', 'unwatch_elected_as_leader', 'stand_down_group_leader', 'get_leader', ] print_methods(leader_methods) print_header("Driver support", delim="-") print("") leader_table = [ [ "No", # Consul "No", # Etcd "No", # File "No", # IPC "Yes", # Memcached "No", # MySQL "No", # PostgreSQL "Yes", # Redis "Yes", # Zake "Yes", # Zookeeper ], ] print(tabulate(leader_table, driver_headers, tablefmt="rst")) print("") print_header("Locking") print("") print_header("APIs", delim="-") print("") lock_methods = [ 'get_lock', ] print_methods(lock_methods) print_header("Driver support", delim="-") print("") lock_table = [ [ "Yes", # Consul "Yes", # Etcd "Yes", # File "Yes", # IPC "Yes", # Memcached "Yes", # MySQL "Yes", # PostgreSQL "Yes", # Redis "Yes", # Zake "Yes", # Zookeeper ], ] print(tabulate(lock_table, driver_headers, tablefmt="rst")) print("") tooz-2.0.0/releasenotes/0000775000175000017500000000000013616633265015232 5ustar zuulzuul00000000000000tooz-2.0.0/releasenotes/source/0000775000175000017500000000000013616633265016532 5ustar zuulzuul00000000000000tooz-2.0.0/releasenotes/source/unreleased.rst0000664000175000017500000000016013616633007021402 0ustar zuulzuul00000000000000============================== Current Series Release Notes ============================== .. release-notes:: tooz-2.0.0/releasenotes/source/queens.rst0000664000175000017500000000022313616633007020553 0ustar zuulzuul00000000000000=================================== Queens Series Release Notes =================================== .. release-notes:: :branch: stable/queens tooz-2.0.0/releasenotes/source/index.rst0000664000175000017500000000025213616633007020364 0ustar zuulzuul00000000000000==================== tooz Release Notes ==================== .. toctree:: :maxdepth: 1 unreleased train stein rocky queens pike ocata tooz-2.0.0/releasenotes/source/ocata.rst0000664000175000017500000000023013616633007020340 0ustar zuulzuul00000000000000=================================== Ocata Series Release Notes =================================== .. release-notes:: :branch: origin/stable/ocata tooz-2.0.0/releasenotes/source/_static/0000775000175000017500000000000013616633265020160 5ustar zuulzuul00000000000000tooz-2.0.0/releasenotes/source/_static/.placeholder0000664000175000017500000000000013616633007022423 0ustar zuulzuul00000000000000tooz-2.0.0/releasenotes/source/conf.py0000664000175000017500000002131013616633007020020 0ustar zuulzuul00000000000000# -*- coding: utf-8 -*- # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. # This file is execfile()d with the current directory set to its # containing dir. # # Note that not all possible configuration values are present in this # autogenerated file. # # All configuration values have a default; values that are commented out # serve to show the default. # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # sys.path.insert(0, os.path.abspath('.')) # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. # needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ 'openstackdocstheme', 'reno.sphinxext', ] # openstackdocstheme options repository_name = 'openstack/tooz' bug_project = 'tooz' bug_tag = '' # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] # The suffix of source filenames. source_suffix = '.rst' # The encoding of source files. # source_encoding = 'utf-8-sig' # The master toctree document. master_doc = 'index' # General information about the project. project = u'tooz Release Notes' copyright = u'2016, tooz Developers' # Release do not need a version number in the title, they # cover multiple versions. # The full version, including alpha/beta/rc tags. release = '' # The short X.Y version. version = '' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: # today = '' # Else, today_fmt is used as the format for a strftime call. # today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. exclude_patterns = [] # The reST default role (used for this markup: `text`) to use for all # documents. # default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. # add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). # add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. # show_authors = False # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' # A list of ignored prefixes for module index sorting. # modindex_common_prefix = [] # If true, keep warnings as "system message" paragraphs in the built documents. # keep_warnings = False # -- Options for HTML output ---------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. html_theme = 'openstackdocs' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. # html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. # html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". # html_title = None # A shorter title for the navigation bar. Default is the same as html_title. # html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. # html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. # html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied # directly to the root of the documentation. # html_extra_path = [] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. html_last_updated_fmt = '%Y-%m-%d %H:%M' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. # html_use_smartypants = True # Custom sidebar templates, maps document names to template names. # html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. # html_additional_pages = {} # If false, no module index is generated. # html_domain_indices = True # If false, no index is generated. # html_use_index = True # If true, the index is split into individual pages for each letter. # html_split_index = False # If true, links to the reST sources are added to the pages. # html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. # html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. # html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. # html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). # html_file_suffix = None # Output file base name for HTML help builder. htmlhelp_basename = 'toozReleaseNotesDoc' # -- Options for LaTeX output --------------------------------------------- latex_elements = { # The paper size ('letterpaper' or 'a4paper'). # 'papersize': 'letterpaper', # The font size ('10pt', '11pt' or '12pt'). # 'pointsize': '10pt', # Additional stuff for the LaTeX preamble. # 'preamble': '', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ ('index', 'toozReleaseNotes.tex', u'tooz Release Notes Documentation', u'tooz Developers', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of # the title page. # latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. # latex_use_parts = False # If true, show page references after internal links. # latex_show_pagerefs = False # If true, show URL addresses after external links. # latex_show_urls = False # Documents to append as an appendix to all manuals. # latex_appendices = [] # If false, no module index is generated. # latex_domain_indices = True # -- Options for manual page output --------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ ('index', 'toozReleaseNotes', u'tooz Release Notes Documentation', [u'tooz Developers'], 1) ] # If true, show URL addresses after external links. # man_show_urls = False # -- Options for Texinfo output ------------------------------------------- # Grouping the document tree into Texinfo files. List of tuples # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ ('index', 'toozReleaseNotes', u'tooz Release Notes Documentation', u'tooz Developers', 'toozReleaseNotes', 'One line description of project.', 'Miscellaneous'), ] # Documents to append as an appendix to all manuals. # texinfo_appendices = [] # If false, no module index is generated. # texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. # texinfo_show_urls = 'footnote' # If true, do not generate a @detailmenu in the "Top" node's menu. # texinfo_no_detailmenu = False # -- Options for Internationalization output ------------------------------ locale_dirs = ['locale/'] tooz-2.0.0/releasenotes/source/rocky.rst0000664000175000017500000000022113616633007020400 0ustar zuulzuul00000000000000=================================== Rocky Series Release Notes =================================== .. release-notes:: :branch: stable/rocky tooz-2.0.0/releasenotes/source/stein.rst0000664000175000017500000000022113616633007020373 0ustar zuulzuul00000000000000=================================== Stein Series Release Notes =================================== .. release-notes:: :branch: stable/stein tooz-2.0.0/releasenotes/source/pike.rst0000664000175000017500000000021713616633007020206 0ustar zuulzuul00000000000000=================================== Pike Series Release Notes =================================== .. release-notes:: :branch: stable/pike tooz-2.0.0/releasenotes/source/_templates/0000775000175000017500000000000013616633265020667 5ustar zuulzuul00000000000000tooz-2.0.0/releasenotes/source/_templates/.placeholder0000664000175000017500000000000013616633007023132 0ustar zuulzuul00000000000000tooz-2.0.0/releasenotes/source/train.rst0000664000175000017500000000017613616633007020377 0ustar zuulzuul00000000000000========================== Train Series Release Notes ========================== .. release-notes:: :branch: stable/train tooz-2.0.0/releasenotes/notes/0000775000175000017500000000000013616633265016362 5ustar zuulzuul00000000000000tooz-2.0.0/releasenotes/notes/etcd3-group-support-b039cf19f4a268a3.yaml0000664000175000017500000000012013616633007025314 0ustar zuulzuul00000000000000--- features: - | The etcd3 driver now supports the group membership API. tooz-2.0.0/releasenotes/notes/etcd3gw-group-support-598832a8764a8aa6.yaml0000664000175000017500000000012213616633007025532 0ustar zuulzuul00000000000000--- features: - | The etcd3gw driver now supports the group membership API. tooz-2.0.0/releasenotes/notes/join_group_create-5095ec02e20c7242.yaml0000664000175000017500000000023413616633007025000 0ustar zuulzuul00000000000000--- features: - Coordination drivers now have a method `join_group_create` that is able to create a group before joining it if it does not exist yet. tooz-2.0.0/releasenotes/notes/hashring-0470f9119ef63d49.yaml0000664000175000017500000000011613616633007023130 0ustar zuulzuul00000000000000--- features: - Add `tooz.hashring`, a consistent hash ring implementation. tooz-2.0.0/releasenotes/notes/partitioner-4005767d287dc7c9.yaml0000664000175000017500000000030113616633007023663 0ustar zuulzuul00000000000000--- features: - >- Introduce a new partitioner object. This object is synchronized within a group of nodes and exposes a way to distribute object management across several nodes. tooz-2.0.0/releasenotes/notes/add-reno-996dd44974d53238.yaml0000664000175000017500000000007213616633007022752 0ustar zuulzuul00000000000000--- other: - Introduce reno for deployer release notes. tooz-2.0.0/releasenotes/notes/drop-python-2-7-73d3113c69d724d6.yaml0000664000175000017500000000020313616633007024101 0ustar zuulzuul00000000000000--- upgrade: - | Python 2.7 support has been dropped. The minimum version of Python now supported by tooz is Python 3.6. tooz-2.0.0/setup-consul-env.sh0000775000175000017500000000134513616633007016324 0ustar zuulzuul00000000000000#!/bin/bash set -eux if [ -z "$(which consul)" ]; then CONSUL_VERSION=0.6.3 CONSUL_RELEASE_URL=https://releases.hashicorp.com/consul case `uname -s` in Darwin) consul_file="consul_${CONSUL_VERSION}_darwin_amd64.zip" ;; Linux) consul_file="consul_${CONSUL_VERSION}_linux_amd64.zip" ;; *) echo "Unknown operating system" exit 1 ;; esac consul_dir=`basename $consul_file .zip` mkdir -p $consul_dir curl -L $CONSUL_RELEASE_URL/$CONSUL_VERSION/$consul_file > $consul_dir/$consul_file unzip $consul_dir/$consul_file -d $consul_dir export PATH=$PATH:$consul_dir fi # Yield execution to venv command $* tooz-2.0.0/tox.ini0000664000175000017500000000436513616633007014056 0ustar zuulzuul00000000000000[tox] minversion = 3.1.0 envlist = py37,py{36,37}-{zookeeper,redis,sentinel,memcached,postgresql,mysql,consul,etcd,etcd3,etcd3gw},pep8 ignore_basepython_conflict = True [testenv] basepython = python3 # We need to install a bit more than just `test-requirements' because those drivers have # custom tests that we always run deps = .[zake,ipc,memcached,mysql,etcd,etcd3,etcd3gw] zookeeper: .[zookeeper] redis: .[redis] sentinel: .[redis] memcached: .[memcached] postgresql: .[postgresql] mysql: .[mysql] etcd: .[etcd] etcd3: .[etcd3] etcd3gw: .[etcd3gw] consul: .[consul] -r{toxinidir}/test-requirements.txt setenv = TOOZ_TEST_URLS = file:///tmp zake:// ipc:// zookeeper: TOOZ_TEST_DRIVERS = zookeeper redis: TOOZ_TEST_DRIVERS = redis sentinel: TOOZ_TEST_DRIVERS = redis --sentinel memcached: TOOZ_TEST_DRIVERS = memcached mysql: TOOZ_TEST_DRIVERS = mysql postgresql: TOOZ_TEST_DRIVERS = postgresql etcd: TOOZ_TEST_DRIVERS = etcd,etcd --cluster etcd3: TOOZ_TEST_DRIVERS = etcd etcd3: TOOZ_TEST_ETCD3 = 1 etcd3gw: TOOZ_TEST_DRIVERS = etcd etcd3gw: TOOZ_TEST_ETCD3GW = 1 consul: TOOZ_TEST_DRIVERS = consul # NOTE(tonyb): This project has chosen to *NOT* consume upper-constraints.txt commands = {toxinidir}/run-tests.sh stestr run "{posargs}" {toxinidir}/run-examples.sh [testenv:venv] # This target is used by the gate go run Sphinx to build the doc deps = {[testenv:docs]deps} commands = {posargs} [testenv:cover] deps = {[testenv]deps} setenv = {[testenv]setenv} PYTHON=coverage run --source tooz --parallel-mode commands = {toxinidir}/run-tests.sh stestr run "{posargs}" {toxinidir}/run-examples.sh coverage combine coverage html -d cover coverage xml -o cover/coverage.xml [testenv:docs] deps = -r{toxinidir}/doc/requirements.txt commands = sphinx-build -W -b html doc/source doc/build/html [testenv:pep8] deps = hacking<2.1,>=2.0 doc8 commands = flake8 doc8 doc/source [flake8] exclude=.venv,.git,.tox,dist,*egg,*.egg-info,build,examples,doc show-source = True [testenv:releasenotes] deps = -r{toxinidir}/doc/requirements.txt commands = sphinx-build -a -E -W -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html