pax_global_header00006660000000000000000000000064131264447470014526gustar00rootroot0000000000000052 comment=0e1a9b8dd19ab9b9e409f60a4f4f10ff85511ceb pygtrie-2.2/000077500000000000000000000000001312644474700130545ustar00rootroot00000000000000pygtrie-2.2/.gitignore000066400000000000000000000000531312644474700150420ustar00rootroot00000000000000*.pyc .version MANIFEST build/ dist/ html/ pygtrie-2.2/.pylintrc000066400000000000000000000207161312644474700147270ustar00rootroot00000000000000[MASTER] # Specify a configuration file. #rcfile= # Python code to execute, usually for sys.path manipulation such as # pygtk.require(). #init-hook= # Profiled execution. profile=no # Add files or directories to the blacklist. They should be base names, not # paths. ignore=CVS # Pickle collected data for later comparisons. persistent=yes # List of plugins (as comma separated values of python modules names) to load, # usually to register additional checkers. load-plugins= [MESSAGES CONTROL] # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option # multiple time. See also the "--disable" option for examples. #enable= # Disable the message, report, category or checker with the given id(s). You # can either give multiple identifiers separated by comma (,) or put this # option multiple times (only on the command line, not in the configuration # file where it should appear only once).You can also use "--disable=all" to # disable everything first and then reenable specific checks. For example, if # you want to run only the similarities checker, you can use "--disable=all # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use"--disable=all --enable=classes # --disable=W" disable=locally-disabled,star-args,too-many-public-methods,too-many-lines [REPORTS] # Set the output format. Available formats are text, parseable, colorized, msvs # (visual studio) and html. You can also give a reporter class, eg # mypackage.mymodule.MyReporterClass. output-format=text # Put messages in a separate file for each module / package specified on the # command line instead of printing them on stdout. Reports (if any) will be # written in a file name "pylint_global.[txt|html]". files-output=no # Tells whether to display a full report or only the messages reports=no # Python expression which should return a note less than 10 (10 is the highest # note). You have access to the variables errors warning, statement which # respectively contain the number of errors / warnings messages and the total # number of statements analyzed. This is used by the global evaluation report # (RP0004). evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) # Add a comment according to your evaluation note. This is used by the global # evaluation report (RP0004). comment=no # Template used to display messages. This is a python new-style format string # used to format the message information. See doc for all details msg-template={path}:{line}: {msg} [{symbol}] [MISCELLANEOUS] # List of note tags to take in consideration, separated by a comma. notes=FIXME,XXX,TODO [FORMAT] # Maximum number of characters on a single line. max-line-length=80 # Regexp for a line that is allowed to be longer than the limit. ignore-long-lines=^\s*(# )??$ # Allow the body of an if to be on the same line as the test if there is no # else. single-line-if-stmt=no # List of optional constructs for which whitespace checking is disabled no-space-check= # String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 # tab). indent-string=' ' [TYPECHECK] # Tells whether missing members accessed in mixin class should be ignored. A # mixin class is detected if its name ends with "mixin" (case insensitive). ignore-mixin-members=yes # List of classes names for which member attributes should not be checked # (useful for classes with attributes dynamically set). ignored-classes=SQLObject # When zope mode is activated, add a predefined set of Zope acquired attributes # to generated-members. zope=no # List of members which are set dynamically and missed by pylint inference # system, and so shouldn't trigger E0201 when accessed. Python regular # expressions are accepted. generated-members=REQUEST,acl_users,aq_parent [VARIABLES] # Tells whether we should check for unused import in __init__ files. init-import=no # A regular expression matching the beginning of the name of dummy variables # (i.e. not used). dummy-variables-rgx=_$|unused_ # List of additional names supposed to be defined in builtins. Remember that # you should avoid to define new builtins when possible. additional-builtins= [SIMILARITIES] # Minimum lines number of a similarity. min-similarity-lines=4 # Ignore comments when computing similarities. ignore-comments=yes # Ignore docstrings when computing similarities. ignore-docstrings=yes # Ignore imports when computing similarities. ignore-imports=no [BASIC] # Required attributes for module, separated by a comma required-attributes= # List of builtins function names that should not be used, separated by a comma bad-functions=map,filter,apply,input # Regular expression which should only match correct module names module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ # Regular expression which should only match correct module level names const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ # Regular expression which should only match correct class names class-rgx=[A-Z_][a-zA-Z0-9]+$ # Regular expression which should only match correct function names function-rgx=[a-z_][a-z0-9_]{2,30}$ # Regular expression which should only match correct method names method-rgx=[a-z_][a-z0-9_]{2,30}$ # Regular expression which should only match correct instance attribute names attr-rgx=[a-z_][a-z0-9_]{2,30}$ # Regular expression which should only match correct argument names argument-rgx=[a-z_][a-z0-9_]{2,30}$ # Regular expression which should only match correct variable names variable-rgx=[a-z_][a-z0-9_]*$ # Regular expression which should only match correct attribute names in class # bodies class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ # Regular expression which should only match correct list comprehension / # generator expression variable names inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ # Good variable names which should always be accepted, separated by a comma good-names=i,j,k,ex,Run,_ # Bad variable names which should always be refused, separated by a comma bad-names=foo,bar,baz,toto,tutu,tata # Regular expression which should only match function or class names that do # not require a docstring. no-docstring-rgx=__.*__ # Minimum line length for functions/classes that require docstrings, shorter # ones are exempt. docstring-min-length=-1 [CLASSES] # List of interface methods to ignore, separated by a comma. This is used for # instance to not check methods defines in Zope's Interface base class. ignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by # List of method names used to declare (i.e. assign) instance attributes. defining-attr-methods=__init__,__new__,setUp # List of valid names for the first argument in a class method. valid-classmethod-first-arg=cls # List of valid names for the first argument in a metaclass class method. valid-metaclass-classmethod-first-arg=mcs [IMPORTS] # Deprecated modules which should not be used, separated by a comma deprecated-modules=regsub,TERMIOS,Bastion,rexec # Create a graph of every (i.e. internal and external) dependencies in the # given file (report RP0402 must not be disabled) import-graph= # Create a graph of external dependencies in the given file (report RP0402 must # not be disabled) ext-import-graph= # Create a graph of internal dependencies in the given file (report RP0402 must # not be disabled) int-import-graph= [DESIGN] # Maximum number of arguments for function / method max-args=5 # Argument names that match this expression will be ignored. Default to name # with leading underscore ignored-argument-names=_.* # Maximum number of locals for function / method body max-locals=15 # Maximum number of return / yield for function / method body max-returns=6 # Maximum number of branch for function / method body max-branches=12 # Maximum number of statements in function / method body max-statements=50 # Maximum number of parents for a class (see R0901). max-parents=7 # Maximum number of attributes for a class (see R0902). max-attributes=7 # Minimum number of public methods for a class (see R0903). min-public-methods=2 # Maximum number of public methods for a class (see R0904). max-public-methods=20 [EXCEPTIONS] # Exceptions that will emit a warning when being caught. Defaults to # "Exception" overgeneral-exceptions=Exception pygtrie-2.2/.rtd-requirements.txt000066400000000000000000000000361312644474700172040ustar00rootroot00000000000000sphinxcontrib-napoleon==0.5.4 pygtrie-2.2/LICENSE000066400000000000000000000261361312644474700140710ustar00rootroot00000000000000 Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. pygtrie-2.2/MANIFEST.in000066400000000000000000000001701312644474700146100ustar00rootroot00000000000000include test.py example.py include README.rst include conf.py index.rst version-history.rst include version.py .version pygtrie-2.2/README.rst000066400000000000000000000036271312644474700145530ustar00rootroot00000000000000pygtrie ======= .. image:: https://readthedocs.org/projects/pygtrie/badge/?version=latest :target: http://pygtrie.readthedocs.io/en/latest/ :alt: Documentation Status (latest) .. image:: https://readthedocs.org/projects/pygtrie/badge/?version=stable :target: http://pygtrie.readthedocs.io/en/stable/ :alt: Documentation Status (stable) pygtrie is a Python library implementing a trie data structure. `Trie data structure `_, also known as radix or prefix tree, is a tree associating keys to values where all the descendants of a node have a common prefix (associated with that node). The trie module contains ``Trie``, ``CharTrie`` and ``StringTrie`` classes each implementing a mutable mapping interface, i.e. ``dict`` interface. As such, in most circumstances, ``Trie`` could be used as a drop-in replacement for a ``dict``, but the prefix nature of the data structure is trie’s real strength. The module also contains ``PrefixSet`` class which uses a trie to store a set of prefixes such that a key is contained in the set if it or its prefix is stored in the set. Features -------- - A full mutable mapping implementation. - Supports iterating over as well as deleting a subtrie. - Supports prefix checking as well as shortest and longest prefix look-up. - Extensible for any kind of user-defined keys. - A PrefixSet supports “all keys starting with given prefix” logic. - Can store any value including None. Installation ------------ To install pygtrie, run:: pip install pygtrie Or download the sources and save ``pygtrie.py`` file with your project. Upgrading from 0.9.x -------------------- The 1.0 release introduced backwards incompatibility in naming. The module has been renamed from ``trie`` to ``pygtrie``. Fortunately, updating scripts using pygtrie should boil down to replacing:: from pytrie import trie with:: import pygtrie as trie pygtrie-2.2/conf.py000066400000000000000000000125551312644474700143630ustar00rootroot00000000000000# -*- coding: utf-8 -*- # # pygtrie documentation build configuration file, created by # sphinx-quickstart on Tue Feb 4 02:47:23 2014. # # This file is execfile()d with the current directory set to its containing dir. # # Note that not all possible configuration values are present in this # autogenerated file. # # All configuration values have a default; values that are commented out # serve to show the default. # -- General configuration ----------------------------------------------------- # If your documentation needs a minimal Sphinx version, state it here. #needs_sphinx = '1.0' import os import sys sys.path.insert(0, os.getcwd()) # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'sphinx.ext.viewcode', 'sphinxcontrib.napoleon'] # Add any paths that contain templates here, relative to this directory. templates_path = [] # The suffix of source filenames. source_suffix = '.rst' # The encoding of source files. source_encoding = 'utf-8-sig' # The master toctree document. master_doc = 'index' # General information about the project. project = u'pygtrie' copyright = u'2014, Google Inc.' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. #language = '' # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: #today = '' # Else, today_fmt is used as the format for a strftime call. #today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. with open('.gitignore') as fd: exclude_patterns = fd.read().splitlines() # The reST default role (used for this markup: `text`) to use for all documents. #default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. add_function_parentheses = False # If true, the current module name will be prepended to all description # unit titles (such as .. function::). add_module_names = False # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. #show_authors = False # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' # A list of ignored prefixes for module index sorting. #modindex_common_prefix = [] # -- Options for HTML output --------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. html_theme = 'classic' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. #html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. #html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". #html_title = None # A shorter title for the navigation bar. Default is the same as html_title. #html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. #html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. #html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = [] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. #html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. html_use_smartypants = True # Custom sidebar templates, maps document names to template names. #html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. #html_additional_pages = {} # If false, no module index is generated. #html_domain_indices = True # If false, no index is generated. #html_use_index = True # If true, the index is split into individual pages for each letter. #html_split_index = False # If true, links to the reST sources are added to the pages. #html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. #html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. #html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. html_use_opensearch = 'http://pythonhosted.org/pygtrie' # This is the file name suffix for HTML files (e.g. ".xhtml"). #html_file_suffix = None # Output file base name for HTML help builder. htmlhelp_basename = 'pygtriedoc' # Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = {'http://docs.python.org/': None} napoleon_google_docstring = True napoleon_include_init_with_doc = True napoleon_include_special_with_doc = True pygtrie-2.2/example.py000077500000000000000000000073671312644474700151010ustar00rootroot00000000000000#!/usr/bin/python """trie module example code.""" __author__ = 'Michal Nazarewicz ' __copyright__ = 'Copyright 2014 Google Inc.' # pylint: disable=invalid-name,superfluous-parens import os import stat import sys import pygtrie print('Storing file information in the trie') print('====================================\n') ROOT_DIR = '/usr/local' SUB_DIR = os.path.join(ROOT_DIR, 'lib') SUB_DIRS = tuple(os.path.join(ROOT_DIR, d) for d in ('lib', 'lib32', 'lib64', 'share')) t = pygtrie.StringTrie(separator=os.path.sep) # Read sizes regular files into a Trie for dirpath, unused_dirnames, filenames in os.walk(ROOT_DIR): for filename in filenames: filename = os.path.join(dirpath, filename) try: filestat = os.stat(filename) except OSError: continue if stat.S_IFMT(filestat.st_mode) == stat.S_IFREG: t[filename] = filestat.st_size # Size of all files we've scanned print('Size of %s: %d' % (ROOT_DIR, sum(t.itervalues()))) # Size of all files of a sub-directory print('Size of %s: %d' % (SUB_DIR, sum(t.itervalues(prefix=SUB_DIR)))) # Check existence of some directories for directory in SUB_DIRS: print(directory, 'exists' if t.has_subtrie(directory) else 'does not exist') # Drop a subtrie print('Dropping', SUB_DIR) del t[SUB_DIR:] print('Size of %s: %d' % (ROOT_DIR, sum(t.itervalues()))) for directory in SUB_DIRS: print(directory, 'exists' if t.has_subtrie(directory) else 'does not exist') print('\nStoring URL handlers map') print('========================\n') t = pygtrie.CharTrie() t['/'] = lambda url: sys.stdout.write('Root handler: %s\n' % url) t['/foo'] = lambda url: sys.stdout.write('Foo handler: %s\n' % url) t['/foobar'] = lambda url: sys.stdout.write('FooBar handler: %s\n' % url) t['/baz'] = lambda url: sys.stdout.write('Baz handler: %s\n' % url) for url in ('/', '/foo', '/foot', '/foobar', 'invalid', '/foobarbaz', '/ba'): key, handler = t.longest_prefix(url) if key is not None: handler(url) else: print('Unable to handle', repr(url)) if not os.isatty(0): sys.exit(0) try: import termios import tty def getch(): """Reads single character from standard input.""" attr = termios.tcgetattr(0) try: tty.setraw(0) return sys.stdin.read(1) finally: termios.tcsetattr(0, termios.TCSADRAIN, attr) except ImportError: try: from msvcrt import getch # pylint: disable=import-error except ImportError: sys.exit(0) print('\nPrefix set') print('==========\n') ps = pygtrie.PrefixSet(factory=pygtrie.StringTrie) ps.add('/etc/rc.d') ps.add('/usr/local/share') ps.add('/usr/local/lib') ps.add('/usr') # Will handle the above two as well ps.add('/usr/lib') # Does not change anything print('Path prefixes:', ', '.join(iter(ps))) for path in ('/etc', '/etc/rc.d', '/usr', '/usr/local', '/usr/local/lib'): print('Is', path, 'in the set:', ('yes' if path in ps else 'no')) print('\nDictionary test') print('===============\n') t = pygtrie.CharTrie() t['cat'] = True t['caterpillar'] = True t['car'] = True t['bar'] = True t['exit'] = False print('Start typing a word, "exit" to stop') print('(Other words you might want to try: %s)\n' % ', '.join(sorted( k for k in t if k != 'exit'))) text = '' while True: ch = getch() if ord(ch) < 32: print('Exiting') break text += ch value = t.get(text) if value is False: print('Exiting') break if value is not None: print(repr(text), 'is a word') if t.has_subtrie(text): print(repr(text), 'is a prefix of a word') else: print(repr(text), 'is not a prefix, going back to empty string') text = '' pygtrie-2.2/index.rst000066400000000000000000000014141312644474700147150ustar00rootroot00000000000000pygtrie ======= .. automodule:: pygtrie Installation ------------ To install pygtrie, run:: pip install pygtrie Or download the sources and save ``pygtrie.py`` file with your project. Upgrading from 0.9.x -------------------- The 1.0 release introduced backwards incompatibility in naming. The module has been renamed from ``trie`` to ``pygtrie``. Fortunately, updating scripts using pygtrie should boil down to replacing:: from pytrie import trie with:: import pygtrie as trie Trie classes ------------ .. autoclass:: pygtrie.Trie :members: .. autoclass:: pygtrie.CharTrie :members: .. autoclass:: pygtrie.StringTrie :members: PrefixSet class --------------- .. autoclass:: pygtrie.PrefixSet :members: .. include:: version-history.rst pygtrie-2.2/pygtrie.py000066400000000000000000001416061312644474700151210ustar00rootroot00000000000000# -*- coding: utf-8 -*- """Implementation of a trie data structure. `Trie data structure `_, also known as radix or prefix tree, is a tree associating keys to values where all the descendants of a node have a common prefix (associated with that node). The trie module contains :class:`pygtrie.Trie`, :class:`pygtrie.CharTrie` and :class:`pygtrie.StringTrie` classes each implementing a mutable mapping interface, i.e. :class:`dict` interface. As such, in most circumstances, :class:`pygtrie.Trie` could be used as a drop-in replacement for a :class:`dict`, but the prefix nature of the data structure is trie’s real strength. The module also contains :class:`pygtrie.PrefixSet` class which uses a trie to store a set of prefixes such that a key is contained in the set if it or its prefix is stored in the set. Features -------- - A full mutable mapping implementation. - Supports iterating over as well as deleting a subtrie. - Supports prefix checking as well as shortest and longest prefix look-up. - Extensible for any kind of user-defined keys. - A PrefixSet supports “all keys starting with given prefix” logic. - Can store any value including None. For some simple examples see ``example.py`` file. """ __author__ = 'Michal Nazarewicz ' __copyright__ = 'Copyright 2014 Google Inc.' import collections as _collections # Python 2.x and 3.x compatibility stuff if hasattr(dict, 'iteritems'): # pylint: disable=invalid-name _iteritems = lambda d: d.iteritems() _iterkeys = lambda d: d.iterkeys() def _sorted_iteritems(d): """Returns d's items in sorted order.""" items = d.items() items.sort() return iter(items) else: _sorted_iteritems = lambda d: sorted(d.items()) # pylint: disable=invalid-name _iteritems = lambda d: iter(d.items()) # pylint: disable=invalid-name _iterkeys = lambda d: iter(d.keys()) # pylint: disable=invalid-name class ShortKeyError(KeyError): """Raised when given key is a prefix of a longer key.""" pass _SENTINEL = object() class _Node(object): """A single node of a trie. Stores value associated with the node and dictionary of children. """ __slots__ = ('children', 'value') def __init__(self): self.children = {} self.value = _SENTINEL def iterate(self, path, shallow, iteritems): """Yields all the nodes with values associated to them in the trie. Args: path: Path leading to this node. Used to construct the key when returning value of this node and as a prefix for children. shallow: Perform a shallow traversal, i.e. do not yield nodes if their prefix has been yielded. iteritems: A function taking dictionary as argument and returning iterator over its items. Something other than dict.iteritems may be given to enable sorting. Yields: ``(path, value)`` tuples. """ # Use iterative function with stack on the heap so we don't hit Python's # recursion depth limits. node = self stack = [] while True: if node.value is not _SENTINEL: yield path, node.value if (not shallow or node.value is _SENTINEL) and node.children: stack.append(iter(iteritems(node.children))) path.append(None) while True: try: step, node = next(stack[-1]) path[-1] = step break except StopIteration: stack.pop() path.pop() except IndexError: return def traverse(self, node_factory, path_conv, path, iteritems): """Traverses the node and returns another type of node from factory. Args: node_factory: Callable function to construct new nodes. path_conv: Callable function to convert node path to a key. path: Current path for this node. iteritems: A function taking dictionary as argument and returning iterator over its items. Something other than dict.iteritems may be given to enable sorting. Returns: An object constructed by calling node_factory(path_conv, path, children, value=...), where children are constructed by node_factory from the children of this node. There doesn't need to be 1:1 correspondence between original nodes in the trie and constructed nodes (see make_test_node_and_compress in test.py). """ def children(): """Recursively traverses all of node's children.""" for step, node in iteritems(self.children): yield node.traverse(node_factory, path_conv, path + [step], iteritems) args = [path_conv, tuple(path), children()] if self.value is not _SENTINEL: args.append(self.value) return node_factory(*args) def __eq__(self, other): # Like iterate, we don't recurse so this works on deep tries. a, b = self, other stack = [] while True: if a.value != b.value or len(a.children) != len(b.children): return False if a.children: stack.append((_iteritems(a.children), b.children)) while True: try: key, a = next(stack[-1][0]) b = stack[-1][1].get(key) if b is None: return False break except StopIteration: stack.pop() except IndexError: return True return self.value == other.value and self.children == other.children def __ne__(self, other): return not self.__eq__(other) def __bool__(self): return bool(self.value is not _SENTINEL or self.children) __nonzero__ = __bool__ __hash__ = None def __getstate__(self): """Get state used for pickling. The state is encoded as a list of simple commands which consist of an integer and some command-dependent number of arguments. The commands modify what the current node is by navigating the trie up and down and setting node values. Possible commands are: * [n, step0, step1, ..., stepn-1, value], for n >= 0, specifies step needed to reach the next current node as well as its new value. There is no way to create a child node without setting its (or its descendant's) value. * [-n], for -n < 0, specifies to go up n steps in the trie. When encoded as a state, the commands are flattened into a single list. For example:: [ 0, 'Root', 2, 'Foo', 'Bar', 'Root/Foo/Bar Node', -1, 1, 'Baz', 'Root/Foo/Baz Node', -2, 1, 'Qux', 'Root/Qux Node' ] Creates the following hierarchy:: -* value: Root +-- Foo --* no value | +-- Bar -- * value: Root/Foo/Bar Node | +-- Baz -- * value: Root/Foo/Baz Node +-- Qux -- * value: Root/Qux Node Returns: A pickable state which can be passed to :func:`_Node.__setstate__` to reconstruct the node and its full hierarchy. """ # Like iterate, we don't recurse so pickling works on deep tries. state = [] if self.value is _SENTINEL else [0] last_cmd = 0 node = self stack = [] while True: if node.value is not _SENTINEL: last_cmd = 0 state.append(node.value) stack.append(_iteritems(node.children)) while True: try: step, node = next(stack[-1]) except StopIteration: if last_cmd < 0: state[-1] -= 1 else: last_cmd = -1 state.append(-1) stack.pop() continue except IndexError: if last_cmd < 0: state.pop() return state if last_cmd > 0: last_cmd += 1 state[-last_cmd] += 1 else: last_cmd = 1 state.append(1) state.append(step) break def __setstate__(self, state): """Unpickles node. See :func:`_Node.__getstate__`.""" self.__init__() state = iter(state) stack = [self] for cmd in state: if cmd < 0: del stack[cmd:] else: while cmd > 0: stack.append(type(self)()) stack[-2].children[next(state)] = stack[-1] cmd -= 1 stack[-1].value = next(state) _NONE_PAIR = type('NonePair', (tuple,), { '__nonzero__': lambda _: False, '__bool__': lambda _: False, '__slots__': (), })((None, None)) class Trie(_collections.MutableMapping): """A trie implementation with dict interface plus some extensions. Keys used with the :class:`pygtrie.Trie` must be iterable, yielding hashable objects. In other words, for a given key, ``dict.fromkeys(key)`` must be valid. In particular, strings work fine as trie keys, however when getting keys back from iterkeys() method for example, instead of strings, tuples of characters are produced. For that reason, :class:`pygtrie.CharTrie` or :class:`pygtrie.StringTrie` may be preferred when using :class:`pygtrie.Trie` with string keys. """ def __init__(self, *args, **kwargs): """Initialises the trie. Arguments are interpreted the same way :func:`Trie.update` interprets them. """ self._root = _Node() self._sorted = False self.update(*args, **kwargs) @property def _iteritems(self): """Returns function yielding over dict's items possibly in sorted order. Returns: A function iterating over items of a dictionary given as an argument. If child nodes sorting has been enabled (via :func:`Trie.enable_sorting` method), returned function will go through the items in sorted order.. """ return _sorted_iteritems if self._sorted else _iteritems def enable_sorting(self, enable=True): """Enables sorting of child nodes when iterating and traversing. Normally, child nodes are not sorted when iterating or traversing over the trie (just like dict elements are not sorted). This method allows sorting to be enabled (which was the behaviour prior to pygtrie 2.0 release). For Trie class, enabling sorting of children is identical to simply sorting the list of items since Trie returns keys as tuples. However, for other implementations such as StringTrie the two may behove subtly different. For example, sorting items might produce:: root/foo-bar root/foo/baz even though foo comes before foo-bar. Args: enable: Whether to enable sorting of child nodes. """ self._sorted = enable def clear(self): """Removes all the values from the trie.""" self._root = _Node() def update(self, *args, **kwargs): """Updates stored values. Works like :func:`dict.update`.""" if len(args) > 1: raise ValueError('update() takes at most one positional argument, ' '%d given.' % len(args)) # We have this here instead of just letting MutableMapping.update() # handle things because it will iterate over keys and for each key # retrieve the value. With Trie, this may be expensive since the path # to the node would have to be walked twice. Instead, we have our own # implementation where iteritems() is used avoiding the unnecessary # value look-up. if args and isinstance(args[0], Trie): for key, value in _iteritems(args[0]): self[key] = value args = () super(Trie, self).update(*args, **kwargs) def copy(self): """Returns a shallow copy of the trie.""" return self.__class__(self) @classmethod def fromkeys(cls, keys, value=None): """Creates a new trie with given keys set. This is roughly equivalent to calling the constructor with a ``(key, value) for key in keys`` generator. Args: keys: An iterable of keys that should be set in the new trie. value: Value to associate with given keys. Returns: A new trie where each key from ``keys`` has been set to the given value. """ trie = cls() for key in keys: trie[key] = value return trie def _get_node(self, key, create=False): """Returns node for given key. Creates it if requested. Args: key: A key to look for. create: Whether to create the node if it does not exist. Returns: ``(node, trace)`` tuple where ``node`` is the node for given key and ``trace`` is a list specifying path to reach the node including all the encountered nodes. Each element of trace is a ``(step, node)`` tuple where ``step`` is a step from parent node to given node and ``node`` is node on the path. The first element of the path is always ``(None, self._root)``. Raises: KeyError: If there is no node for the key and ``create`` is ``False``. """ node = self._root trace = [(None, node)] for step in self.__path_from_key(key): if create: node = node.children.setdefault(step, _Node()) else: node = node.children.get(step) if not node: raise KeyError(key) trace.append((step, node)) return node, trace def __iter__(self): return self.iterkeys() # pylint: disable=arguments-differ def iteritems(self, prefix=_SENTINEL, shallow=False): """Yields all nodes with associated values with given prefix. Only nodes with values are output. For example:: >>> import pygtrie >>> t = pygtrie.StringTrie() >>> t['foo'] = 'Foo' >>> t['foo/bar/baz'] = 'Baz' >>> t['qux'] = 'Qux' >>> t.items() [('qux', 'Qux'), ('foo', 'Foo'), ('foo/bar/baz', 'Baz')] Items are generated in topological order but the order of siblings is unspecified by default. In other words, in the above example, the ``('qux', 'Qux')`` pair might have been at the end of the list. At an expense of efficiency, this can be changed via :func:`Trie.enable_sorting`. With ``prefix`` argument, only items with specified prefix are generated (i.e. only given subtrie is traversed) as demonstrated by:: >>> t.items(prefix='foo/bar') [('foo/bar/baz', 'Baz')] With ``shallow`` argument, if a node has value associated with it, it's children are not traversed even if they exist which can be seen in:: >>> t.items(shallow=True) [('qux', 'Qux'), ('foo', 'Foo')] Args: prefix: Prefix to limit iteration to. shallow: Perform a shallow traversal, i.e. do not yield items if their prefix has been yielded. Yields: ``(key, value)`` tuples. Raises: KeyError: If ``prefix`` does not match any node. """ node, _ = self._get_node(prefix) for path, value in node.iterate(list(self.__path_from_key(prefix)), shallow, self._iteritems): yield (self._key_from_path(path), value) def iterkeys(self, prefix=_SENTINEL, shallow=False): """Yields all keys having associated values with given prefix. This is equivalent to taking first element of tuples generated by :func:`Trie.iteritems` which see for more detailed documentation. Args: prefix: Prefix to limit iteration to. shallow: Perform a shallow traversal, i.e. do not yield keys if their prefix has been yielded. Yields: All the keys (with given prefix) with associated values in the trie. Raises: KeyError: If ``prefix`` does not match any node. """ for key, _ in self.iteritems(prefix=prefix, shallow=shallow): yield key def itervalues(self, prefix=_SENTINEL, shallow=False): """Yields all values associated with keys with given prefix. This is equivalent to taking second element of tuples generated by :func:`Trie.iteritems` which see for more detailed documentation. Args: prefix: Prefix to limit iteration to. shallow: Perform a shallow traversal, i.e. do not yield values if their prefix has been yielded. Yields: All the values associated with keys (with given prefix) in the trie. Raises: KeyError: If ``prefix`` does not match any node. """ node, _ = self._get_node(prefix) for _, value in node.iterate(list(self.__path_from_key(prefix)), shallow, self._iteritems): yield value def items(self, prefix=_SENTINEL, shallow=False): """Returns a list of ``(key, value)`` pairs in given subtrie. This is equivalent to constructing a list from generator returned by :func:`Trie.iteritems` which see for more detailed documentation. """ return list(self.iteritems(prefix=prefix, shallow=shallow)) def keys(self, prefix=_SENTINEL, shallow=False): """Returns a list of all the keys, with given prefix, in the trie. This is equivalent to constructing a list from generator returned by :func:`Trie.iterkeys` which see for more detailed documentation. """ return list(self.iterkeys(prefix=prefix, shallow=shallow)) def values(self, prefix=_SENTINEL, shallow=False): """Returns a list of values in given subtrie. This is equivalent to constructing a list from generator returned by :func:`Trie.iterivalues` which see for more detailed documentation. """ return list(self.itervalues(prefix=prefix, shallow=shallow)) # pylint: enable=arguments-differ def __len__(self): """Returns number of values in a trie. Note that this method is expensive as it iterates over the whole trie. """ return sum(1 for _ in self.itervalues()) def __nonzero__(self): return bool(self._root) HAS_VALUE = 1 HAS_SUBTRIE = 2 def has_node(self, key): """Returns whether given node is in the trie. Return value is a bitwise or of ``HAS_VALUE`` and ``HAS_SUBTRIE`` constants indicating node has a value associated with it and that it is a prefix of another existing key respectively. Both of those are independent of each other and all of the four combinations are possible. For example:: >>> import pygtrie >>> t = pygtrie.StringTrie() >>> t['foo/bar'] = 'Bar' >>> t['foo/bar/baz'] = 'Baz' >>> t.has_node('qux') == 0 True >>> t.has_node('foo/bar/baz') == pygtrie.Trie.HAS_VALUE True >>> t.has_node('foo') == pygtrie.Trie.HAS_SUBTRIE True >>> t.has_node('foo/bar') == (pygtrie.Trie.HAS_VALUE | ... pygtrie.Trie.HAS_SUBTRIE) True There are two higher level methods built on top of this one which give easier interface for the information. :func:`Trie.has_key` and returns whether node has a value associated with it and :func:`Trie.has_subtrie` checks whether node is a prefix. Continuing previous example:: >>> t.has_key('qux'), t.has_subtrie('qux') False, False >>> t.has_key('foo/bar/baz'), t.has_subtrie('foo/bar/baz') True, False >>> t.has_key('foo'), t.has_subtrie('foo') False, True >>> t.has_key('foo/bar'), t.has_subtrie('foo/bar') True, True Args: key: A key to look for. Returns: Non-zero if node exists and if it does a bit-field denoting whether it has a value associated with it and whether it has a subtrie. """ try: node, _ = self._get_node(key) except KeyError: return 0 return ((self.HAS_VALUE * int(node.value is not _SENTINEL)) | (self.HAS_SUBTRIE * int(bool(node.children)))) def has_key(self, key): """Indicates whether given key has value associated with it. See :func:`Trie.has_node` for more detailed documentation. """ return bool(self.has_node(key) & self.HAS_VALUE) def has_subtrie(self, key): """Returns whether given key is a prefix of another key in the trie. See :func:`Trie.has_node` for more detailed documentation. """ return bool(self.has_node(key) & self.HAS_SUBTRIE) @staticmethod def _slice_maybe(key_or_slice): """Checks whether argument is a slice or a plain key. Args: key_or_slice: A key or a slice to test. Returns: ``(key, is_slice)`` tuple. ``is_slice`` indicates whether ``key_or_slice`` is a slice and ``key`` is either ``key_or_slice`` itself (if it's not a slice) or slice's start position. Raises: TypeError: If ``key_or_slice`` is a slice whose stop or step are not ``None`` In other words, only ``[key:]`` slices are valid. """ if isinstance(key_or_slice, slice): if key_or_slice.stop is not None or key_or_slice.step is not None: raise TypeError(key_or_slice) return key_or_slice.start, True return key_or_slice, False def __getitem__(self, key_or_slice): """Returns value associated with given key or raises KeyError. When argument is a single key, value for that key is returned (or :class:`KeyError` exception is thrown if the node does not exist or has no value associated with it). When argument is a slice, it must be one with only `start` set in which case the access is identical to :func:`Trie.itervalues` invocation with prefix argument. Example: >>> import pygtrie >>> t = pygtrie.StringTrie() >>> t['foo/bar'] = 'Bar' >>> t['foo/baz'] = 'Baz' >>> t['qux'] = 'Qux' >>> t['foo/bar'] 'Bar' >>> list(t['foo':]) ['Baz', 'Bar'] >>> t['foo'] Traceback (most recent call last): ... pygtrie.ShortKeyError: 'foo' Args: key_or_slice: A key or a slice to look for. Returns: If a single key is passed, a value associated with given key. If a slice is passed, a generator of values in specified subtrie. Raises: ShortKeyError: If the key has no value associated with it but is a prefix of some key with a value. Note that :class:`ShortKeyError` is subclass of :class:`KeyError`. KeyError: If key has no value associated with it nor is a prefix of an existing key. TypeError: If ``key_or_slice`` is a slice but it's stop or step are not ``None``. """ if self._slice_maybe(key_or_slice)[1]: return self.itervalues(key_or_slice.start) node, _ = self._get_node(key_or_slice) if node.value is _SENTINEL: raise ShortKeyError(key_or_slice) return node.value def _set(self, key, value, only_if_missing=False, clear_children=False): """Sets value for a given key. Args: key: Key to set value of. value: Value to set to. only_if_missing: If ``True``, value won't be changed if the key is already associated with a value. clear_children: If ``True``, all children of the node, if any, will be removed. Returns: Value of the node. """ node, _ = self._get_node(key, create=True) if not only_if_missing or node.value is _SENTINEL: node.value = value if clear_children: node.children.clear() return node.value def __setitem__(self, key_or_slice, value): """Sets value associated with given key. If `key_or_slice` is a key, simply associate it with given value. If it is a slice (which must have `start` set only), it in addition clears any subtrie that might have been attached to particular key. For example:: >>> import pygtrie >>> t = pygtrie.StringTrie() >>> t['foo/bar'] = 'Bar' >>> t['foo/baz'] = 'Baz' >>> t.keys() ['foo/baz', 'foo/bar'] >>> t['foo':] = 'Foo' >>> t.keys() ['foo'] Args: key_or_slice: A key to look for or a slice. If it is a slice, the whole subtrie (if present) will be replaced by a single node with given value set. value: Value to set. Raises: TypeError: If key is a slice whose stop or step are not None. """ key, is_slice = self._slice_maybe(key_or_slice) self._set(key, value, clear_children=is_slice) def setdefault(self, key, value=None): """Sets value of a given node if not set already. Also returns it. In contrast to :func:`Trie.__setitem__`, this method does not accept slice as a key. """ return self._set(key, value, only_if_missing=True) @staticmethod def _cleanup_trace(trace): """Removes empty nodes present on specified trace. Args: trace: Trace to the node to cleanup as returned by :func:`Trie._get_node`. """ i = len(trace) - 1 # len(path) >= 1 since root is always there step, node = trace[i] while i and not node: i -= 1 parent_step, parent = trace[i] del parent.children[step] step, node = parent_step, parent def _pop_from_node(self, node, trace, default=_SENTINEL): """Removes a value from given node. Args: node: Node to get value of. trace: Trace to that node as returned by :func:`Trie._get_node`. default: A default value to return if node has no value set. Returns: Value of the node or ``default``. Raises: ShortKeyError: If the node has no value associated with it and ``default`` has not been given. """ if node.value is not _SENTINEL: value = node.value node.value = _SENTINEL self._cleanup_trace(trace) return value elif default is _SENTINEL: raise ShortKeyError() else: return default def pop(self, key, default=_SENTINEL): """Deletes value associated with given key and returns it. Args: key: A key to look for. default: If specified, value that will be returned if given key has no value associated with it. If not specified, method will throw KeyError in such cases. Returns: Removed value, if key had value associated with it, or ``default`` (if given). Raises: ShortKeyError: If ``default`` has not been specified and the key has no value associated with it but is a prefix of some key with a value. Note that :class:`ShortKeyError` is subclass of :class:`KeyError`. KeyError: If default has not been specified and key has no value associated with it nor is a prefix of an existing key. """ try: return self._pop_from_node(*self._get_node(key)) except KeyError: if default is not _SENTINEL: return default raise def popitem(self): """Deletes an arbitrary value from the trie and returns it. There is no guarantee as to which item is deleted and returned. Neither in respect to its lexicographical nor topological order. Returns: ``(key, value)`` tuple indicating deleted key. Raises: KeyError: If the trie is empty. """ if not self: raise KeyError() node = self._root trace = [(None, node)] while node.value is _SENTINEL: step = next(_iterkeys(node.children)) node = node.children[step] trace.append((step, node)) return (self._key_from_path((step for step, _ in trace[1:])), self._pop_from_node(node, trace)) def __delitem__(self, key_or_slice): """Deletes value associated with given key or raises KeyError. If argument is a key, value associated with it is deleted. If the key is also a prefix, its descendents are not affected. On the other hand, if the argument is a slice (in which case it must have only start set), the whole subtrie is removed. For example:: >>> import pygtrie >>> t = pygtrie.StringTrie() >>> t['foo'] = 'Foo' >>> t['foo/bar'] = 'Bar' >>> t['foo/bar/baz'] = 'Baz' >>> del t['foo/bar'] >>> t.keys() ['foo', 'foo/bar/baz'] >>> del t['foo':] >>> t.keys() [] Args: key_or_slice: A key to look for or a slice. If key is a slice, the whole subtrie will be removed. Raises: ShortKeyError: If the key has no value associated with it but is a prefix of some key with a value. This is not thrown is key_or_slice is a slice -- in such cases, the whole subtrie is removed. Note that :class:`ShortKeyError` is subclass of :class:`KeyError`. KeyError: If key has no value associated with it nor is a prefix of an existing key. TypeError: If key is a slice whose stop or step are not ``None``. """ key, is_slice = self._slice_maybe(key_or_slice) node, trace = self._get_node(key) if is_slice: node.children.clear() elif node.value is _SENTINEL: raise ShortKeyError(key) node.value = _SENTINEL self._cleanup_trace(trace) def prefixes(self, key): """Walks towards the node specified by key and yields all found items. Example: >>> import pygtrie >>> t = pygtrie.StringTrie() >>> t['foo'] = 'Foo' >>> t['foo/bar/baz'] = 'Baz' >>> list(t.prefixes('foo/bar/baz/qux')) [('foo', 'Foo'), ('foo/bar/baz', 'Baz')] >>> list(t.prefixes('does/not/exist')) [] Args: key: Key to look for. Yields: ``(k, value)`` pairs denoting keys with associated values encountered on the way towards the specified key. """ node = self._root path = self.__path_from_key(key) pos = 0 while True: if node.value is not _SENTINEL: yield self._key_from_path(path[:pos]), node.value if pos == len(path): break node = node.children.get(path[pos]) if not node: break pos += 1 def shortest_prefix(self, key): """Finds the shortest prefix of a key with a value. This is equivalent to taking the first object yielded by :func:`Trie.prefixes` with a default of `(None, None)` if said method yields no items. As an added bonus, the pair in that case will be a falsy value (as opposed to regular two-element tuple of ``None`` values). Example: >>> import pygtrie >>> t = pygtrie.StringTrie() >>> t['foo'] = 'Foo' >>> t['foo/bar/baz'] = 'Baz' >>> t.shortest_prefix('foo/bar/baz/qux') ('foo', 'Foo') >>> t.shortest_prefix('does/not/exist') (None, None) >>> bool(t.shortest_prefix('does/not/exist')) False Args: key: Key to look for. Returns: ``(k, value)`` where ``k`` is the shortest prefix of ``key`` (it may equal ``key``) and ``value`` is a value associated with that key. If no node is found, ``(None, None)`` is returned. """ return next(self.prefixes(key), _NONE_PAIR) def longest_prefix(self, key): """Finds the longest prefix of a key with a value. This is equivalent to taking the last object yielded by :func:`Trie.prefixes` with a default of `(None, None)` if said method yields no items. As an added bonus, the pair in that case will be a falsy value (as opposed to regular two-element tuple of ``None`` values). Example: >>> import pygtrie >>> t = pygtrie.StringTrie() >>> t['foo'] = 'Foo' >>> t['foo/bar/baz'] = 'Baz' >>> t.longest_prefix('foo/bar/baz/qux') ('foo/bar/baz', 'Baz') >>> t.longest_prefix('does/not/exist') (None, None) >>> bool(t.longest_prefix('does/not/exist')) False Args: key: Key to look for. Returns: ``(k, value)`` where ``k`` is the longest prefix of ``key`` (it may equal ``key``) and ``value`` is a value associated with that key. If no node is found, ``(None, None)`` is returned. """ ret = _NONE_PAIR for ret in self.prefixes(key): pass return ret def __eq__(self, other): return self._root == other._root # pylint: disable=protected-access def __ne__(self, other): return self._root != other._root # pylint: disable=protected-access def __str__(self): return 'Trie(%s)' % ( ', '.join('%s: %s' % item for item in self.iteritems())) def __repr__(self): if self: return 'Trie((%s,))' % ( ', '.join('(%r, %r)' % item for item in self.iteritems())) else: return 'Trie()' def __path_from_key(self, key): """Converts a user visible key object to internal path representation. Args: key: User supplied key or ``_SENTINEL``. Returns: An empty tuple if ``key`` was ``_SENTINEL``, otherwise whatever :func:`Trie._path_from_key` returns. Raises: TypeError: If ``key`` is of invalid type. """ return () if key is _SENTINEL else self._path_from_key(key) def _path_from_key(self, key): # pylint: disable=no-self-use """Converts a user visible key object to internal path representation. The default implementation simply returns key. Args: key: User supplied key. Returns: A path, which is an iterable of steps. Each step must be hashable. Raises: TypeError: If key is of invalid type. """ return key def _key_from_path(self, path): # pylint: disable=no-self-use """Converts an internal path into a user visible key object. The default implementation creates a tuple from the path. Args: path: Internal path representation. Returns: A user visible key object. """ return tuple(path) def traverse(self, node_factory, prefix=_SENTINEL): """Traverses the tree using node_factory object. node_factory is a callable function which accepts (path_conv, path, children, value=...) arguments, where path_conv is a lambda converting path representation to key, path is the path to this node, children is an iterable of children nodes constructed by node_factory, optional value is the value associated with the path. node_factory's children argument is a generator which has a few consequences: * To traverse into node's children, the generator must be iterated over. This can by accomplished by a simple "children = list(children)" statement. * Ignoring the argument allows node_factory to stop the traversal from going into the children of the node. In other words, whole subtrie can be removed from traversal if node_factory chooses so. * If children is stored as is (i.e. as a generator) when it is iterated over later on it will see state of the trie as it is during the iteration and not when traverse method was called. :func:`Trie.traverse` has two advantages over :func:`Trie.iteritems` and similar methods: 1. it allows subtries to be skipped completely when going through the list of nodes based on the property of the parent node; and 2. it represents structure of the trie directly making it easy to convert structure into a different representation. For example, the below snippet prints all files in current directory counting how many HTML files were found but ignores hidden files and directories (i.e. those whose names start with a dot):: import os import pygtrie t = pygtrie.StringTrie(separator=os.sep) # Construct a trie with all files in current directory and all # of its sub-directories. Files get set a True value. # Directories are represented implicitly by being prefixes of # files. for root, _, files in os.walk('.'): for name in files: t[os.path.join(root, name)] = True def traverse_callback(path_conv, path, children, is_file=False): if path and path[-1] != '.' and path[-1][0] == '.': # Ignore hidden directory (but accept root node and '.') return 0 elif is_file: print path_conv(path) return int(path[-1].endswith('.html')) else: # Otherwise, it's a directory. Traverse into children. return sum(int(is_html) for is_html in children) print t.traverse(traverse_callback) As documented, ignoring the children argument causes subtrie to be omitted and not walked into. In the next example, the trie is converted to a tree representation where child nodes include a pointer to their parent. As before, hidden files and directories are ignored:: import os import pygtrie t = pygtrie.StringTrie(separator=os.sep) for root, _, files in os.walk('.'): for name in files: t[os.path.join(root, name)] = True class File(object): def __init__(self, name): self.name = name self.parent = None class Directory(File): def __init__(self, name, children): super(Directory, self).__init__(name) self._children = children for child in children: child.parent = self def traverse_callback(path_conv, path, children, is_file=False): if not path or path[-1] == '.' or path[-1][0] != '.': if is_file: return File(path[-1]) children = filter(None, children) return Directory(path[-1] if path else '', children) root = t.traverse(traverse_callback) Note: Unlike iterators, traverse method uses stack recursion which means that using it on deep tries may lead to a RuntimeError exception thrown once Python's maximum recursion depth is reached. Args: node_factory: Makes opaque objects from the keys and values of the trie. prefix: Prefix for node to start traversal, by default starts at root. Returns: Node object constructed by node_factory corresponding to the root node. """ node, _ = self._get_node(prefix) return node.traverse(node_factory, self._key_from_path, list(self.__path_from_key(prefix)), self._iteritems) class CharTrie(Trie): """A variant of a :class:`pygtrie.Trie` which accepts strings as keys. The only difference between :class:`pygtrie.CharTrie` and :class:`pygtrie.Trie` is that when :class:`pygtrie.CharTrie` returns keys back to the client (for instance in keys() method is called), those keys are returned as strings. Canonical example where this class can be used is a dictionary of words in a natural language. For example:: >>> import pygtrie >>> t = pygtrie.CharTrie() >>> t['wombat'] = True >>> t['woman'] = True >>> t['man'] = True >>> t['manhole'] = True >>> t.has_subtrie('wo') True >>> t.has_key('man') True >>> t.has_subtrie('man') True >>> t.has_subtrie('manhole') False """ def _key_from_path(self, path): return ''.join(path) class StringTrie(Trie): """:class:`pygtrie.Trie` variant accepting strings with a separator as keys. The trie accepts strings as keys which are split into components using a separator specified during initialisation ("/" by default). Canonical example where this class can be used is when keys are paths. For example, it could map from a path to a request handler:: import pygtrie def handle_root(): pass def handle_admin(): pass def handle_admin_images(): pass handlers = pygtrie.StringTrie() handlers[''] = handle_root handlers['/admin'] = handle_admin handlers['/admin/images'] = handle_admin_images request_path = '/admin/images/foo' handler = handlers.longest_prefix(request_path) """ def __init__(self, *args, **kwargs): """Initialises the trie. Except for a ``separator`` named argument, all other arguments are interpreted the same way :func:`Trie.update` interprets them. Args: *args: Passed to super class initialiser. **kwargs: Passed to super class initialiser. separator: A separator to use when splitting keys into paths used by the trie. "/" is used if this argument is not specified. This named argument is not specified on the function's prototype because of Python's limitations. """ self._separator = kwargs.pop('separator', '/') super(StringTrie, self).__init__(*args, **kwargs) @classmethod def fromkeys(cls, keys, value=None, separator='/'): # pylint: disable=arguments-differ trie = cls(separator=separator) for key in keys: trie[key] = value return trie def _path_from_key(self, key): return key.split(self._separator) def _key_from_path(self, path): return self._separator.join(path) class PrefixSet(_collections.MutableSet): # pylint: disable=abstract-class-not-used """A set of prefixes. :class:`pygtrie.PrefixSet` works similar to a normal set except it is said to contain a key if the key or it's prefix is stored in the set. For instance, if "foo" is added to the set, the set contains "foo" as well as "foobar". The set supports addition of elements but does *not* support removal of elements. This is because there's no obvious consistent and intuitive behaviour for element deletion. """ def __init__(self, iterable=None, factory=Trie, **kwargs): """Initialises the prefix set. Args: iterable: A sequence of keys to add to the set. factory: A function used to create a trie used by the :class:`pygtrie.PrefixSet`. kwargs: Additional keyword arguments passed to the factory function. """ super(PrefixSet, self).__init__() trie = factory(**kwargs) if iterable: trie.update((key, True) for key in iterable) self._trie = trie def copy(self): """Returns a copy of the prefix set.""" return self.__class__(self._trie) def clear(self): """Removes all keys from the set.""" self._trie.clear() def __contains__(self, key): """Checks whether set contains key or its prefix.""" return bool(self._trie.shortest_prefix(key)[1]) def __iter__(self): """Return iterator over all prefixes in the set. See :func:`PrefixSet.iter` method for more info. """ return self._trie.iterkeys() def iter(self, prefix=_SENTINEL): """Iterates over all keys in the set optionally starting with a prefix. Since a key does not have to be explicitly added to the set to be an element of the set, this method does not iterate over all possible keys that the set contains, but only over the shortest set of prefixes of all the keys the set contains. For example, if "foo" has been added to the set, the set contains also "foobar", but this method will *not* iterate over "foobar". If ``prefix`` argument is given, method will iterate over keys with given prefix only. The keys yielded from the function if prefix is given does not have to be a subset (in mathematical sense) of the keys yielded when there is not prefix. This happens, if the set contains a prefix of the given prefix. For example, if only "foo" has been added to the set, iter method called with no arguments will yield "foo" only. However, when called with "foobar" argument, it will yield "foobar" only. """ if prefix is _SENTINEL: return iter(self) elif self._trie.has_node(prefix): return self._trie.iterkeys(prefix=prefix) elif prefix in self: # Make sure the type of returned keys is consistent. # pylint: disable=protected-access return self._trie._key_from_path(self._trie._path_from_key(prefix)), else: return () def __len__(self): """Returns number of keys stored in the set. Since a key does not have to be explicitly added to the set to be an element of the set, this method does not count over all possible keys that the set contains (since that would be infinity), but only over the shortest set of prefixes of all the keys the set contains. For example, if "foo" has been added to the set, the set contains also "foobar", but this method will *not* count "foobar". """ return len(self._trie) def add(self, key): """Adds given key to the set. If the set already contains prefix of the key being added, this operation has no effect. If the key being added is a prefix of some existing keys in the set, those keys are deleted and replaced by a single entry for the key being added. For example, if the set contains key "foo" adding a key "foobar" does not change anything. On the other hand, if the set contains keys "foobar" and "foobaz", adding a key "foo" will replace those two keys with a single key "foo". This makes a difference when iterating over the keys or counting number of keys. Counter intuitively, adding of a key can *decrease* size of the set. Args: key: Key to add. """ if key not in self: self._trie[key:] = True def discard(self, key): raise NotImplementedError( 'Removing keys from PrefixSet is not implemented.') def remove(self, key): raise NotImplementedError( 'Removing keys from PrefixSet is not implemented.') def pop(self): raise NotImplementedError( 'Removing keys from PrefixSet is not implemented.') pygtrie-2.2/setup.py000066400000000000000000000031111312644474700145620ustar00rootroot00000000000000from distutils.core import setup import os import re import sys import codecs import version release = version.get_version() if len(sys.argv) == 2 and sys.argv[1] == 'builddoc': os.execlp('sphinx-build', '-Drelease=' + release, '-Dversion=' + '.'.join(release.split('.', 2)[0:2]), '.', 'html') with codecs.open('README.rst', 'r', 'utf-8') as f: readme = f.read() with codecs.open('version-history.rst', 'r', 'utf-8') as f: readme += '\n' + f.read() kwargs = { 'name': 'pygtrie', 'version': release, 'description': 'Trie data structure implementation.', 'long_description': readme, 'author': 'Michal Nazarewicz', 'author_email': 'mina86@mina86.com', 'url': 'https://github.com/google/pygtrie', 'py_modules': ['pygtrie'], 'license': 'Apache-2.0', 'platforms': 'Platform Independent', 'keywords': ['trie', 'prefix tree', 'data structure'], 'classifiers': [ 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', 'License :: OSI Approved :: Apache Software License', 'Operating System :: OS Independent', 'Programming Language :: Python', 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Topic :: Software Development :: Libraries :: Python Modules', ], } if re.search(r'(?:\d+\.)*\d+', release): kwargs['download_url'] = kwargs['url'] + '/tarball/v' + release setup(**kwargs) pygtrie-2.2/test.py000077500000000000000000000503101312644474700144070ustar00rootroot00000000000000#!/usr/bin/python """trie module unit tests.""" __author__ = 'Michal Nazarewicz ' __copyright__ = 'Copyright 2014 Google Inc.' import array import collections import pickle import unittest import pygtrie # pylint: disable=missing-docstring def _update_trie_factory(trie_cls, *args, **kw): t = trie_cls() t.update(*args, **kw) return t def _setter_trie_factory(trie_cls, d): # pylint: disable=invalid-name t = trie_cls() for k, v in d.items(): t[k] = v return t _TRIE_FACTORIES = (( 'TrieFromNamedArgs', lambda trie_cls, d: trie_cls(**d) ), ( 'TrieFromTuples', lambda trie_cls, d: trie_cls(d.items()) ), ( 'TrieFromDict', lambda trie_cls, d: trie_cls(d) ), ( 'TrieFromTrie', lambda trie_cls, d: trie_cls(trie_cls(d)) ), ( 'CopyOfTrie', lambda trie_cls, d: trie_cls(d).copy() ), ( 'UpdateWithNamedArgs', lambda trie_cls, d: _update_trie_factory(trie_cls, **d) ), ( 'UpdateWithTuples', lambda trie_cls, d: _update_trie_factory(trie_cls, d.items()) ), ( 'UpdateWithDict', _update_trie_factory ), ( 'Setters', _setter_trie_factory )) class TrieTestCase(unittest.TestCase): # The below need to be overwritten by subclasses # A Trie class being tested _TRIE_CLS = pygtrie.Trie # A key to set _SHORT_KEY = 'foo' # Another key to set such that _SHORT_KEY is it's prefix _LONG_KEY = _SHORT_KEY + 'bar' # A key that is not set but _LONG_KEY is it's prefix _VERY_LONG_KEY = _LONG_KEY + 'baz' # A key that is not set and has no relation to other keys _OTHER_KEY = 'qux' # A list of prefixes of _SHORT_KEY _SHORT_PREFIXES = ('', 'f', 'fo') # A list of prefixes of _LONG_KEY which are not prefixes of _SHORT_KEY nor # _SHORT_KEY itself _LONG_PREFIXES = ('foob', 'fooba') @classmethod def path_from_key(cls, key): """Turns key into a path as used by Trie class being tested.""" return key @classmethod def key_from_path(cls, path): """Turns path as used by Trie class being tested into a key.""" return tuple(path) # End of stuff that needs to be overwritten by subclasses def key_from_key(self, key): """Turns a key into a form that the Trie will return e.g. in keys().""" return self.key_from_path(self.path_from_key(key)) # pylint: disable=invalid-name def assertNodeState(self, t, key, prefix=False, value=None): """Asserts a state of given node in a trie. Args: t: Trie to check the node in. key: A key for the node. prefix: Whether the node is a prefix of a longer key that is in the trie. value: If given, value associated with the key. If missing, node has no value associated with it. Raises: AssertionError: If any assumption is not met. """ if prefix: self.assertTrue(t.has_subtrie(key)) self.assertTrue(bool(t.has_node(key) & pygtrie.Trie.HAS_SUBTRIE)) else: self.assertFalse(t.has_subtrie(key)) self.assertFalse(bool(t.has_node(key) & pygtrie.Trie.HAS_SUBTRIE)) if value is None: o = object() self.assertNotIn(key, t) key_error_exception = pygtrie.ShortKeyError if prefix else KeyError self.assertRaises(key_error_exception, lambda: t[key]) self.assertRaises(key_error_exception, t.pop, key) self.assertIsNone(t.get(key)) self.assertIs(o, t.get(key, o)) self.assertIs(o, t.pop(key, o)) self.assertFalse(t.has_key(key)) self.assertNotIn(self.key_from_key(key), list(t.iterkeys())) self.assertNotIn(self.key_from_key(key), t.keys()) self.assertEqual(pygtrie.Trie.HAS_SUBTRIE if prefix else 0, t.has_node(key)) else: self.assertIn(key, t) self.assertEqual(value, t[key]) self.assertEqual(value, t.get(key)) self.assertEqual(value, t.get(key, object())) self.assertTrue(t.has_key(key)) self.assertTrue(bool(t.has_node(key) & pygtrie.Trie.HAS_VALUE)) self.assertIn(self.key_from_key(key), list(t.iterkeys())) self.assertIn(self.key_from_key(key), t.keys()) def assertFullTrie(self, t, value=42): """Asserts a trie has _SHORT_KEY and _LONG_KEY set to value.""" self.assertEqual(2, len(t)) for prefix in self._SHORT_PREFIXES + self._LONG_PREFIXES: self.assertNodeState(t, prefix, prefix=True) self.assertNodeState(t, self._SHORT_KEY, prefix=True, value=value) self.assertNodeState(t, self._LONG_KEY, value=value) self.assertNodeState(t, self._VERY_LONG_KEY) self.assertNodeState(t, self._OTHER_KEY) def assertShortTrie(self, t, value=42): """Asserts a trie has only _SHORT_KEY set to value.""" self.assertEqual(1, len(t)) for prefix in self._SHORT_PREFIXES: self.assertNodeState(t, prefix, prefix=True) for key in self._LONG_PREFIXES + ( self._LONG_KEY, self._VERY_LONG_KEY, self._OTHER_KEY): self.assertNodeState(t, key) self.assertNodeState(t, self._SHORT_KEY, value=value) def assertEmptyTrie(self, t): """Asserts a trie is empty.""" self.assertEqual(0, len(t), '%r should be empty: %d' % (t, len(t))) for key in self._SHORT_PREFIXES + self._LONG_PREFIXES + ( self._SHORT_KEY, self._LONG_KEY, self._VERY_LONG_KEY, self._OTHER_KEY): self.assertNodeState(t, key) self.assertRaises(KeyError, t.popitem) self.assertEqual('Trie()', str(t)) self.assertEqual('Trie()', repr(t)) # pylint: enable=invalid-name def _do_test_basics(self, trie_factory): """Basic trie tests.""" d = dict.fromkeys((self._SHORT_KEY, self._LONG_KEY), 42) t = trie_factory(self._TRIE_CLS, d) self.assertFullTrie(t) self.assertEqual(42, t.pop(self._LONG_KEY)) self.assertShortTrie(t) self.assertEqual(42, t.setdefault(self._SHORT_KEY, 24)) self.assertShortTrie(t) t[self._SHORT_KEY] = 24 self.assertShortTrie(t, 24) self.assertEqual(24, t.setdefault(self._LONG_KEY, 24)) self.assertFullTrie(t, 24) del t[self._LONG_KEY] self.assertShortTrie(t, 24) self.assertEqual((self.key_from_key(self._SHORT_KEY), 24), t.popitem()) self.assertEmptyTrie(t) def _do_test_iterator(self, trie_factory): """Trie iterator tests.""" d = dict.fromkeys((self._SHORT_KEY, self._LONG_KEY), 42) t = trie_factory(self._TRIE_CLS, d) self.assertEqual([42, 42], t.values()) self.assertEqual([42, 42], list(t.itervalues())) short_key = self.key_from_key(self._SHORT_KEY) long_key = self.key_from_key(self._LONG_KEY) expected_items = [(short_key, 42), (long_key, 42)] self.assertEqual(expected_items, sorted(t.items())) self.assertEqual(expected_items, sorted(t.iteritems())) self.assertEqual([short_key, long_key], sorted(t)) self.assertEqual([short_key, long_key], sorted(t.keys())) self.assertEqual([short_key, long_key], sorted(t.iterkeys())) def _do_test_subtrie_iterator(self, trie_factory): """Subtrie iterator tests.""" d = dict.fromkeys((self._SHORT_KEY, self._LONG_KEY), 42) t = trie_factory(self._TRIE_CLS, d) long_key = self.key_from_key(self._LONG_KEY) prefix = self._LONG_PREFIXES[0] self.assertEqual([42, 42], t.values(prefix=self._SHORT_KEY)) self.assertEqual([42, 42], list(t.itervalues(prefix=self._SHORT_KEY))) self.assertEqual([42], t.values(prefix=prefix)) self.assertEqual([42], list(t.itervalues(prefix=prefix))) expected_items = [(long_key, 42)] self.assertEqual(expected_items, sorted(t.items(prefix=prefix))) self.assertEqual(expected_items, sorted(t.iteritems(prefix=prefix))) self.assertEqual([long_key], sorted(t.keys(prefix=prefix))) self.assertEqual([long_key], sorted(t.iterkeys(prefix=prefix))) def _do_test_shallow_iterator(self, trie_factory): """Shallow iterator test.""" d = dict.fromkeys((self._SHORT_KEY, self._LONG_KEY), 42) t = trie_factory(self._TRIE_CLS, d) self.assertEqual([42], t.values(shallow=True)) self.assertEqual([42], list(t.itervalues(shallow=True))) short_key = self.key_from_key(self._SHORT_KEY) expected_items = [(short_key, 42)] self.assertEqual(expected_items, sorted(t.items(shallow=True))) self.assertEqual(expected_items, sorted(t.iteritems(shallow=True))) self.assertEqual([short_key], sorted(t.keys(shallow=True))) self.assertEqual([short_key], sorted(t.iterkeys(shallow=True))) def _do_test_splice_operations(self, trie_factory): """Splice trie operations tests.""" d = dict.fromkeys((self._SHORT_KEY, self._LONG_KEY), 42) t = trie_factory(self._TRIE_CLS, d) self.assertEqual([42, 42], list(t[self._SHORT_KEY:])) self.assertEqual([42], list(t[self._LONG_PREFIXES[0]:])) t[self._SHORT_KEY:] = 24 self.assertShortTrie(t, 24) self.assertEqual([24], list(t[self._SHORT_KEY:])) self.assertRaises(KeyError, lambda: list(t[self._LONG_PREFIXES[0]:])) t[self._LONG_KEY:] = 24 self.assertFullTrie(t, 24) del t[self._SHORT_KEY:] self.assertEmptyTrie(t) def _do_test_find_prefix(self, trie_factory): """Prefix finding methods tests.""" d = dict.fromkeys((self._SHORT_KEY, self._LONG_KEY), 42) t = trie_factory(self._TRIE_CLS, d) short_pair = (self.key_from_key(self._SHORT_KEY), 42) long_pair = (self.key_from_key(self._LONG_KEY), 42) none_pair = (None, None) def assert_pair(expected, got): self.assertEqual(expected, got) if expected[0]: self.assertTrue(got) else: self.assertFalse(got) assert_pair(short_pair, t.shortest_prefix(self._VERY_LONG_KEY)) assert_pair(short_pair, t.shortest_prefix(self._LONG_KEY)) assert_pair(short_pair, t.shortest_prefix(self._VERY_LONG_KEY)) assert_pair(short_pair, t.shortest_prefix(self._LONG_PREFIXES[-1])) assert_pair(short_pair, t.shortest_prefix(self._SHORT_KEY)) assert_pair(none_pair, t.shortest_prefix(self._SHORT_PREFIXES[-1])) assert_pair(long_pair, t.longest_prefix(self._VERY_LONG_KEY)) assert_pair(long_pair, t.longest_prefix(self._LONG_KEY)) assert_pair(long_pair, t.longest_prefix(self._VERY_LONG_KEY)) assert_pair(short_pair, t.shortest_prefix(self._LONG_PREFIXES[-1])) assert_pair(short_pair, t.longest_prefix(self._SHORT_KEY)) assert_pair(none_pair, t.shortest_prefix(self._SHORT_PREFIXES[-1])) self.assertEqual([], list(t.prefixes(self._SHORT_PREFIXES[-1]))) self.assertEqual([short_pair], list(t.prefixes(self._SHORT_KEY))) self.assertEqual([short_pair], list(t.prefixes(self._LONG_PREFIXES[-1]))) self.assertEqual([short_pair, long_pair], list(t.prefixes(self._LONG_KEY))) self.assertEqual([short_pair, long_pair], list(t.prefixes(self._VERY_LONG_KEY))) def _do_test_pickle(self, trie_factory): """https://github.com/google/pygtrie/issues/7""" d = dict.fromkeys((self._SHORT_KEY, self._LONG_KEY, self._VERY_LONG_KEY, self._OTHER_KEY), 42) t = trie_factory(self._TRIE_CLS, d) pickled = pickle.dumps(t) u = pickle.loads(pickled) self.assertEqual(t, u) def test_prefix_set(self): """PrefixSet test.""" ps = pygtrie.PrefixSet(factory=self._TRIE_CLS) short_key = self.key_from_key(self._SHORT_KEY) long_key = self.key_from_key(self._LONG_KEY) very_long_key = self.key_from_key(self._VERY_LONG_KEY) other_key = self.key_from_key(self._OTHER_KEY) for key in (self._LONG_KEY, self._VERY_LONG_KEY): ps.add(key) self.assertEqual(1, len(ps)) self.assertEqual([long_key], list(ps.iter())) self.assertEqual([long_key], list(iter(ps))) self.assertEqual([long_key], list(ps.iter(self._SHORT_KEY))) self.assertEqual([long_key], list(ps.iter(self._LONG_KEY))) self.assertEqual([very_long_key], list(ps.iter(self._VERY_LONG_KEY))) self.assertEqual([], list(ps.iter(self._OTHER_KEY))) ps.add(self._SHORT_KEY) self.assertEqual(1, len(ps)) self.assertEqual([short_key], list(ps.iter())) self.assertEqual([short_key], list(iter(ps))) self.assertEqual([short_key], list(ps.iter(self._SHORT_KEY))) self.assertEqual([long_key], list(ps.iter(self._LONG_KEY))) self.assertEqual([], list(ps.iter(self._OTHER_KEY))) ps.add(self._OTHER_KEY) self.assertEqual(2, len(ps)) self.assertEqual(sorted((short_key, other_key)), sorted(ps.iter())) self.assertEqual([short_key], list(ps.iter(self._SHORT_KEY))) self.assertEqual([long_key], list(ps.iter(self._LONG_KEY))) self.assertEqual([other_key], list(ps.iter(self._OTHER_KEY))) def test_equality(self): """Tests equality comparison.""" d = dict.fromkeys((self._SHORT_KEY, self._LONG_KEY), 42) # pylint: disable=redefined-outer-name tries = [factory(self._TRIE_CLS, d) for _, factory in _TRIE_FACTORIES] for i in range(1, len(tries)): self.assertEqual(tries[i-1], tries[i], '%r (factory: %s) should equal %r (factory: %s)' % (tries[i-1], _TRIE_FACTORIES[i-1][0], tries[i], _TRIE_FACTORIES[i][0])) for i in range(1, len(tries)): tries[i-1][self._OTHER_KEY] = 42 self.assertNotEqual( tries[i-1], tries[i], '%r (factory: %s) should not be equal %r (factory: %s)' % (tries[i-1], _TRIE_FACTORIES[i-1][0], tries[i], _TRIE_FACTORIES[i][0])) def _construct_trie_test_cases(): def make_test_method(method, factory): return lambda self: method(self, trie_factory=factory) for name in list(TrieTestCase.__dict__.keys()): if not name.startswith('_do_test_'): continue orig = getattr(TrieTestCase, name) for factory_name, factory in _TRIE_FACTORIES: method = make_test_method(orig, factory) method.__doc__ = '%s using %s trie factory.' % ( orig.__doc__[:-2], factory_name) setattr(TrieTestCase, '%s_%s' % (name[4:], factory_name), method) _construct_trie_test_cases() class CharTrieTestCase(TrieTestCase): _TRIE_CLS = pygtrie.CharTrie @classmethod def key_from_path(cls, path): return ''.join(path) class StringTrieTestCase(TrieTestCase): _TRIE_CLS = pygtrie.StringTrie _SHORT_KEY = '/home/foo' _LONG_KEY = _SHORT_KEY + '/bar/baz' _VERY_LONG_KEY = _LONG_KEY + '/qux' _OTHER_KEY = '/hom' _SHORT_PREFIXES = ('', '/home') _LONG_PREFIXES = ('/home/foo/bar',) @classmethod def path_from_key(cls, key): return key.split('/') @classmethod def key_from_path(cls, path): return '/'.join(path) class SortTest(unittest.TestCase): def test_enable_sorting(self): keys = sorted(chr(x) for x in range(32, 128) if x != ord('/')) t = pygtrie.StringTrie.fromkeys(keys) # Unless dict's hash function is weird, trie's keys should not be # returned in order. self.assertNotEqual(keys, t.keys()) self.assertEqual(keys, sorted(t.keys())) t.enable_sorting() self.assertEqual(keys, t.keys()) t.enable_sorting(False) self.assertNotEqual(keys, t.keys()) class TraverseTest(unittest.TestCase): _SENTINEL = object() _TestNode = collections.namedtuple('TestNode', 'key children value') @classmethod def _make_test_node(cls, path_conv, path, children, value=_SENTINEL): return cls._TestNode(path_conv(path), sorted(children), value) def assertNode(self, node, key, children=0, value=_SENTINEL): # pylint: disable=invalid-name self.assertTrue(node) self.assertEqual(key, node.key) self.assertEqual(children, len(node.children)) self.assertEqual(value, node.value) return node def test_traverse_empty_tree(self): t = pygtrie.CharTrie() r = t.traverse(self._make_test_node) self.assertNode(r, '', 0) def test_traverse_singleton_tree(self): t = pygtrie.CharTrie() t.update({'a': 10}) r = t.traverse(self._make_test_node) self.assertNode(r, '', 1) self.assertNode(r.children[0], 'a', 0, 10) def test_traverse(self): t = pygtrie.CharTrie() t.update({'aaa': 1, 'aab': 2, 'aac': 3, 'bb': 4}) r = t.traverse(self._make_test_node) # Result: # <> # a # aa # aaa:1 # aab:2 # aac:3 # b # bb:4 self.assertNode(r, '', 2) # For some reason pylint thinks a_node et al. are strings. # pylint: disable=no-member a_node = self.assertNode(r.children[0], 'a', 1) aa_node = self.assertNode(a_node.children[0], 'aa', 3) self.assertNode(aa_node.children[0], 'aaa', 0, 1) self.assertNode(aa_node.children[2], 'aac', 0, 3) b_node = self.assertNode(r.children[1], 'b', 1) self.assertNode(b_node.children[0], 'bb', 0, 4) def test_traverse_compressing(self): t = pygtrie.CharTrie() t.update({'aaa': 1, 'aab': 2, 'aac': 3, 'bb': 4}) def make(path_conv, path, children, value=self._SENTINEL): children = sorted(children) if value is self._SENTINEL and len(children) == 1: # There is only one prefix. return children[0] else: return self._TestNode(path_conv(path), children, value) r = t.traverse(make) # Result: # <> # aa # aaa:1 # aab:2 # aac:3 # bb:4 self.assertNode(r, '', 2) # For some reason pylint thinks a_node et al. are strings. # pylint: disable=no-member aa_node = self.assertNode(r.children[0], 'aa', 3) self.assertNode(aa_node.children[0], 'aaa', 0, 1) self.assertNode(aa_node.children[1], 'aab', 0, 2) self.assertNode(aa_node.children[2], 'aac', 0, 3) self.assertNode(r.children[1], 'bb', 0, 4) def test_traverse_ignore_subtrie(self): t = pygtrie.CharTrie() t.update({'aaa': 1, 'aab': 2, 'aac': 3, 'b': 4}) cnt = [0] def make(path_conv, path, children, value=self._SENTINEL): cnt[0] += 1 if path and path[0] == 'a': return None else: children = [ch for ch in children if ch is not None] return self._TestNode(path_conv(path), children, value) r = t.traverse(make) # Result: # <> # b:4 self.assertNode(r, '', 1) self.assertNode(r.children[0], 'b', 0, 4) self.assertEqual(3, cnt[0]) class RecursionTest(unittest.TestCase): """Test for deep recursion. https://github.com/google/pygtrie/issues/8 """ @staticmethod def create_trie(): tostring = (getattr(array.array, 'tobytes', None) or # Python 3 getattr(array.array, 'tostring')) # Python 3 trie = pygtrie.Trie() for x in range(100): y = tostring(array.array('h', range(x, 1000))) trie.update([(y, x)]) return trie def test_iterator(self): for _ in self.create_trie().iteritems(): pass def test_copy(self): self.create_trie().copy() if __name__ == '__main__': unittest.main() pygtrie-2.2/version-history.rst000066400000000000000000000034321312644474700167740ustar00rootroot00000000000000Version History --------------- 2.2: 2017/06/03 - Fixes to setup.py breaking on Windows which prevents installation among other things. 2.1: 2017/03/23 - The library is now Python 3 compatible. - Value returend by ``shortest_prefix`` and ``longest_prefix`` evaluates to false if no prefix was found. This is in addition to it being a pair of Nones of course. 2.0: 2016/07/06 - Sorting of child nodes is disabled by default for better performance. ``enable_sorting`` method can be used to bring back old behaviour. - Tries of arbitrary depth can be pickled without reaching Python’s recursion limits. (N.B. The pickle format is incompatible with one from 1.2 release). ``_Node``’s ``__getstate__`` and ``__setstate__`` method can be used to implement other serialisation methods such as JSON. 1.2: 2016/06/21 [pulled back from PyPi] - Tries can now be pickled. - Iterating no longer uses recursion so tries of arbitrary depth can be iterated over. The ``traverse`` method, however, still uses recursion thus cannot be used on big structures. 1.1: 2016/01/18 - Fixed PyPi installation issues; all should work now. 1.0: 2015/12/16 - The module has been renamed from ``trie`` to ``pygtrie``. This could break current users but see documentation for how to quickly upgrade your scripts. - Added ``traverse`` method which goes through the nodes of the trie preserving structure of the tree. This is a depth-first traversal which can be used to search for elements or translate a trie into a different tree structure. - Minor documentation fixes. 0.9.3: 2015/05/28 - Minor documentation fixes. 0.9.2: 2015/05/28 - Added Sphinx configuration and updated docstrings to work better with Sphinx. 0.9.1: 2014/02/03 - New name. 0.9: 2014/02/03 - Initial release. pygtrie-2.2/version.py000066400000000000000000000075521312644474700151240ustar00rootroot00000000000000# -*- coding: utf-8 -*- """Calculates the current version number. If possible, uses output of “git describe” modified to conform to the visioning scheme that setuptools uses (see PEP 386). Releases must be labelled with annotated tags (signed tags are annotated) of the following format: v(.)+ [ {a|b|c|rc} (.)* ] If “git describe” returns an error (likely because we're in an unpacked copy of a release tarball, rather than a git working copy), or returns a tag that does not match the above format, version is read from .version file. To use this script, simply import it in your setup.py file, and use the results of get_version() as your package version: import version setup( version=version.get_version(), . . . ) This will automatically update the .version file. The .version file should *not* be checked into git but it *should* be included in sdist tarballs (as should version.py file). To do this, run: echo include .version version.py >>MANIFEST.in echo .version >>.gitignore With that setup, a new release can be labelled by simply invoking: git tag -s v1.0 """ __author__ = ('Douglas Creager ', 'Michal Nazarewicz ') __license__ = 'This file is placed into the public domain.' __maintainer__ = 'Michal Nazarewicz' __email__ = 'mina86@mina86.com' __all__ = ('get_version',) import re import subprocess import sys RELEASE_VERSION_FILE = '.version' # http://www.python.org/dev/peps/pep-0386/ _PEP386_SHORT_VERSION_RE = r'\d+(?:\.\d+)+(?:(?:[abc]|rc)\d+(?:\.\d+)*)?' _PEP386_VERSION_RE = r'^%s(?:\.post\d+)?(?:\.dev\d+)?$' % ( _PEP386_SHORT_VERSION_RE) _GIT_DESCRIPTION_RE = r'^v(?P%s)-(?P\d+)-g(?P[\da-f]+)$' % ( _PEP386_SHORT_VERSION_RE) def read_git_version(): """Reads version from "git describe" command.""" try: proc = subprocess.Popen(('git', 'describe', '--long', '--match', 'v[0-9]*.*'), stdout=subprocess.PIPE, stderr=subprocess.PIPE) data, _ = proc.communicate() if proc.returncode: return None data = data.decode(getattr(sys.stdin, 'encoding', 'ascii')) ver = data.splitlines()[0].strip() except: # pylint: disable=bare-except return None if not ver: return None m = re.search(_GIT_DESCRIPTION_RE, ver) if not m: sys.stderr.write('version: git description (%s) is invalid, ' 'ignoring\n' % ver) return None commits = int(m.group('commits')) if not commits: return m.group('ver') else: return '%s.post%d.dev%d' % ( m.group('ver'), commits, int(m.group('sha'), 16)) def read_release_version(warn): """Reads release version from a .version file.""" try: fd = open(RELEASE_VERSION_FILE) try: ver = fd.readline().strip() finally: fd.close() if warn and not re.search(_PEP386_VERSION_RE, ver): sys.stderr.write('version: release version (%s) is invalid, ' 'will use it anyway\n' % ver) return ver except: # pylint: disable=bare-except return None def write_release_version(version): """Writes release version to a .version file.""" fd = open(RELEASE_VERSION_FILE, 'w') fd.write('%s\n' % version) fd.close() def get_version(): """Figures out package's version; also stores it in a file.""" version = read_git_version() release_version = read_release_version(not version) version = version or release_version if not version: raise ValueError('Cannot find the version number') if version != release_version: write_release_version(version) return version if __name__ == '__main__': print(get_version())