pax_global_header00006660000000000000000000000064145553155640014527gustar00rootroot0000000000000052 comment=69cd6ce7476870f33d1ec6ea494980447fb1cc31 uqfoundation-klepto-69cd6ce/000077500000000000000000000000001455531556400162445ustar00rootroot00000000000000uqfoundation-klepto-69cd6ce/.codecov.yml000066400000000000000000000015001455531556400204630ustar00rootroot00000000000000comment: false coverage: status: project: default: # Commits pushed to master should not make the overall # project coverage decrease by more than 1%: target: auto threshold: 1% patch: default: # Be tolerant on slight code coverage diff on PRs to limit # noisy red coverage status on github PRs. # Note The coverage stats are still uploaded # to codecov so that PR reviewers can see uncovered lines # in the github diff if they install the codecov browser # extension: # https://github.com/codecov/browser-extension target: auto threshold: 1% fixes: # reduces pip-installed path to git root and # remove dist-name from setup-installed path - "*/site-packages/::" - "*/site-packages/klepto-*::" uqfoundation-klepto-69cd6ce/.coveragerc000066400000000000000000000010611455531556400203630ustar00rootroot00000000000000[run] # source = klepto include = */klepto/* omit = */tests/* */info.py branch = true # timid = true # parallel = true # and need to 'combine' data files # concurrency = multiprocessing # thread # data_file = $TRAVIS_BUILD_DIR/.coverage # debug = trace [paths] source = klepto */site-packages/klepto */site-packages/klepto-*/klepto [report] include = */klepto/* exclude_lines = pragma: no cover raise NotImplementedError if __name__ == .__main__.: # show_missing = true ignore_errors = true # pragma: no branch # noqa uqfoundation-klepto-69cd6ce/.gitignore000066400000000000000000000000401455531556400202260ustar00rootroot00000000000000.tox/ .cache/ *.egg-info/ *.pyc uqfoundation-klepto-69cd6ce/.readthedocs.yml000066400000000000000000000005151455531556400213330ustar00rootroot00000000000000# readthedocs configuration file # see https://docs.readthedocs.io/en/stable/config-file/v2.html version: 2 # configure sphinx: configuration: docs/source/conf.py # build build: os: ubuntu-22.04 tools: python: "3.10" # install python: install: - method: pip path: . - requirements: docs/requirements.txt uqfoundation-klepto-69cd6ce/.travis.yml000066400000000000000000000033351455531556400203610ustar00rootroot00000000000000dist: jammy language: python matrix: include: - python: '3.8' env: - python: '3.9' env: - COVERAGE="true" - PANDAS="true" - SQLALCHEMY="true" - H5PY="true" - python: '3.10' env: - python: '3.11' env: - python: '3.12' env: - python: '3.13-dev' env: - DILL="master" - python: 'pypy3.8-7.3.9' # at 7.3.11 env: - python: 'pypy3.9-7.3.9' # at 7.3.15 env: - python: 'pypy3.10-7.3.15' env: allow_failures: - python: '3.13-dev' - python: 'pypy3.10-7.3.15' # CI missing fast_finish: true cache: pip: true before_install: - set -e # fail on any error - if [[ $COVERAGE == "true" ]]; then pip install coverage; fi - if [[ $PANDAS == "true" ]]; then pip install pandas; fi - if [[ $SQLALCHEMY == "true" ]]; then pip install "sqlalchemy<2.0.0"; fi - if [[ $H5PY == "true" ]]; then pip install h5py; fi - if [[ $DILL == "master" ]]; then pip install "https://github.com/uqfoundation/dill/archive/master.tar.gz"; fi install: - python -m pip install . script: - for test in klepto/tests/__init__.py; do echo $test ; if [[ $COVERAGE == "true" ]]; then coverage run -a $test > /dev/null; else python $test > /dev/null; fi ; done - for test in klepto/tests/test_*.py; do echo $test ; if [[ $COVERAGE == "true" ]]; then coverage run -a $test > /dev/null; else python $test > /dev/null; fi ; done after_success: - if [[ $COVERAGE == "true" ]]; then bash <(curl -s https://codecov.io/bash); else echo ''; fi - if [[ $COVERAGE == "true" ]]; then coverage report; fi uqfoundation-klepto-69cd6ce/LICENSE000066400000000000000000000033761455531556400172620ustar00rootroot00000000000000Copyright (c) 2004-2016 California Institute of Technology. Copyright (c) 2016-2024 The Uncertainty Quantification Foundation. All rights reserved. This software is available subject to the conditions and terms laid out below. By downloading and using this software you are agreeing to the following conditions. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - Neither the names of the copyright holders nor the names of any of the contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. uqfoundation-klepto-69cd6ce/MANIFEST.in000066400000000000000000000002711455531556400200020ustar00rootroot00000000000000include LICENSE include README* include MANIFEST.in include pyproject.toml include tox.ini include version.py recursive-include docs * include .* prune .git prune .coverage prune .eggs uqfoundation-klepto-69cd6ce/README.md000066400000000000000000000143351455531556400175310ustar00rootroot00000000000000klepto ====== persistent caching to memory, disk, or database About Klepto ------------ ``klepto`` extends Python's ``lru_cache`` to utilize different keymaps and alternate caching algorithms, such as ``lfu_cache`` and ``mru_cache``. While caching is meant for fast access to saved results, ``klepto`` also has archiving capabilities, for longer-term storage. ``klepto`` uses a simple dictionary-sytle interface for all caches and archives, and all caches can be applied to any Python function as a decorator. Keymaps are algorithms for converting a function's input signature to a unique dictionary, where the function's results are the dictionary value. Thus for ``y = f(x)``, ``y`` will be stored in ``cache[x]`` (e.g. ``{x:y}``). ``klepto`` provides both standard and *"safe"* caching, where *"safe"* caches are slower but can recover from hashing errors. ``klepto`` is intended to be used for distributed and parallel computing, where several of the keymaps serialize the stored objects. Caches and archives are intended to be read/write accessible from different threads and processes. ``klepto`` enables a user to decorate a function, save the results to a file or database archive, close the interpreter, start a new session, and reload the function and it's cache. ``klepto`` is part of ``pathos``, a Python framework for heterogeneous computing. ``klepto`` is in active development, so any user feedback, bug reports, comments, or suggestions are highly appreciated. A list of issues is located at https://github.com/uqfoundation/klepto/issues, with a legacy list maintained at https://uqfoundation.github.io/project/pathos/query. Major Features -------------- ``klepto`` has standard and *"safe"* variants of the following: * ``lfu_cache`` - the least-frequently-used caching algorithm * ``lru_cache`` - the least-recently-used caching algorithm * ``mru_cache`` - the most-recently-used caching algorithm * ``rr_cache`` - the random-replacement caching algorithm * ``no_cache`` - a dummy caching interface to archiving * ``inf_cache`` - an infinitely-growing cache ``klepto`` has the following archive types: * ``file_archive`` - a dictionary-style interface to a file * ``dir_archive`` - a dictionary-style interface to a folder of files * ``sqltable_archive`` - a dictionary-style interface to a sql database table * ``sql_archive`` - a dictionary-style interface to a sql database * ``hdfdir_archive`` - a dictionary-style interface to a folder of hdf5 files * ``hdf_archive`` - a dictionary-style interface to a hdf5 file * ``dict_archive`` - a dictionary with an archive interface * ``null_archive`` - a dictionary-style interface to a dummy archive ``klepto`` provides the following keymaps: * ``keymap`` - keys are raw Python objects * ``hashmap`` - keys are a hash for the Python object * ``stringmap`` - keys are the Python object cast as a string * ``picklemap`` - keys are the serialized Python object ``klepto`` also includes a few useful decorators providing: * simple, shallow, or deep rounding of function arguments * cryptographic key generation, with masking of selected arguments Current Release [![Downloads](https://static.pepy.tech/personalized-badge/klepto?period=total&units=international_system&left_color=grey&right_color=blue&left_text=pypi%20downloads)](https://pepy.tech/project/klepto) [![Conda Downloads](https://img.shields.io/conda/dn/conda-forge/klepto?color=blue&label=conda%20downloads)](https://anaconda.org/conda-forge/klepto) [![Stack Overflow](https://img.shields.io/badge/stackoverflow-get%20help-black.svg)](https://stackoverflow.com/questions/tagged/klepto) --------------- The latest released version of ``klepto`` is available from: https://pypi.org/project/klepto ``klepto`` is distributed under a 3-clause BSD license. Development Version [![Support](https://img.shields.io/badge/support-the%20UQ%20Foundation-purple.svg?style=flat&colorA=grey&colorB=purple)](http://www.uqfoundation.org/pages/donate.html) [![Documentation Status](https://readthedocs.org/projects/klepto/badge/?version=latest)](https://klepto.readthedocs.io/en/latest/?badge=latest) [![Build Status](https://travis-ci.com/uqfoundation/klepto.svg?label=build&logo=travis&branch=master)](https://travis-ci.com/github/uqfoundation/klepto) [![codecov](https://codecov.io/gh/uqfoundation/klepto/branch/master/graph/badge.svg)](https://codecov.io/gh/uqfoundation/klepto) ------------------- You can get the latest development version with all the shiny new features at: https://github.com/uqfoundation If you have a new contribution, please submit a pull request. Installation ------------ ``klepto`` can be installed with ``pip``:: $ pip install klepto To include optional archive backends, such as HDF5 and SQL, in the install:: $ pip install klepto[archives] To include optional serializers, such as ``jsonpickle``, in the install:: $ pip install klepto[crypto] Requirements ------------ ``klepto`` requires: * ``python`` (or ``pypy``), **>=3.8** * ``setuptools``, **>=42** * ``dill``, **>=0.3.8** * ``pox``, **>=0.3.4** Optional requirements: * ``h5py``, **>=2.8.0** * ``pandas``, **>=0.17.0** * ``sqlalchemy``, **>=1.4.0** * ``jsonpickle``, **>=0.9.6** * ``cloudpickle``, **>=0.5.2** More Information ---------------- Probably the best way to get started is to look at the documentation at http://klepto.rtfd.io. Also see ``klepto.tests`` for a set of scripts that test the caching and archiving functionalities in ``klepto``. You can run the test suite with ``python -m klepto.tests``. The source code is also generally well documented, so further questions may be resolved by inspecting the code itself. Please feel free to submit a ticket on github, or ask a question on stackoverflow (**@Mike McKerns**). If you would like to share how you use ``klepto`` in your work, please send an email (to **mmckerns at uqfoundation dot org**). Citation -------- If you use ``klepto`` to do research that leads to publication, we ask that you acknowledge use of ``klepto`` by citing the following in your publication:: Michael McKerns and Michael Aivazis, "pathos: a framework for heterogeneous computing", 2010- ; https://uqfoundation.github.io/project/pathos Please see https://uqfoundation.github.io/project/pathos or http://arxiv.org/pdf/1202.1056 for further information. uqfoundation-klepto-69cd6ce/docs/000077500000000000000000000000001455531556400171745ustar00rootroot00000000000000uqfoundation-klepto-69cd6ce/docs/Makefile000066400000000000000000000010401455531556400206270ustar00rootroot00000000000000# Minimal makefile for Sphinx documentation # # You can set these variables from the command line. SPHINXOPTS = SPHINXBUILD = sphinx-build SPHINXPROJ = klepto SOURCEDIR = source BUILDDIR = build # Internal variables ALLSPHINXOPTS = $(SPHINXOPTS) $(SOURCEDIR) # Put it first so that "make" without argument is like "make help". help: @echo "Please use \`make html' to generate standalone HTML files" .PHONY: help clean html Makefile clean: -rm -rf $(BUILDDIR) html: $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR) uqfoundation-klepto-69cd6ce/docs/requirements.txt000066400000000000000000000015461455531556400224660ustar00rootroot00000000000000# Packages required to build docs # dependencies pinned as: # https://github.com/readthedocs/readthedocs.org/blob/4dd655eaa5a36aa2cb9eed3e98961419536f99e8/requirements/docs.txt alabaster==0.7.13 babel==2.12.1 certifi==2023.7.22 charset-normalizer==3.2.0 click==8.1.6 colorama==0.4.6 docutils==0.18.1 idna==3.4 imagesize==1.4.1 jinja2==3.1.3 livereload==2.6.3 markdown-it-py==3.0.0 markupsafe==2.1.3 mdit-py-plugins==0.4.0 mdurl==0.1.2 myst-parser==2.0.0 packaging==23.1 pygments==2.16.1 pyyaml==6.0.1 readthedocs-sphinx-search==0.3.2 requests==2.31.0 six==1.16.0 snowballstemmer==2.2.0 sphinx==6.2.1 sphinx-autobuild==2021.3.14 sphinx-copybutton==0.5.2 sphinx-design==0.5.0 sphinx-hoverxref==1.3.0 sphinx-intl==2.1.0 sphinx-multiproject==1.0.0rc1 sphinx-notfound-page==0.8.3 sphinx-prompt==1.6.0 sphinx-rtd-theme==1.2.2 sphinx-tabs==3.4.1 tornado==6.3.3 urllib3==2.0.7 uqfoundation-klepto-69cd6ce/docs/source/000077500000000000000000000000001455531556400204745ustar00rootroot00000000000000uqfoundation-klepto-69cd6ce/docs/source/_static/000077500000000000000000000000001455531556400221225ustar00rootroot00000000000000uqfoundation-klepto-69cd6ce/docs/source/_static/css/000077500000000000000000000000001455531556400227125ustar00rootroot00000000000000uqfoundation-klepto-69cd6ce/docs/source/_static/css/custom.css000066400000000000000000000001251455531556400247340ustar00rootroot00000000000000div.sphinxsidebar { height: 100%; /* 100vh */ overflow: auto; /* overflow-y */ } uqfoundation-klepto-69cd6ce/docs/source/conf.py000066400000000000000000000173211455531556400217770ustar00rootroot00000000000000# -*- coding: utf-8 -*- # # klepto documentation build configuration file, created by # sphinx-quickstart on Tue Aug 8 06:50:58 2017. # # This file is execfile()d with the current directory set to its # containing dir. # # Note that not all possible configuration values are present in this # autogenerated file. # # All configuration values have a default; values that are commented out # serve to show the default. # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # import os from datetime import datetime # import sys # sys.path.insert(0, os.path.abspath('../../scripts')) # Import the project import klepto # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. # # needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'sphinx.ext.imgmath', 'sphinx.ext.ifconfig', 'sphinx.ext.napoleon'] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # # source_suffix = ['.rst', '.md'] source_suffix = '.rst' # The master toctree document. master_doc = 'index' # General information about the project. project = 'klepto' year = datetime.now().year copyright = '%d, The Uncertainty Quantification Foundation' % year author = 'Mike McKerns' # extension config github_project_url = "https://github.com/uqfoundation/klepto" autoclass_content = 'both' autodoc_default_options = { 'members': True, 'undoc-members': True, 'private-members': True, 'special-members': True, 'show-inheritance': True, 'imported-members': True, 'exclude-members': ( '__dict__,' '__slots__,' '__weakref__,' '__module__,' '_abc_impl,' '__init__,' '__annotations__,' '__dataclass_fields__,' ) } autodoc_typehints = 'description' autodoc_typehints_format = 'short' napoleon_include_private_with_doc = False napoleon_include_special_with_doc = True napoleon_use_ivar = True napoleon_use_param = True # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. version = klepto.__version__ # The full version, including alpha/beta/rc tags. release = version # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. language = 'en' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This patterns also effect to html_static_path and html_extra_path exclude_patterns = [] # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = False # Configure how the modules, functions, etc names look add_module_names = False modindex_common_prefix = ['klepto.'] # -- Options for HTML output ---------------------------------------------- # on_rtd is whether we are on readthedocs.io on_rtd = os.environ.get('READTHEDOCS', None) == 'True' # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # if not on_rtd: html_theme = 'alabaster' #'bizstyle' html_css_files = ['css/custom.css',] #import sphinx_rtd_theme #html_theme = 'sphinx_rtd_theme' #html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] else: html_theme = 'sphinx_rtd_theme' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. # html_theme_options = { 'github_user': 'uqfoundation', 'github_repo': 'klepto', 'github_button': False, 'github_banner': True, 'travis_button': True, 'codecov_button': True, 'donate_url': 'http://uqfoundation.org/pages/donate.html', 'gratipay_user': False, # username 'extra_nav_links': {'Module Index': 'py-modindex.html'}, # 'show_related': True, # 'globaltoc_collapse': True, 'globaltoc_maxdepth': 4, 'show_powered_by': False } # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] # Custom sidebar templates, must be a dictionary that maps document names # to template names. # # This is required for the alabaster theme # refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars if on_rtd: toc_style = 'localtoc.html', # display the toctree else: toc_style = 'globaltoc.html', # collapse the toctree html_sidebars = { '**': [ 'about.html', 'donate.html', 'searchbox.html', # 'navigation.html', toc_style, # defined above 'relations.html', # needs 'show_related':True option to display ] } # -- Options for HTMLHelp output ------------------------------------------ # Output file base name for HTML help builder. htmlhelp_basename = 'kleptodoc' # Logo for sidebar html_logo = 'pathos.png' # -- Options for LaTeX output --------------------------------------------- latex_elements = { # The paper size ('letterpaper' or 'a4paper'). # # 'papersize': 'letterpaper', # The font size ('10pt', '11pt' or '12pt'). # # 'pointsize': '10pt', # Additional stuff for the LaTeX preamble. # # 'preamble': '', # Latex figure (float) alignment # # 'figure_align': 'htbp', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ (master_doc, 'klepto.tex', 'klepto Documentation', 'Mike McKerns', 'manual'), ] # -- Options for manual page output --------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ (master_doc, 'klepto', 'klepto Documentation', [author], 1) ] # -- Options for Texinfo output ------------------------------------------- # Grouping the document tree into Texinfo files. List of tuples # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ (master_doc, 'klepto', 'klepto Documentation', author, 'klepto', 'Persistent caching to memory, disk, or database.', 'Miscellaneous'), ] # Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = {'https://docs.python.org/3/': None} # {'python': {'https://docs.python.org/': None}, # 'mystic': {'https://mystic.readthedocs.io/en/latest/', None}, # 'pathos': {'https://pathos.readthedocs.io/en/latest/', None}, # 'pox': {'https://pox.readthedocs.io/en/latest/', None}, # 'dill': {'https://dill.readthedocs.io/en/latest/', None}, # 'multiprocess': {'https://multiprocess.readthedocs.io/en/latest/', None}, # 'ppft': {'https://ppft.readthedocs.io/en/latest/', None}, # 'pyina': {'https://pyina.readthedocs.io/en/latest/', None}, # } uqfoundation-klepto-69cd6ce/docs/source/index.rst000066400000000000000000000004541455531556400223400ustar00rootroot00000000000000.. klepto documentation master file klepto package documentation ============================ .. toctree:: :hidden: :maxdepth: 2 self klepto .. automodule:: klepto .. :exclude-members: + Indices and tables ================== * :ref:`genindex` * :ref:`modindex` * :ref:`search` uqfoundation-klepto-69cd6ce/docs/source/klepto.rst000066400000000000000000000010661455531556400225270ustar00rootroot00000000000000klepto module documentation =========================== archives module --------------- .. automodule:: klepto.archives .. :exclude-members: + crypto module ------------- .. automodule:: klepto.crypto .. :exclude-members: + keymaps module -------------- .. automodule:: klepto.keymaps .. :exclude-members: + rounding module --------------- .. automodule:: klepto.rounding .. :exclude-members: + safe module ----------- .. automodule:: klepto.safe .. :exclude-members: + tools module ------------ .. automodule:: klepto.tools .. :exclude-members: + uqfoundation-klepto-69cd6ce/docs/source/pathos.png000066400000000000000000002314661455531556400225140ustar00rootroot00000000000000PNG  IHDRI("WCiCCPICC ProfilexXy8?L-0kf0'EٗDEDDHRHJ$zs>9>ss&A&?J6#>eii KYy `Ԯ)79rLGm8B@2>T)KX@ ޻vq;ཀྵC= @~==~N#AY_B鲮Au\*i@6\PIK? 8h[+wo ys064#ۋPlilnoomooD|p`Ȏ, }(9Pa.x<I#5(gbflU 49-$N? /꠿ؖD$T lU%r# j3BU-SlzeƬ&*!+zkc6wlGԏtvjqreuSte=f>\2*Dm?cs:<8H,"6<"?xJ%6j3vgbXb |b|l%vq WSSCӊ{fgfd^IͿ~/>`K⍭t%enIޖzW\G+ԫT+=~$Y'X~_OX +_mz\ђ,ܵM][gcW nް/* S_x}P[?]V!gDnc}tcX8ç ']jui*o=ӄLwͳvS?b5,a+W=~ot6ކ]|zۨ .%u+Ef [+g .(_B`DDl+KKݓޒ5;"7hܪ눔zF 斖Abp0֨ɈOs. iK=+'kʱK6ٶ۷;:>irCm{,G/0Xo}PycB T܏=)R q7CR2(YbrJ &Jf*jG0,_G uNӗ52DM4YZXYsa3dhW`pM']]/<4q\lOcmA >̈́ēD-p!185;r!X-GHqSKXs8%B,]d)쩹tmgRϺHl=wxQ=;|2%h1cEimbΖ5{uO@)p P@@4 vLبxp3_X {&# (`a0_%MX3#l.gEVpFG# H7d@#JHJӁBT-jV6vN. 0}}# ;C5#=;#4aaJcZg1hcfbcgKbcf_ q|&s/e;Wo?!%+Xz!eL>'aVEDD$ J z"Y"U(}_YܢLģWED4Ǖ5:b3Gi]ж}H?!Dkbaeiheep gCK/uxr3[{ {{{7A'| ROF0GFv1vLy FRg`=[_TyQ}]ʂj:|Z&;5zg+^]v8)Wo3Ms= 2W|dMo5`H 0 eî`e8/{F ыF"Ld (M$M:* DKGkE{3ygMOLLre,,y{X D˱?PЙRKUirPcUК"Ȉ"h3!z: M8hQ#/Bo30 0j3VLL9,,ͬlمٟrrlqr@k;Gg:?Z=={L18jTSWP=!Ґvɖ})VQ U@_uAM@P#[sYO|i96ٌ`*{XS[WW/WaDcWfm_;^< ,{f[FF6u[B$3K2S}yz 'B<3@1 a <@ ^@ x &A^ ؑ#e&-)pb^Ȏ/t1]q#{GvJן֎G?&P뿙]K'$Ѡz{p"$~)ΜH $ 0Hn$?B* UZHMԧ3S;w,cM8@Έ/}عC P-VbB P 9qݶaJh:`]NBSx]ƊπΚegt ^qYV" YbVTTl%ϲb=k`T"AryDl]nkok}!۳wu (% ***K 4$QV (C`۫Z`ih.{姴T@am>(-I@dqqMdg vq>S' >OdZY~`($q>&! CC0R7g{0_UH)('XIt y2ǒ2a!z:CͿ"m=)~h3};پ;oևZV>we ZsE~uz`"t$!b~^NֽiΜeEjz/}ť$=JPQcI3-TR$voG>wqkgv)^_( Gee;lw(D,).ur@R#1dٮ],jN>$@OAb IG _:\F &UU%e**Wm+<ě/Rd*+,IpќܖΆvu. αɯK+k AḀd(T^ٮQ"6ƒIV|U,_b__oy%CF{ 1q:52 hGKq9@͒4`z D4t)fpaƢ@RؑOpjZ: IvX!vɔMmIx7iuNUpT"VB^<7_ \3V,[n]JTCl$btڥKPRf:|~Q嫡΋7֯V:U_gկ|}%$MHE}JWb;8dڶkChx_oUͷ>݊WcEN4V2 2qD Gji$UVԛzFTJmzAġH NDŽ >> ]s`Rzd$emf :{(I2psI2$y1Q` rʖ)? pH.&#s̱E>z(zT+V~O%S E`e?|p6O6GNWmחc=1.ʧڜO|.|^' zcs\@fZ+T<581*dtR+ m߿ȦmV2V_Ҫy^ "xo/y1S6.#.m}گvM.LQj@]u9L"Qo驆 nw-tlCHE$Nܸ$ĚHÔ"iLmIJM"IP(IN3{Qb`Wc@2_L'AG%q^7&MlR++eT0%wJ58؃<_ֳ]߂߯,)γɦ^0%tNJ$7~m{A~Eыkl7fVqjttϊ4z4@>w|M[F'O6JT:i9hJŠt^G|"EH@5;$P_ZÈ4t< qKf# !j@KFީ Jwj+/PЅ(@i-6ٗ8/+p `BF#*&Eb'FzuW5ՁQa\-69ȌjD'Q `coFT4u`]_R IE;/^>.<H%( .Ê6M6os !cFXAKXaG^p4(29I;YQS+) Ss3yR9.E*3]oV%t"Iz@4D sUo͐Yc7m0ɜBw on}uNO6 ?/)^ilû>-̈%WGN=(3n &Ou$J_hS D3}1ɈZZ 8*&rڋWXKt@ƚ*S$NkUk8.嗆Ȁ c!‛ M&*P=$Ϩ+YpIn6ǟ"4,Pɫ<%j:L!BeR "OD m;^ߡHHn #e؛V5kMĉUV\ч hFb%?)/B,dV91_SLl>?wBgX'eji&GZ-5RB Hbs[]a-?RVnθ㿬x^#\Ɍ{p N&4ΖVN1[6oE䃴 HrQ5ht4@ƝlxO#Euܩm~mnmH0.#F0Lt3NP$0)MX% Cπ(!T^; PX ED"}S*# [?)Z`]ulY6Iӈ*4P6ҝǩ5:d ]pHyBÁȥ/}L BaoF3^HA}3)-A/uc 5HCMx}Ix,[ a >D@c#_/~~/Yb.1;HA)\M\@})M'udFMq:8p#KMة%OE:p  )3nh`&pcbvi&I.-q&0{Tm$Jm3''~ ^`/6s DCep(:ꖁXfRB1(k>d[ui^$T8pT=R_$pLػ68ºZ"VsMmOOp'j a=kCmni/~nb5:\`dEQfai=nko]lvij>htHPc>Ƣ=z8|/?h?$ȉ.0ū(.r =P1GHq Z3ءLw6o\>mMTɀQw'=&re\3umů.5u+,Uϻ\ U$La:nL@B3>t5_<c?*^] c.66\omR͛N N ]H7JLAv/5z}NH7*̣U' pt<OBTT)D AŒ%I XPi>q)fӆy2(Y z{n6}L3Z` y{=dwmtT&;Vly_ف_Ob&$Ɗϲ߿]U hY$Kb|O:& mWT5"ĉ)UmcڧBCv}B'􇣩0Hnme >Z`2H;vtɱYFYHEJ{?L-sd_3f:-A+FYt ad;v*\Q0"p^Nf|ZZt!p +ܮҳ9#S`CEh4ŸLFdbO0n߱v#nTp!6|C|b8<>8*\zPyyK_tɀOH? )DtVy='CѮ={_Ϸ)ZXQ?dL %^2&$zIwuVau4 }Aݱm%iE2F},GvW%i)ZFHCsiF.1.զ.U=ˬEFwivlH6U2g2J16h|%d -vp>Z8qD]dwPS{:۬UڑzkHwڷfSxhi#ؼzwbTe+:3̓y$[W|N(S= s [?𯾔mz+ʨѠ|HL(4L%6Q#Rv!iq:tkvɀLp.éC1ugz$)d+>Ξ))1NVpDDh肃 9{h?kG*#}94XQ6^V&0>e;Amٶ/lP57)^i{1 @Nv%^J!`_^t׳~5FBc"Şc^gQ9U:z"*lN??Ts*OF 4V Oڭ73.ihPA6/sӸCRFM.߹c7BVhbGҌwGrM5"5h0f℉N6|@0`mYHj2Ϥ_{{%-CLg:?:GsCnde 8lߊ. U&"qAhPAl& W_}]~+/~ͫoUUX'j¢Av뉜d?,?eZ LTf{ֵXG| -d!mcT6i ]7p P F[/ Z?U' m*Ӂ/'lӚ_6 <){PpCX.S~:v3q%,_q@x%i[8UHx%ݯu}2 C{6PްYrZ KkI~'3@@-d@@З6bϓᛃ?4WʀVƪ"@ƘQP4lμPg0TXbPo[͋_gE53Nzڛ 緿[nz]@]JF) )^i0ajԊbrg8_DtZ!tk5O`X=JGHr0U6óe]F:,cqu$\RCe!J_@H,tR4ͧCeyfga{寴e6glIe6$}RBd-vD+R7lѣy`i,܈F"8*֦RR jR9Tv3W2u(V\3E:U IDATUg[Aʇ>C?`{ ]3- .q]\S~[EwEIa7}{GmiIc S2av?䙼s xmF~T;tw`$qLj5F&MҞUDՒAl 6hw̌ 88͒ټM@٬@iV5I._ KyYTOW*OBB@Igf R"4n:EϋSQ}ֹF]ЪԨC0Mm\%UR>,tWqCXU@IJ5H@(|;:ԚU*wBRg(՜"^S֎Eo>2D#?T2w:,SXm-kKi0wT:С^$\-G\-mݲ֬]#Fd%#A$˗ 5gvf.QGW|w^vh$y<`f7$zwHL?Pq}xsIzxp֊ʌ>gQVaL:4AӧLHg(Lˣ/;/)nL efΜiO;,ڏnV\iv\ʖTו~]z-=y-[nW>.̀ L|hyH2U bŻaZ[l+='^_ъo@sߏe?|MX3Tdwtxz,*V~VM"wXFNN.j[p܈@AFI-Pmxs; SIE WtC ̃Qˎ [%*Fʪg^"b2  F>XV#׬1#[+[mڄv^]mD9͆l;5N <nzUP3gzÖq|>_h۷mtlA<_ҶKlض6Jn@nmyc tK;$0T6ve3fҀLOA Lw$ā wv؟x/O7,ZϝVzd :wz-D<\iP%M3g؜stI/xr\r_AAU]8P`lcׯ~JmܲYwZ:zҮOtx-+|LV\iz9cs.,=-8(y#!}$I:9qC)N<+PH$ǀ $+8h t(Sa xyioeXrt?#5%z~c؆G=0TX< ௕Y 7f}pTbi($QNzB0z'IXcW<7o?GWj>c 20T?ye sE5$QTe$\,u2dUetlEqpަmz͛6B4:TLfZv̪8 UI^I~O~mh5Z ηB{R#1Κ!ZG h~#M68B4xWrG+x d -wiFKJ]!d-nͶ 8Efz:S*Zus܄qkR|SDsRW@J]8 Fޜm:2Ө8Uga~:M:y5ѥz/Zb(AjW@y-R^]43Kdy_O6G W>[ο?7Mcv+4NL4yLw< S9V/CWK !o :HJd5{=OY 8#Of'5_P[K{@9V^`_5xH6]>~% 5?[%Բ/!R/Vn|8z`!~ze4%hP*ɘ&t:Y̻SR MA Bm2AdHg9쎀'I.UkT%pf _Ial.f:0H< u ^J65* .yHa?nhAߏ>\=@ą4'~rްM Wfs~Zj@IZg[#,{T55N|Pٜ:udفYl\XbA]f0 턲Ń .]̻w#)17s0He:Fz|PoevQT4wѳNR8ɀM˚5@nj;l;6(}Z/K ֮@ iHObϻ;Y3%~ߜ 'YybFAB~m{H DKFq>@d$rP^D]N\D쎯 4+tgj.}*Cc"p #nЫ |t޻&ȡQ:a % Z<;HRk9\1a(F5Ja|83WEBozF!p8ԽEvHԝ֪'ߩILׁy ˮd4?*IYKZPWo6wc?`"c%>@xi:yqtMRձYPBߡEA6^^/⻑^x Ri7(O"(/ bk2mUk5I~D?Wy[luPwS%c:eKH ;}:_o@)*.% $Hg/ux4] 4 }c/SK+z ҬZ%9?‹l%R/g1S08EQkAM ='Fr_)!K] Pzpx–㠱f\"UO:S@1L= p LJ=t3{n|+; 8U^H瀩 (wsu `1QuEtv$oKh/{W ~N!E4}b>F%m}__W~(i,^Pғ$ P"K3}0qå%Mt礨<ô0> K})ħr`yI#ti)Heۚ]=B-4?_H6mJ5]d }VO]v|PjgYh?}=c 3u1l\M`]{g.ٯ3 F朘DGR(X4B?䤔oDo "2 Q!-v>gwka$EkN+G`P960R!{rIǥH"O9M($aŶN^쿹f3ʠ[nQ-WxI ޒX(OWnl\״t @/PbN鹰Z5g)N+_4NI>nk >w9J ,t3 zu=Ț/OhP84xs W%OQ(P#x}V VٔWmoz.8KB E0sMtF$(G_ͽ{Rh=ˤ.a+?˴~ꗾjwFZz2S=՛P8#(<@( L#=::kʨlThA vh]AA?9q:,Ɏ&;} ?+: OrI09|iCd)#>/{(ט5u@Η:6/hܨ/|?"]([!~@ 6hFz[y=9 ֦4y `-'%ghMeAdxA$o|0 a @'x/~/ !E<ՒfHz)) : ;8Yň"r¼0HH/ z4P @KV <. jL|B<F= I [&}l[|)\aiN?L;Yѩ{#6/mspL.v tU*S݋B44Dc]yB0YRxe^B[ϓΟk -+PQ`ZB@8\]&K4ol{ֶ`(B;qfv7k5*?6`RH1(k4%8ũȌ0R I E  @{I}6ﴋ_ y#PBS@Bq @H# " \ vO';]Njd< )KLf?ۖ8ퟛJTZn^ pOS>[(= 7P z5A/h tWpBW#Qx&q>7Q9Qhf IDATH'߉k s‡O$yA6sG{&>uˌnvmחnz[J CKTCHA})J+-[3t2L|!cDH ;m|ں7j3=NU۟dlt[ |`VbkW9[z[DW"PHw/}(BS(jOz[j4?a>;a @prQLHaB<;i!wOd1i鴸9HRbt! J;h P`k QԶb⅋. 鲑Cڍ=:wn?@ =4=T'NiFptKM~"o'bqw?C'7Wt1.+aჴ]  zӍ$~D*Ųv5?|jIv\S  9Vul/EOYuŶ~[w)|t*&m~-]UJEߕ8M׽޷רd{mN0sp!G00bFH9 ;  G&jEQFIWV17ѓS2TKgA-7{ 6M?k Z|23R AԦ8f~ܨO@ug!N5غ"A ]d29@JgYsϴ o՗^aN=%@D󯟷m\,ǽ{eU^Cs09 2v{Ga4HWK",<U_ER!ON]ZW[-WZW79\qm>`4U5e/@\=<~"M=g|AA#hK;dUژ>-?}wtp~hgVo`7,]ciSfLk F^7j" ,3nV9`g0gˉt:umD`LI'.}R ɪYz۶^{k^k_lk}s|~y6s#Vqj9QIxГ(0|GKm_;P01H[H, cBA B[C:lG$>몗JL0`l>J*-#1A[R,UE`Y)._}Ze@Au{Es3~NB>ɗ.ЋqeAZ" қ0u)%A6?H5k%Cӈ4^YKL? ۶Tugh#,zQjܖ঩o֫܃m?ܘ|QQ1kJW(Ufo@QX>f*ajt<huϻ^_3/uZ*6gnɒ$+00_]̙rF<—FyThDPt_| 6)^5@v'zGΟ1I rb)Oci$Y$AVpU Ѻz6:]Tpu W |wꪛQn ]W[[߰O\kE3%E,'tftkeF a `s}ӰaQ}r\>7hhWE'>ܥC:h' Jz&q|jV4GS 'ghua#%yL#qEW3ۦ^<mX–S?lt(fG{tس%0]lLp5 Ґ vTmMP? 'ilƎyqrvlÚ>9LN<#f.]ZuPja,~i@R\wOu 摤}((KZm[<(9:z}0!?t#a)$d[Nr^_zH h92$ݟG,㪘es²'%ӵT}nwtΑb>U ڠgq㕩2Q Ơ(O(eE_lP)ly0H΃4١CtqꅎДp: EE9X5ІY'„#x KH35~ FHlČ5;;](9BO;p;@zz.BmGx)h~_:gw~ͪ.^lc:wǰ8$p5G#T=b(Jl aD*/T<hA& MC"Q*Jg-n.ŨR]n DFZm   O&Qbl-;n-U_QuGc.vhG`EMQOH+i(N%ٜu'כ54ѬUB\DPH@UGU QAFf{6ɮS13(>x|O#5L$`tr$Wu&.5/+龧qB#J6q_+)42Q8X1؈*Zo J=O3Z:r DUfh)iPgժilK-Pry#(&!iJ7U8:ߥJ:,ؠnF:u#Р ^ vg%ׯ ..SGlǙH؍#Prہ_}OtDtJcI@2Q\׉FGaV3ÜcQ8ŵMiUyڊϓV0dIy,s%cOy94:]pƕ̺vnҡM#U8"ٌFJܧ[+ 6>oI#GuKgx}!tMN%m ,#}pXa{6jg{pw Bo/o@M}Pu<>||wp*㡞2jRqԵ r w!?ܝ<2r6HI&eܗ{鑺/](@*ő4GsyfAp*!C@ t~-6IK"} eұv)aηNi EJB8+O2ogs?x_f WC 3`%kG/A%hN'Ji"pFk: > zS8$vttt Bs@lko ҈VDT&q9zхݧ=֭ŋ<+pQ}P91byhu)&HsuWNh 1?Pg^IMm$ӎBR?NW'((e* V+-Om+f(fۿ}jʦ^vM|Cr$ l$@Ndc"IVGDiBlO~bOr3Sa[]颾W`A J{J7I 0Bdo Ns+u # Q)``ʹH'}[X5."VۨHы@*S=5P|^\y_PI+L5XOeuP;ѮRc#BNuXHf;_F]:.N5o{6$@9ĄA!8Hg@n߳z5Fs٢xKTVtWvϛ2SghJ 5Y up`ڹ|DpQVh4,TkrG$HlG "pn? $*<}aֵ+o/+O^=!)4ű>g!q𴷿*/̚W*!x$`8GyPM`[e='TWpKfR&U 1zL\ Qj2~GYyݺuc-Z?-.a:+Zҍ\gE5aJUX X4u7ýCrGꅏr-nݺU_DqݴqNqRuy ͛6Xwv&)VB:bz7ZvxM}ӹs45w5); cM/"GT`gID]Ds˱t831HkbLn^t|_ I;Ux/4Lzb}Ǭ[\q!ÑKJ@NĔ ׽u b"Os)^.$>4bL8o?V$.a 0\x2S'(#mOUH*rȿ*{ o^n ;l6!rBwk@]b2t֭[>2?cm9;HdB.^C zzen2֍֩S: 9\J#u: :w&nrаmrx945.S:я'OY3+eLuƨ}ȼSFk]>m1t9A2-vl2TT~ԥrV)hb6?{,kCΕ恽:q瀨Jyx4@PP-4ng$hAG$-D TH88.5}+4fPԁ&@Zʗ۰ ZʅHOwg*@TR.Hރ-P9ʼnc8WWFyF;̳2tXՄ hL#ƫOo}ᇭJȳacAhP[_o.ڼeҚ&a ճx=&C%0BO$h5$MpHzd@l8Td^tX-_RcFu- Ifp\N@Ј$rlyT4= QÄHEr' ^L@T=;:uk$Lޑ̡e~YN !,@ٙeھ)peT HسBc|0NUIoR͞6Ac9iLVgeMsTtZW:@:$ ͇ ԰Imf۫k)_x'wujPiبNUʵC<3ub1sM|sΣrfN /'I [N<[/8ᬐ"8Rꓱcn suh|"2X)@$BMUpDb6'bxh4b ?/!i#aF2Dq1Iʞ@{Ψf.n+g3BǦA텲7WIJ8wU6Pӧ8$;M|tP^:ذztY~ tMHPOu ;wxD )RdF`]Tn1઴cGF1cgn+eI/-ieVT*:G`UJ3Pfv X_B-Wڨ'jт>͚t?xUNk*_Ӂf; ՚ev:,ή=fq XEڸzu֑`ӡARDfi&8J[!iKІi q T]met3 F@(wf`@26&W?iWeYQ0ZURQ0"q! `)ЀKXGA6)(ay%6OSb66X] 82yRWݯ:Ou f34_Nh f͞e[l f؀G򜉟Uևͱ͵uvFaէE" |eu$?m~kTF*j[3Jm`bÀ&S5󣥐s882q1۞]ivEv^a/[z%$_zU5<w[a $ڱSftZ- Iy|Z1<Tݚg@: htvY>gC80 $#`KP e_!BOΠ{$I/ٳg۶m[>74nIa7mm.=x)F2 ۲~@W0ɱSv.Pi핉DHEoCӰ.sI$-2 JN ^C/]jSj4KJxāNB'ɾlϣ8+}q5=đ9l˼-7kwR]S*J.9'.h[ڣtQ~M5O*JsbJ9 %9JB9Ȧ#1 +ei'yz4O>%.VlR71˘a]ĩ"01=E{􎟈]R/b;Np$ӎP,g7 22#YNjRI1Qtr)P4+J:IDE3jD+LP|m$P~<}TGh Z (swN<$߃~)9M<%,K4۶yBo0/z .s_<콌tz^t+lC^=[rt`uA z8$I:r1to"|-ޫQ{79X"=>U4>m͜9N? :}c؉NXdG"ⵯ+W`I`ι{|j7v"8Ryt ^tI,=|}J%4~H.јGɑ 8L(!Ѓ{]%(N#YI0#AۮALYd5Ȅ QAp$|!9SU|gO…3%˲R?x<,@ z޼btd 'ǎ :}v}_Uox&:30<)*Ɏsz(tL ob|<&ynLIs4Xsye2N)7zy$z^}啲I۫W=Iqh$.eڬ ?*_.EA0%>vJ~X앥-!QԡDdezO4 8!ٚQ}aVMfȷ^J{w4g~Js@QΤ./%bЛ{߄ȟ{ӎ$zmuѱTiI+NO~N_y9{^ 珪^o*wv $ͦK t 2&n5$ RҎF)F]: CEGbZChH@! G #>Qp5dp÷o!@ BRNJ=2W\hjwJ} z<޵flɤjONpbVz\-C0!?Fx D_!M1DfDzIܳ~$qa5έ^00} #y8 qFs zZ!;ƀTJI8G^eݱYku_J8F)>;SX"ۢ킋."ڜ*w ѝ ^ʲCHfےel=vZOsc;[Ѡnzopw{$DҨ&iyğ`taLLFH x9DO40Z|VOI>h_öS9b; - '_dݰv4`~P< (=!q/8R1zt`q iIG\+҂V%I%m ?.a8FVˈ$Ej䨖3f،Y<8:h$A줺dزܫ,Z8vuv/N|SܔߍV)mEZUIH ͞s6Ht{Ϟ=uo̬YC=COڑ!)ܼ[NS^$+< Wᆰrْ~X'2RcKm3 "p!!^R0 gvl%8}nv&c0"V*m5Zb! Ip%ZfO`8eEu#4Abڧ,u}Mm>]Vsj3H \8'&~W,#1pCusYE]qU>hɒeޞ8DF,e1, śA+/~7Q+И iI2%H/\x$c.a5w /TKw <#9>32c<Gތ} D+ =)I#UVj Xb<#[q('ŭ,LnZm՜'$E ꄷݟZy4qDl?X:1+iJs|9/PIucz#xoU_q8V4giwW/]IӠCȕtl/ThrHx_š0YG+7 1`,7͟#*Mwܡ"wH)5ҏ̝aAT&fw_qrWkm@i AO^/2G[ÒŋEyc˘:#G|<E{ر 2k^r,B#oXt.@)Ճ",#GCUF*C׬|w]>iq6JR[g]?5C[V$07p]סhϹ Ww1OR'O>D9{#|"aw%.̟R0٧\3;^-l>8,T}R/xD 8DbS:ixxw,HZNT"3DF4Q(z5a݆ QG'@ e(=F&p'Z(2y/̉+Fqx&qJ"@ n79` $ ,$l!8eY30fLT7J)~1i:M/{iXk+堻kw\J")|4c%ŋ^~ы$΋GGFQW:-$J><IW:!);4y)$_Ño$G:tLRgq:)rwFC]s󭷪ԟ܋WYdbir.(lظ!Y&d|F^WEI3@8ÜbFg=[ڊ_7I88]'p)$HKW!ǜCp9"Q_*8`wG"6PS"0&7H1_f8Kׯ7y٢ B>+i94w]S]t?~cO9pEUk΃0 |bDa,ba%?%WRٵ[hP~q?99b\`)6fČ<5H@>ůz*+$/_R& ^.D腡;޲R&i d(X=pwzJ(@xG4GHwdqH5uFU<ҠJ`w0p zau Fd'Ц>,UNI~c$eRý/ G}PTǽ}/$pUCfUڸ-u8OhF:86ִǙfǙ=Q#X3#ynq,O03ɴX;gIT׼޶E z a(ÈEI21 _\ $< _7>D"wģw.?=˜y3 I͠%RJ(cEj1'`ɔfkZ,rJʌ {HNHԐ\uGd[a5ڻ+H̨@ RʙUK x{K3ewG+gol25-ښخ}q1B<=CocgjI~hY`y"R|ox3,m=?(ЇlA')"ӦM_rIغu-Qtcè0 1-DO\PKqޏ9a_,ч!+$過'.~%^wS孒02 H" ͜lL䉹 $,务0Y;Q_6.j1Tt_b<={ Bc>5ZܘM JT(DwX+L/Y9A~I:T&>=ۭ̔?0i0ed[=zy~g0ɕ-Hn+8D_իpktH(8HVsL8yKD0FT d ^E@(.w1,UQ XdIΙC^Lh.LK8ՠ,ϝLUH^SY̚YiP%'iS_(wIRL܎2K;,@Padzﰒ&z.w;80 Qas $ب]55>?\un7a)Vp iv0?c~G?UVlb)iAHq42h z$bCsCvlkȨ 1 Hdz֢yb7L,ڥB}4.uetɒiP9V2ԥ-v({w 'Lޏg?Aa!! 3I6')Fk1#RsoVgy0;wNx>zc^KDdf5 AZE>p8xW]m;eW^ sB!ͼOңET@dc3[?Q:#}oɅx`J aa>M!Tk#z὚ +6Fow(ke1IZ`l:?E@YmEru㩋./:mH$&M a06u%L: A.wp7ì8j쬗$x> ̓? C80 E.*@^/2U3LO,CY<đ't~H- oi)s8ḧP=]zbEʚ>$jC$PmGTq ~̵3LSYI_ʍ;shKS6nfhY4 )XzamwMGbVm5d&4\^ nQ*%r%PdB\Y3&pܩ6Śv00%uS]XVv:d㒜ܪߧt!ü' om+BR#xأiڤq &8ڈh,A2Ql HťY?{y=k h /o}փ\0^KXts<@Z?`,3Rc>WW j(NA?EV"cta+Ze2lnp|d*APj%2Arܩp25V#mLj$o%]ZX` 8pMacPDfsG|<"씽ϭ2 DuXᨛhCvNaLDYv?3&tq;]P-hlͱ yxmw$a+d: :c{-w 5 1!ha?!}2A [M}j7hD#Ed)RV!*,P&Xw7"xUù:x'i 3\ ,3 Z3fR){HpgUxsDRp)߆;,Zt؉(.%vK3FW@~y:#:G:q.hFC0x J2J „ IDAT )_;H{pȾ4 Cg,I:m~ FH6 u"}PپBIP2 D\e :zAl6!G I/k%ך<>Oh,K0s\FxPQj+" [ I9 ;;QZ(1*+4UEg٘o>rD {Y"q\H;vlq(W\9_Voא|vQ!Ez1 pJbyޱD.HGc@, L6ئ=.le}Πpm䨻ڛ}kgШ?? /Zb@gdH] ʫs%}@0Q$xGLF7 K wޡ3`<[ygoT@M@*I'$oPs`B1 s11_؎:_9+ lSf QRlj ‹ݙC_AX =5\0R9,+;{,DUDxN]щDpYirxC10-@Mu[Ct(FXa#zD9aQfzC6,_ՖOp4 gց^t=>iC\>y޶}%IND6uFde}K[KS`gqMϷpW_v I*x-ᜳSd&eƊ @2خ)O>Zq8e;z#|SLG+\Bxh(4G֣a1g(G >\ùcBem!+Ia? `XtYظy1#Ifƺc:0>o/аPW-X_QPaJ wz:\C8:MEBA^@"̵ШrHuvR]XO;E CevFd_y[tW;]vqQ Ef i,A"&q0Cj"!d3"C] 8Dۿ_͟tKpIE!fxcy-Ñ$Cvޘ ?4H} 4 Qf 9+[%bE"PnEg3:uJ&hGWt|eJA2Sj\&(gh~5 U  )%tFDlbb$56p9 Z:W[NjڟVZ4Wv&;$]M25\wn҂>`mK5mʜ^=/YDGη.%{9C!uڞsq2iOw4$2=p0)􅂸%Hi?duZKڳ'צw W:+z/?t[@ ^Qkd:0S7K{vхZ͎'v7d!jbqF@te 4HjD}1LhR%: KUJwjns _ Zh)rEa`ĴJ|,,`fxg InĶe%fi$4da&63Wt Maًꨛ1G{kނaCپ_9DM&JUz*L7NqaŊᥗ_n.m;YFrx̢, mhl-Z(}iޜwa)Rl@l&itjFׄ1C pSG>0Z1O EC[aTRck#t[A:$-F1'ifw$L` w R! Z/xSaᒄa)RfX O@=4M^$ߍ4S\pQ@:!FԿkяjgz`X׶ƊUJ7Վ B -AH>RaHWfRiȃ4jƎ*_ 9[[ojS7i!{QfckԹhWK$H?ǟ۪clG Of2)#YӯD|G@ a=#)yzEiGbP  Dz3uJpΧ9)y"YBO-˯Fb>E?t&s@te+Id$Β¿+ 3ݹח- / ~Z9pmeX.vY`t0tu|&-0/8 Heڬ=sXU_x`1@4tew&:n$_h  X!@D&/Z0k@G'P)޴ichyeυTE)0PEr< z8bE;$d~&W8rZB@jXzp$[yȋ R/ʾ*;tQ\K8F)U3 7 NQӕFӕILǪ7"~ 8jTFLPgwsf&1a~-?)|XVPfl SIa%,DV:'q'=U1=AJ+O޲$H_@rN V*$qYʁ4/wGNS)@kwpUfFy4hu*CXQnW`.Nut`y@-\tf_5d;dAy%pEq8  0Պ+kaogJ;XA^ /}07>dW24͌IZᥜI5#Xbĸ%2@ |$Cq<<.wHH82 Kp f ɥɡ!sz?8i If ٳU螫U]#A"]5Qvm۵X2dixvGU(ϕu:x$ﭖ*)ti+yo@“O=B҆ m掼}hϠeA2I\Y-bCؖ +R TH ﳥh~Ż ӧG&58H@|-NցDGԚ4F_MGsc:DoAfB] mZ-e?OӓO V(A2?)Рx+:]@ Lm21I\js׮YC4Cnj HojSEqsX>">.NxF]{CZo1,  A;ky8gi9 /o|tthI3@Hɗz޹ɽ#DSξ. Ks<0R+-V$R߮Ϧwyk"=}ŒY/V)s\.DXYN@#O]G)Dg!J1v?#2 /l jdv]rKD|>ܬFL-(w v*4[/epJ6R|{_6|oXwkT4[[lǟqXxqS~s78E=\}d0]e%I *FcoETP,5dAʾ'GI$'~Pι$,_FcL%q#~"cNTgb=GM7Nk&ሑ}Rb _TCPf( +K. ښ*OZdF$ӾSlۺ5lCGNPZ64eZ:qᐤ}с!Eת~lAGRm7^13TlI&}} dwJȴ$L5:3NG& GVGU\ )#p&/iwYduvn0Qk 1R8g*X^1V0+C$ ݣkb@F=a-b||cŒ I+@2GDhšW=zv( Qqea]'%Ow=RapB{嗭` H<ʶQh0s{@wݥLᩎ}ա]!:XJ9M'-!bzHH.-FVߪX{7D,fQ.Q FA~'s6:;~I50\x .4-P'DDL$u}8^DTGK<߲IL:@0=(*!/w>\ib,'I=`yi{bd$̕+%$F\!u=w2vCٖ WN~| J9wF9#*P/lg۷wOh\kjKGg7jY ܳυ.\2\ygw¢EΝ;ıw타2Z# 3tJiB)<g7]U;̔H[7kBsNdϜ93'+͟Qaޒ`*FfzL"\̓bB_y8&pz-H+tZD* LmrO(b˳@ +f ["*7P4: RtnOFq Y"_޵j~CI:m2zѠ-{$-(qaqO%vkzI;|Qv$O=~Zi׾W_O?C=TʞzNWD!pdahBIwu7"T{{qO7tI=QJpF780:RH~])?L4E9_X,̉*U I"/X+߾5!}mb K',$A8OR"l=*|O{ /_T28+,q$3L4BA3 IDATΧ05Zw/%\r<ڽWxiq/L7spט1 @9iqd2x0L56na]Ev(zR[ciau@m-_+s^jr0 (|w^Kr<:šaYº;&8L("F5|iH". -* z ɼd`T9S:7s(Mѕ>IB7L<8#!։{M7C]ѹ++!dHcQ'IJIiu 0Zt gfa0DšP!4sLMr4z3V;ח, eeKo~]u }P Kd]/6op;Z`>*˝ڍ*w=iJ$ςHG:uF's`??̞3'KN֠_edi,UV:8ZaB%LW )&@ٳgG "1cN!6./24ĿCĤ35i?n_&Gm<أW dݒ<%-WjDXS;Tdtx.\fV&H ^9FZ[&.e|Rnm@_4 Z^e 4d)@2s5Z62Q`E2&2*a ˡ1) IH,aA0r4*Α(cC8"g,H:?t5566GX:ê5jVs4䝗x0D'5۵}[x񇽢và]:{ƮDI uɓåx|^ (IHmܴJ۶P}Ns|?UMX)(( oS}Q͞5;\-\`boy)$a(+.]mF}kRdцf_oY%|}jқ Y:&$(_3PCC0}V9%vT%bc(qW2Ѐ/tK5 < MHpir =!Q2j eg̚5v jL5ŹڢasO?!ZyFtQ=%pLYqqiJ$ML/ۏn:eqv$[!٧Wtի%aN]QoH;ZG9WB=31E9ړpjd (_oT[*T}3TQ M}{eHyԀ.Z:,[\ <7IZH0(Ȝ$՟47 ;up2eT\ztߪuhԼpdP$yۦ>Y pM(RD =aJ$<0GmI3τ)OHzוW/z/yƏ+pxtl<en6`b WUwiD>UpY. ,R'h%Zj_ox^1F_4J>)wZ-ƀ=D]s] _B:_dE2=g/szǥ{=0veeϒn>~!Iq0!_l("9j8CHqp@úG?qlAVօ}4lܸ1;Be2z9[帞G9 ʲAWb1Ϧʾߒ~4g㏇ǟxBYZ>õ]tq]NaC$7mem:H7,Q(.J>v,zyR$sS' uuŅEV6^}G#Mh6n;Y'zU0JF5$$M(NJ#$/{t7V=i[tivcx\|.& @Dąމ"L}Eo nIqamڲ61NrND\=jH`jh;)e9sUaXfB]6nX6H`b-Q[#N`HxNhs{&&ؑ,^W\k葇Ȃw8u7hu+Ë jp@Ṗj:@q6m@qst-;O'iNEK.8ϩ k.?`ٲ2KwGm_R?gyVJ;LCظ₾+552\Y$ d"Zڢ|Dxi-},5)[6o <iOT> /{YG4K/3Lׯb=֓Hf4Iz nUFޣEweM QkȄ1b b  Yp`Q<[X ~~|d,Sg!Z?>򈆫˯2+Mk/2fxNNP~f4͔9ׇy01yJɽ;gpD׮Yl@7tz#H[:~ue Tyw Uñ4Ø_$zZ 0`8^t΍7,`8~!-\14RCbՆK٬=' b1QUG6z#WIig :e ]4$zM*:?px駬"sLiFeO=WPj=y`䙟=op$w]qU]Q}?ha]%ɒR[R5c:Jy:$O"%\>z.`k:H顃#5 RI[䢺c!7ChU X|[ ]#yavYʰDb'U2y<`(cȩ]sW5k7}塿~i8AY׉IL͒;4g6u=whg1,J/GE엢3ܓϞ5'lg^qpMp@a{;m A0M#&V2'o)jvtMd1KXnnG'?h9&=SX/vV,?XXlZd :0 8Q+u(]_C2U# ҜsZz%)$UzƀmnV͑"H[8q3ZN@utL],NO(!5P_Оsԣy+ <$zM˥tٲަ6C5ypPe5ք+VX<Y [(Cql.rvOֈQ*M勿꼦|A{#5=gHq(gi?k׵uǏl^5HvWZhPUKr-A1 =xD85 0}ƱXaq!T+O\C4|Ao;t|OUyTҕK.M%@obe ߾ǭ\"LXM\R(q j DxĎ7MH3d= ~Etq!>YtW,9t`K/jo.[󗳵1a:&Y:oqگ>sܰtabP/"4P-4+n l_u\(OE^NӋ3p.wE+!醾,zcht K55[\A8;ڭinIm:mjEm}3_)CKx?3RNɉ$M&Neg)i9gT$Iڐ[c=N~7£ X'X#rݸ.-7%^J]&xbBG.H&ProT.8YtWS-M\Whu\LOznfP>?=.s:dBD\&'@9LIv3*BM<(+䁸<:9OymqkL/ hֈҟ^!&FGʗ;{$$)n2JIr Y- c1g=RRcmMmX/{,s0=G e@E؇=s_8KCNMzn(Wtﴅ#EMzڳ6!~- :uy<-A Y[niOWL Fc^R >T‰4}Jr99Ο#d1cADT@04=@ٻ=䄾=F1 ^Eq;Ja}ɂ!q)m&N HO5E3I_2;"GV;$גV8:6F1癛R0yR2r=yC9hd,rUٙV9`(յu``7OMt#5b}jcRajw5*䃽k7nܡeuXh]Xbu^\1YSSI $Ie[1 ? != $(\E003Cvu# Vj:BqߨDYL;! ڸ?\sML)=kV@<$Hy%Yq'P2YAJ?/w]2'-[>EW6ԊcU0"A8].e@g>SP\wW^~D$#j(+59Cmfc@wNT9,&YL.a`! URy=9 9LCpˀ;^-t\It:R"L A>4ksfhQFSِ3Scprl٥HG w5@ޗFt"?3ԁe>HS6*#HIuɧ nt8ׅlkBՔZFz7cWN=q]tt|$sS8tޟl*>y5-,>o(@flrTFs_y"&gk5+ݨ{KRQIX0OwĴ(OoV#0 F$0kJdHKY#d@ Oq9ѿY<5prZ#4ۢ#Ie4p59UUhjoڼɋ*G|5腺9"euo3LW#twvA6A/4cmB !IWh 3ca$M3nI O~,? c/ oy:N)!w7KUszߩv F2 9lo #d[&?a')˳MA׷Pz QTk1*6%1Y&/Gy6kd,{e-f ĝj$ 49:fä6M[:ٶK囬-EqʮCP82y*ȩ#qG, 'GY+ Lg$,<ۗ<ӷ])cpnU(8g?^|9LewG󵿔zH>`sHhpwLѭnŨLEɡٷgD7 $0d4Y+ @f]+0Ž@ Q!9Ew66{ʾ=[%Zi_zvMטaOudIiUĽFIi3[+BW*,?3̈́E#trkp/C&2\ddksKYasݪ(= 屘"i">{dG(e&DwEH84}Heu%!7{,Y~/f86tM):T0VYt$4pp: Aa[7rZN) Fǥ+eY_y=>oٺ%zk }*]Jc=S WYBf2T#H(\E@C䵸8?՞-|٩{}ѡ42FpJ_ 3ĞɹS.fu`X["iRaMƳbt));7 imO`ULKb?6m^{%J|@_j[c֋dh=[p$}Ko( %*YfޒH[FwIP)HxDtƙL sՇ/7D; LLDBbO\|Pb%ag a $r \i 3M1O%p5c]<BUq-ݺ^H!Käb[H'189XC3F@S'HqN76rleY;sUz fJ#vBTy)Noph"$0+<'tFP/24Ć I!ͮ)nݺ%({'$GJ5q6!1jEr^%A7=NEAA 0\rIgU~0dA#)\aH~m v#x`X0GF P׈‹nj Z^f~2J̕G cb.hrTD5hN?utH@o57 A2F"VFjL?q)8G D`9^ E?|{:kr0qw&*̓e2td@GK άW1Xf%3T;6BU픨2( 3Oi;"]1ojfN S^”TYita?%R88 D+_7ӆ+$yW`h=/y"^0%JUʣ@wH52Q`Ii(Zp8"gΧ1=b;FvLĘh(`K=͇kK8n iHgZcgܸah_CwV_.a,H&4#؜/1Z"izLKz Hz&jjB₝:e{N_O?Lhޡ sELL-D nfLB۶mMXJZs\VH LJa:sRCeg9N›gHeXG4Dщj e@*٦XآX}ۮCS?c3C1Ydq;t|Ji=42hźBGll Z勒#a(aѡSQ^_[E$D vhsφIJ'),OW.ZD(M[66֠1QZ(c>]^LAݤM~tfÄ lC =Ӹ0ac" DxDs xp&Z1+5u5-lT~2zb2Yz1\:y _1s~3-Ny? /!*ۍP%j*RzSP4 W,iy#T;a>Vg#Oj%LYtuT_5\TR9|%dYy%pɑ=o9=RW: yϟ=Γ^6>`-ˡ1AYyAx3ǮWG}>,<4L!8hcaT)CEd.JSNbC](3u4FGr`[QG1Rpp 0;/=tHa R! ZI2 !&&Šgׁ/]Fljz> Q**x[EMdP=mVФ:ʖfӡ* aZN<^5/^o*8ҹIGA0\amr&'HH% ,5m4A%Wi9['_ghT&9cƏ Bމ @Va(,9a)x8a嚵>)ouC's&N ~V_@ dc 0Jfs5^5 S K3iZArL]E][Ш>7*l΃š埇;x]+[1EzSr])B]+p?xĠM%p& *jåNSdž*阶>Ε-cG\,FT,S~Da޵HvԓO#bĂqɒfئm(GōlK'Dc2y+9N6Hn#Co4?9MЋE0*V PXaRfŽ静Rb߮c\G[orr`p0$İ~ʲ-0 bo{C%-IGuˌH}uEEuxY+;eZIbsTջCua!cCs8$DD;>+# JN'#-aVjjES0T#S ZsI߲#u*v+6vK8T'TBd+P#OClSS2Qe$skEEOJ[emFTh ,)V3݀I*h3kCC}ysEur3kBq">ޘ  ,aH -F{:HI~CMl'qHgΚ-%a5sb~#(B)_g1 aFi~TIۜ ;ZJ6 HwRVR8'9֊jvҪC>ŲSTa/P*YY-̳M|L*Q5])nS=Tf< 5RTGY)wjXʢҝ {4w2\-xˬ;V\tmH{:-=7e: j ?J؊ܭCS_0axԷT&?Hؔ6e"W!I״Sq{Z}yORFjQm3诃B2"}Q|A&,WKce~vl ?TRKJ\s I$'Q.R;':VkRbJxIsg -MtEax.na<^ k`?/jtCן+RXao:J+&p .Fies|P!;>ܲYCSaJҡ.OX)Ʃ uhZa%G@Zx>XMx/;ΑzHPa_Hè`vNܧ͇ $G)u#YgϚ.)wb&Թ wHfV9⽏kw(l%ϤqꓺiohY^&Xg+IaU=UQ? l!hһɊeP#93' i&zMTo[-`Iu#.{s? $"h!-JTqF܋㝿q$5>,]$lBL&%@ oQ=gvE*0_X?y?UƩ{:?kppjѳO?c7nA@A1Sap06g3RYA6WkE%49E2܈܏ngoeqћXF. z pOțϳȬY$R>'KhMpZ! NbQRz]- 3/X> K7dj%N9n!`_z2 8qHݻ8 gC,sB>q];ث)I:!jMOҒȿe\Ϊ0T҉8Ey!8:(1cyQ;޼qX,_E~g &M߻g ul8⠈GB|-)uW/R[U❋وյh_=qssU. zXحtX HOc1bv2>Gv sfٯ.鳀7DWUՍ^۝~QGJB&?y|jM79SO>맫Pm5-vt$ks4[|vxeE Lq8`< C$h@U1R/[߶{G9 xVWORi+/}Hn9gGI(,Y У ]lj= LZYxB);@z*oJGRF=u촽*wMgS57ySva;a\ ױlWx}*S-Dj qȮ'-/ʅS^۸h#eY8>m<ӾUku''r_x@7Jsݓ'OqNMsQ\ IM -B|┯{Ǚ/bY^ Cy /bm$}U;^h34m șSbތ$Hll PgNuTXk/}w~?7L3A^:XzpUy$H.]fjuPxG9cA]@# Lj~k[LI?R82N(GE+Z!MsWa?0/ck9L 4<ɓ~FUvy>̈*lӁkik  IDAT'sIY&I ޫX]Cp%'dYaHN:0ze8ISm]uT8 @@7bATH1C.wR-'ϙ(F0b뇿7etTa2yWk]x{$Mm"Xz:K*-zighmvcS~fG!@= ;e$f8io:1Avd;DuP PFh P8~L?ϯz-[A?ɪ?/m;MsH3N/|@UXPB<NvǬQ!]N?"\zEq$伬Psql l@OCGw-VV0o F& DYke0UZ`L76Aʚߏ\I$H1b3F=ٷ|Xϋ? S{O& LeǂO?~wo7mQlu=]t 7jNitM{ aӛ < =~@| 3Njx īޏBi[P䈋̗1ZFY}6,.KM顇ķb~XO/":zaBiC٘!<?/mb2-1>03T.] F7-%\ ڤiv ,eOjpqS=ʆ'}߲7I|Ÿa06Wc*+W^LdygYqL7,Hz!Dt k?kɢ&}Ӫ2B*CM!=< Ã|_er8y*6GЌ7m˖/P2X:8R`Vhil}z܆dY߯9y왧 NCɬzwOݲ?.m_=$ɟkkYsE֫ P,\ߪK:^ Qm(_ s/  cQ,<1]<2|( ̏d_ʗmݺuVYk105M#@n銕n>*-oT%M 41Oܭ<]e7 =27tK ;:+k_ъ/9PTCE/_ָڵ6oH@xf,Wشi3 C驳gl"WFIAU2 6xB0 s (q([*pRJld2u˫ A@gAs~ vyI$OG(a+;Fz?G]fE"Nb1r&L)A^%!#>2~//P0Wҏ,)\픒fƍc`!ߐxZp4"ď@@7rHz @ kC&x9z k뮹td@9 ѭt#6{172Ԉc\vKnfa;h:Ycoy->n*6C$bntK." ٲr-p]6l+r"j7Ξ5 }w6>2H1@c Îvb*/DQ~y"sdRbF c!^crDc|%[v6hs6\FqxBO`"G6p|taHAF`DX>m}N@FwYjEFnj3̰Cxy^3*9u.\z  Հ;'=#@\@A<'|3Uڎ׬UcI%GE.k%vѠ\slaQLR8sy쑼~2/Ǔ `2{ǒƕSr;@D=xxr&l ܫ鉃׮7|kl㵲O{|c~]kɼ\oC| ['208RzG?@_#q6Luƀ0٬:UpU?dҖmzS)?P@ҡH:mߦA+!L|o6vlcľbMZق-g.AU$cF)?(h~٬FwD{%>޿@-MM[d-[ܗҿRH@^wfuÇiۮλ[N, vʎ&#I+r%i3[TlV/D'/+,(>[T3)ex9C1uD8XLru@Rm~Kd~WhG |o֞NVx7D8r:if[qs*۟2u-]Ė)#H]{,Obh+#4FɦlvSQo{qn{?`/dQ~.d=ϣ6p ׸Fŧ?){饗m:,p13wͣSqsf ^@k;95 Qp08t;B 6*|Gd;a @gπe쥋8Ԇdx'*g>V,9DI H'&QVGFp8ʹsOdxosp[! ÿ~ 9H8cE<0Uy6CT/qNrVg8_@7OOK伻=G@($0g$%nr+|b܌~;m8{ӱc'I7 xM?aoɓTC&v7Y]\9Sc<^Ư_RŖB ħpaH Ynqt`L0р-J8a#%~ϝ;ߞ{} #{at0 r6|9NnHo( S5jLJf[&'w2?y7’Rba:5kHIC<dScڵ ek[$x) i L?Q_| '>DA)$#xA9{ɯ V@aFe^~-{R|VFefh(.1xY}1:[@W>Eϱ<9᥽3jH4"Fklj۶3ND|x̵ڻמE[=؏9ptՑs#NﴱEK6Yy,HgJJ9 !jywnjeY?Ok&eGeЀbHnw1ARi5Am!W_Y>ao|ѯj [lZlh veYsP <{md.2e^̨uL Qq(ihrE^qx@֕V@PˠRFyxD 4uVZP!s1pnf˃T c[O}Ȝ&,Ss@%.MK#qSy1ʅu$1_Gz۹iy6ۺu(O=nW_xZIѾ!%3F[y|J4C{<S.stEѻFc|Y5@2dr` rZ@1 .!ef4p npm+rY j0 *ďxfKt圗PEf7J^_q .j7ߨ#%0%< 3}5*vQ1yr Sgp9--$vkC[{Hq1)P9k?z(?+Ivp(Ja̓bl0B<$/Nה O2E`U*/Nw8ب:cBc|6~4R&-efcݘ%K_llOo{*ҡ22 9A /j-/ăO؋gb&hW\N*.\ ܵ&b{B/8R\o$bw1A|ʮw+Ta%&MJs簫ltnpz'rpjqhwu[Ul۶ժV{peevUL73+) mG{#ƍ(:!Gډǝ|L{Μ:m{d3列{3@/@S^ =JisJԮo#5 P PQ$AǺ_-]۟J|[`8!'! "1}uu 4Nvrp^A>p28|bgcw7mf;"U?-`+c Щ[(v]m=bFFˬvE {̿lH@ʶ4omP7w,ɒ. $6b2h*MT+D;?_րSHx:v.yH)M޲J6@b3 y 7j&۩"M_4j<{!WgoUhh̝cӴf-NS2:%l]=}ݧDr|l*>+ԻpB"klmq9. @8LXG=Qua?(q{oتկ"(Ba,rtGWu<+g x,GHcB`Dq"DW|re:$ mAR,k@+Qk=D^iJ%2%\s̮={T*ZGα&Mt8Y}qMFm[;V.=^A, qs.|;"zeqxh٠ /f7(_"\x-irN)NIU"cc X2'A3͛6i/]ʏN\\i0@α .q~ljҪߩw =d Jr]G1q멪/jZ'CHh/i#*sL 9tFaÈ|QF%o8OHb>tp5!]`U'i)漢~mngnjr.=/_ܫ}9ɾRˁDAW&iOr6 z\ Gx-7w5X|qk"@@C4~=CyH0 / b _aMuv.DQFh 6gZ(G5Gɦb{cqgsƞ={n}Uj1skayh3M-\폢.28fƕZs 4jy7?KjI}v7 +mn'e:AKfG?{INТ> cޏL pq)8ɋK $eF96I_wP IDATh^ŁSX^%d;g5o1c2E~VhzEߙo}bE >HJc{jK_bCJ?d_OWq32i欰7_QjS1+pjUti͛Ƙ467ʼP!q $,3ќV'qx䨑r:^9{{F[3)RЌ\dƴ:XF@}$bs;JL@4Wƞׯ}RH˧RNbYK$be^06 o93=1b1, 4$Wᙩ(#4-d>_:tH yh_A1fq9ZhU{ߐ΂LySylN8~l.ʞ{Vyf)$[L$j"qҡ1{ő3grweyQ+|>O9J R Nkp<_gNuM}蒥`6ǎ\),qsk?>(S<Ìb9O҉c+ 1rJ 5MBYAjW27# (33%Rc>4M8wJƜs u<&| $/wQ%Cqgi3Y>wsmXbB~0$wkn& Yq:GqgexҞ<06xܹoR&EMbQbz}*iN5O_R#՞@<5sր3* 3CؒQa?:]XO},1&&׾7yH.8:JcO I"9Mmix|kg] p3x o(_;曛K)۲ȉCD<".ƵMFvm۱~;S2"`mN wmUʡZ*y\.xɋ^~zGYacb񢅋˛f} +ɼ0R>I8ה4OW?Q#KM݉SxsP K9 ZO'O#ؤnLx~G)v-{n8hFUy͘Q/p!DpIz zg>cq _ 1纃y1M?g짩Qhq)ªl7TϓqӦMu|r'k~xP>?{T%'TH2*0Q jgX{|-7< uܧ}b0KJ&;jǽqҀeqP`hsmIy2 v*ppR6sg*TW(@rX~F[|P'rR9=i'=qrQ;믻^{ߵm4jM&Drb+Yg&G3gEt@E i۴F~nK+3|GtS}@S 6ͷ|@=uJysLkoymٶELsypq,G.(N"AF|8a/m4,=w{c\z(@2{!nNLmoys<*eX}@"wP0u0#P14*H6$Dl*khmop9"&0X:I3HJ\fgDQ~ǻd7<&,1Wlds}]yț%K]Ͻ}|dԤɄ2 E@R ףF2[([gVFk9ub"H7oC-1}C O-)k͛J)@"C:IU HGfi|.]"mp8̚8 ұEp "VԴ[GaSiH5";p 5pi-(B)HR$]A5dpS#iȹB Q sB38G\ ۵6;sCfy)S\b:GdyC SlRu:& @G"A\9SbLWaUAt\y5 nDU/MӠ{/X93Cc[z}Cr1NJ&BsS ĥ(@2^) %|w)Wcy>y8d?J @v-v뭷~oo+PN$0(PLN!Q`@@L\b5k۶v:mk= ގ*}i Ѱa@[n@7ǧD(@'ʤQ[{][jmڻ{0 q0A&IAα׬;~oށ?KҔ8@.=N:!!mF@vΏ?a^hň#Y]!ͺhgæFq")ۈR92_ $/Rnxν5U/h[lv#GQqΩSH-eoZ4:T'[a1Fx X?Z̆{׻YS]9bb$u:&  $K,"0ƛ\Qcy֭[lͶl>~EquIKӕ= ]w<r̘6AwTm+% K}tL/HR)y(" a) pNNƖn#2*IuuP.(]r}KTJw(Vtޒ j:FA>,;hZMVw*(Q`(~f3O@ pѪ;9 =j8QH:rVfm:Vfm[[}nQRIY3&y.9rY@%(&g*ATG7 N3g9NMcdZ#G߰V(Y;m RY;SH$HVک.@z#vwV@7;#CǣOS 䠓8UPJ n8-w1tErH޺W|;5/Y(Pi $4S}Rz /FT ̤"AM?A~ݤ5s/D R d +gd7]#Fs&6(:e_|ҹKW,_ϐ~*@ rrlQ%j/êXo-@ҥ ơ "tr vbPN@)r۩b4DH我Njk뜜.2׸b8ew@ 5Z}ⰵUGB@sJ$EHʏ?hE bpKlmTCW̫dCԖW \% Rsx+ospl5B;h7@5l$ssG}\e\-vTWuL-IvCkīXrd"ǁNs}O'q07$fm؁{EmDNm@p^S=yI(6.)N>N~F/ {{:GAe3j#N?C $̫؃~O\C% wV XJMwPZ۞3^+ip@P$]r'r+i{L$o'"1N'">p~7Pnf@7[}4+Vى/ yLpZA|pl}=! ]I"8;K]-}nRiˎ>|+>h1y+b9mD"Hi2 tb._;a]h(.QDG\ Z;loRHEΑsDit5@T-vTJPZ8a>1Z~A3tvA(2VܬR3>ݿvJ54$H@Ii )yͪUK}xFgm8-wS\\&!Wܱ^#lPzu1)=M3NPs88EŋdҽW"2Ɋ}7VX]5k6Vr"mrGٞ~َ>ݹiq5|T@P rm/8HՊy6Cѷ׸t%l >v8(tss3G_f .:fXaI$kշda>79pRvWP:]~{vٱd@xԍ >hu#^WH 9@OJ][;' .cN$3܎.:lCzDd h)S7pf;-GPPu/jT*p]n`OpS+eUcFٰsk?q:myy3Fk liI N#GCU=[g[Oɹ#ZN G# t:xW ÜgZ[s\/SNEsY:.\2~kV5Q7Tť0(CӣFjk:%"ژYˎm?:'8TMt,9^~`HHVuUpG6sæ_wa%+dW~FaMR6DuzhY$SumMlYʹF8RFhSHS qyjsj](:WPh۽3Ose KsKIG^8?6sq<*Z6JvxyTSc(>\\Z݃%P63h#W\琧#=RH(Ce2ԣgv85U_\NdS t6=o2+1GLqyqkwO٩WwɦTVB A@[*̅t,cոZ{CM`nE6zIePRkó()@2R"QyI:8,>k?!-IDAT%?έ?+y `S:N H(ImĜi6&.s]Mf?5V%PO8gGݰ:ϝ֣Z9J5WиvU6zNQ7K ąbPn;𜒸io 9 $# 08U9>pEN Dn2Jɝ-^$9|BlJ\T]^G^geУ ʶ,>˧j"p^u~ ioX gRJ3ysO!(DR leõu8A9ȁ^2ܖ$N&,VV8RH]Y4:uzb"<ˁG`!:k}G~^/7Y-6YȖN0=|!=zw $Ai3Bk G͖vO-J)N0E+[2XD.H 2p|W28N[,Q# RuV&U p9W8&E^O!Q,!H P! dӋiIA^'En1J"@q\|x98%̹Ev:'$B@@Ht,P0 J^Ep8lc7`>l X@ ١yL4.blLxzJ/[r9B"?!\0B@@z@8V@@@z@^n% $ $ $L} Q Q Q eIENDB`uqfoundation-klepto-69cd6ce/klepto/000077500000000000000000000000001455531556400175425ustar00rootroot00000000000000uqfoundation-klepto-69cd6ce/klepto/__init__.py000066400000000000000000000030301455531556400216470ustar00rootroot00000000000000#!/usr/bin/env python # # Author: Mike McKerns (mmckerns @caltech and @uqfoundation) # Copyright (c) 2013-2016 California Institute of Technology. # Copyright (c) 2016-2024 The Uncertainty Quantification Foundation. # License: 3-clause BSD. The full license text is available at: # - https://github.com/uqfoundation/klepto/blob/master/LICENSE # author, version, license, and long description try: # the package is installed from .__info__ import __version__, __author__, __doc__, __license__ except: # pragma: no cover import os import sys parent = os.path.dirname(os.path.abspath(os.path.dirname(__file__))) sys.path.append(parent) # get distribution meta info from version import (__version__, __author__, get_license_text, get_readme_as_rst) __license__ = get_license_text(os.path.join(parent, 'LICENSE')) __license__ = "\n%s" % __license__ __doc__ = get_readme_as_rst(os.path.join(parent, 'README.md')) del os, sys, parent, get_license_text, get_readme_as_rst from ._cache import no_cache, inf_cache, lfu_cache, \ lru_cache, mru_cache, rr_cache from ._inspect import signature, isvalid, validate, \ keygen, strip_markup, NULL, _keygen from . import rounding from . import safe from . import archives from . import keymaps from . import tools from . import crypto def license(): """print license""" print (__license__) return def citation(): """print citation""" print (__doc__[-273:-118]) return # end of file uqfoundation-klepto-69cd6ce/klepto/_abc.py000066400000000000000000000047061455531556400210070ustar00rootroot00000000000000#!/usr/bin/env python # # Author: Mike McKerns (mmckerns @caltech and @uqfoundation) # Copyright (c) 2021-2024 The Uncertainty Quantification Foundation. # License: 3-clause BSD. The full license text is available at: # - https://github.com/uqfoundation/klepto/blob/master/LICENSE """ base class for archive to memory, file, or database """ class archive(dict): """dictionary with an archive interface""" def __init__(self, *args, **kwds): """initialize an archive""" dict.__init__(self, *args, **kwds) self.__state__ = {'id': 'abc'} raise NotImplementedError("cannot instantiate archive base class") #return def __asdict__(self): """build a dictionary containing the archive contents""" return dict(self.items()) def __repr__(self): return "%s(%s, cached=False)" % (self.__class__.__name__, self.__asdict__()) __repr__.__doc__ = dict.__repr__.__doc__ def copy(self, name=None): #XXX: always None? or allow other settings? "D.copy(name) -> a copy of D, with a new archive at the given name" adict = self.__class__() adict.update(self.__asdict__()) adict.__state__ = self.__state__.copy() if name is not None: adict.__state__['id'] = name return adict # interface def load(self, *args): """does nothing. required to use an archive as a cache""" return dump = load def archived(self, *on): """check if the cache is a persistent archive""" L = len(on) if not L: return False if L > 1: raise TypeError("archived expected at most 1 argument, got %s" % str(L+1)) raise ValueError("cannot toggle archive") def sync(self, clear=False): "does nothing. required to use an archive as a cache" pass def drop(self): #XXX: or actually drop the backend? "set the current archive to NULL" return self.__archive(None) def open(self, archive): "replace the current archive with the archive provided" return self.__archive(archive) def __get_archive(self): return self def __get_name(self): return self.__state__['id'] def __get_state(self): return self.__state__.copy() def __archive(self, archive): raise ValueError("cannot set new archive") archive = property(__get_archive, __archive) name = property(__get_name, __archive) state = property(__get_state, __archive) pass uqfoundation-klepto-69cd6ce/klepto/_archives.py000066400000000000000000003103331455531556400220620ustar00rootroot00000000000000#!/usr/bin/env python # # Author: Mike McKerns (mmckerns @caltech and @uqfoundation) # Copyright (c) 2013-2016 California Institute of Technology. # Copyright (c) 2016-2024 The Uncertainty Quantification Foundation. # License: 3-clause BSD. The full license text is available at: # - https://github.com/uqfoundation/klepto/blob/master/LICENSE """ custom caching dict, which archives results to memory, file, or database """ import os import sys import shutil from random import random from pickle import PROTO, STOP from collections.abc import KeysView, ValuesView, ItemsView from importlib import util as imp if imp.find_spec('sqlalchemy'): sql = True def __import_sql__(): global sql import sqlalchemy as sql else: sql = None if imp.find_spec('h5py'): hdf = True def __import_hdf__(): global hdf, np import h5py as hdf else: hdf = None if imp.find_spec('pandas'): pandas = True def __import_pandas__(): global pandas import pandas else: pandas = None import json import dill from dill.source import getimportable from pox import mkdir, rmtree, walk from ._abc import archive from .crypto import hash from . import _pickle __all__ = ['cache','dict_archive','null_archive','dir_archive',\ 'file_archive','sql_archive','sqltable_archive',\ 'hdf_archive','hdfdir_archive'] PREFIX = "K_" # hash needs to be importable TEMP = ".I_" # indicates 'temporary' file #DEAD = "D_" # indicates 'deleted' key def _to_frame(archive):#, keymap=None): '''convert a klepto archive to a pandas DataFrame''' if not pandas: raise ValueError('install pandas for dataframe support') __import_pandas__() d = archive df = pandas.DataFrame() cached = d is not d.archive cache = '__archive__' name = d.archive.name name = '' if name is None else name df = df.from_dict({name: d if cached else d.__asdict__()}) if cached: if pandas.__version__ > '0.23': df = pandas.concat([df[name],df.from_dict({cache:d.archive.__asdict__()})[cache]],axis=1,sort=False) else: df = pandas.concat([df[name],df.from_dict({cache:d.archive.__asdict__()})[cache]],axis=1) #df.sort_index(axis=1, ascending=False, inplace=True) df.columns.name = d.archive.__class__.__name__#.rsplit('_archive')[0] df.index.name = repr(d.archive.state) ''' if len(df) and keymap is not None: #FIXME: need a generalized inverse of keymap #HACK: only works in certain special cases from klepto.keymaps import _stub_decoder #if isinstance(df.index[0], bytes): keymap = 'pickle' #elif isinstance(df.index[0], (str, (u'').__class__)): keymap = 'repr' #else: keymap = None inv = _stub_decoder(keymap) df.index = df.index.map(inv)#lambda k: list(inv(k))) #FIXME: correct? #HACK end ''' return df def _from_frame(dataframe):#, keymap=None): '''convert a (formatted) pandas dataframe to a klepto archive''' if not pandas: raise ValueError('install pandas for dataframe support') #if keymap is None: keymap = lambda x:x #XXX: or keymap()? df = dataframe #XXX: apply keymap here? d = df.to_dict() # is cached if has more than one column, one of which is named 'cache' cached = True if len(df.columns) > 1 else False cache = '__archive__' # index should be a dict; if not, set it to the empty dict try: index = eval(df.index.name) if type(index) is not dict: raise TypeError except: index = {} # should have at least one column; if so, get the name of the column name = df.columns[0] if len(df.columns) else None # get the name of the column -- this will be our cached data store = df.columns[1] if cached else cache # get the data from the first column #XXX: apply keymap here? data = {} if name is None else dict((k,v) for (k,v) in d[name].items() if repr(v) not in ['nan','NaN']) # get the archive type, defaulting to dict_archive col = df.columns.name try: col = col if col.endswith('_archive') else ''.join((col,'_archive')) except AttributeError: col = '' import klepto.archives as archives d_ = getattr(archives, col, archives.dict_archive) # get the archive instance d_ = d_(name, data, cached, **index) # if cached, add the cache data #XXX: apply keymap here? if cached: d_.archive.update((k,v) for (k,v) in d.get(store,{}).items() if repr(v) not in ['nan','NaN']) return d_ class cache(dict): """dictionary augmented with an archive backend""" def __init__(self, *args, **kwds): """initialize a dictionary with an archive backend Args: archive (archive, default=null_archive()): instance of archive object """ self.__swap__ = null_archive() self.__archive__ = kwds.pop('archive', null_archive()) dict.__init__(self, *args, **kwds) #self.__state__ = {} return def __repr__(self): archive = self.archive.__class__.__name__ name = self.archive.name if name: return "%s(%r, %s, cached=True)" % (archive, str(name), dict(self)) return "%s(%s, cached=True)" % (archive, dict(self)) __repr__.__doc__ = dict.__repr__.__doc__ def to_frame(self): return _to_frame(self) to_frame.__doc__ = _to_frame.__doc__ def popkeys(self, keys, *value): """ D.popkeys(k[,d]) -> v, remove specified keys and return corresponding values. If key in keys is not found, d is returned if given, otherwise KeyError is raised.""" if not hasattr(keys, '__iter__'): return self.pop(keys, *value) if len(value): return [self.pop(k, *value) for k in keys] memo = self.fromkeys(self.keys()) [memo.pop(k) for k in keys] return [self.pop(k) for k in keys] def load(self, *args): #FIXME: archive may use key 'encoding' (dir_archive) """load archive contents If arguments are given, only load the specified keys """ if not args: self.update(self.archive.__asdict__()) for arg in args: try: self.update({arg:self.archive[arg]}) except KeyError: pass return def dump(self, *args): #FIXME: archive may use key 'encoding' (dir_archive) """dump contents to archive If arguments are given, only dump the specified keys """ if not args: self.archive.update(self) for arg in args: if arg in self: self.archive.update({arg:self.__getitem__(arg)}) return def archived(self, *on): """check if the cache is archived, or toggle archiving If on is True, turn on the archive; if on is False, turn off the archive """ L = len(on) if not L: return not isinstance(self.archive, null_archive) if L > 1: raise TypeError("archived expected at most 1 argument, got %s" % str(L+1)) if bool(on[0]): if not isinstance(self.__swap__, null_archive): self.__swap__, self.archive = self.archive, self.__swap__ elif isinstance(self.archive, null_archive): raise ValueError("no valid archive has been set") else: if not isinstance(self.archive, null_archive): self.__swap__, self.archive = self.archive, self.__swap__ def sync(self, clear=False): """synchronize cache and archive contents If clear is True, clear all archive contents before synchronizing cache """ if clear: self.archive.clear() self.dump() if not clear: self.load() return def drop(self): #XXX: sync first? "set the current archive to NULL" self.archived(True) #XXX: should not throw error if not archived? self.archive = null_archive() return def open(self, archive): "replace the current archive with the archive provided" try: self.archived(True) except ValueError: pass self.archive = archive return def __get_archive(self): #if not isinstance(self.__archive__, null_archive): # return if not isinstance(self.__archive__, archive): self.__archive__ = null_archive() return self.__archive__ def __get_class(self): import klepto.archives as archives return getattr(archives, self.archive.__class__.__name__) def __archive(self, archive): if not isinstance(self.__swap__, null_archive): self.__swap__, self.__archive__ = self.archive, self.__swap__ self.__archive__ = archive # interface archive = property(__get_archive, __archive) __type__ = property(__get_class, __archive) pass class dict_archive(archive): """dictionary with an archive interface""" def __init__(self, *args, **kwds): """initialize a dictionary archive""" name = kwds.pop('__magic_key_0192837465__', None) dict.__init__(self, *args, **kwds) self.__state__ = { 'id': name # can be used to store a 'name' } return def __asdict__(self): """build a dictionary containing the archive contents""" return dict(self.items()) def __repr__(self): return "dict_archive(%s, cached=False)" % (self.__asdict__()) __repr__.__doc__ = dict.__repr__.__doc__ def copy(self, name=None): #XXX: always None? or allow other settings? "D.copy(name) -> a copy of D, with a new archive at the given name" if name is None: name = self.__state__['id'] adict = dict_archive(__magic_key_0192837465__=name) adict.update(self.__asdict__()) return adict def __drop__(self): """drop the associated database EXPERIMENTAL: This method is intended to remove artifacts external to the instance. For a dict_archive there are none, so just call clear. """ self.clear() # interface pass class null_archive(archive): """dictionary interface to nothing -- it's always empty""" def __init__(self, *args, **kwds): """initialize a permanently-empty dictionary""" name = kwds.pop('__magic_key_0192837465__', None) dict.__init__(self) self.__state__ = { 'id': name # can be used to store a 'name' } return def __asdict__(self): """build a dictionary containing the archive contents""" return dict() def __setitem__(self, key, value): pass __setitem__.__doc__ = dict.__setitem__.__doc__ def update(self, adict, **kwds): pass update.__doc__ = dict.update.__doc__ def setdefault(self, key, *value): return self.get(key, *value) setdefault.__doc__ = dict.setdefault.__doc__ def __repr__(self): return "null_archive(cached=False)" __repr__.__doc__ = dict.__repr__.__doc__ def copy(self, name=None): #XXX: always None? or allow other settings? "D.copy(name) -> a copy of D, with a new archive at the given name" if name is None: name = self.__state__['id'] return null_archive(__magic_key_0192837465__=name) def __drop__(self): """drop the associated database EXPERIMENTAL: This method is intended to remove artifacts external to the instance. For a null_archive there are none, so just call clear. """ self.clear() # interface pass class dir_archive(archive): """dictionary-style interface to a folder of files""" def __init__(self, dirname=None, serialized=True, compression=0, permissions=None, **kwds): """initialize a file folder with a synchronized dictionary interface Args: dirname (str, default='memo'): path of the archive root directory serialized (bool, default=True): save python objects in pickled files compression (int, default=0): compression level (0 to 9), 0 is None permissions (octal, default=0o775): read/write permission indicator memmode (str, default=None): mode, one of ``{None, 'r+', 'r', 'w+', 'c'}`` memsize (int, default=100): size (MB) of cache for in-memory compression protocol (int, default=DEFAULT_PROTOCOL): pickling protocol """ #XXX: if compression or mode is given, use joblib-style pickling # (ignoring 'serialized'); else if serialized, use dill unless # fast=True (then use joblib-style pickling), or protocol='json' # (then use json-style pickling). If not serialized, then write # raw objects and load objects with import. Also, if fast=True # and protocol='json', the use protocol=None. #FIXME: needs doc if dirname is None: #FIXME: default root as /tmp or something better dirname = 'memo' #FIXME: need better default # set state self.__state__ = { # undocumented: True=joblib-style, False=dill-style pickling 'fast': kwds.get('fast', False), # settings 'serialized': serialized, 'compression': compression, 'permissions': permissions, 'memmode': kwds.get('memmode', None), 'memsize': kwds.get('memsize', 100), # unused? 'protocol': kwds.get('protocol', None), 'id': dirname } #XXX: add 'cloud' option? # if not serialized, then set fast=False if not serialized: self.__state__['compression'] = 0 self.__state__['memmode'] = None self.__state__['fast'] = False # if compression or mode, then set fast=True elif compression or self.__state__['memmode']: self.__state__['fast'] = True # ELSE: use dill if fast=False, else use _pickle try: self.__state__['id'] = mkdir(dirname, mode=self.__state__['permissions']) except OSError: # then directory already exists self.__state__['id'] = os.path.abspath(dirname) return def __reduce__(self): dirname = self.name serial = self.__state__['serialized'] compress = self.__state__['compression'] perm = self.__state__['permissions'] state = {'__state__': self.__state__} return (self.__class__, (dirname, serial, compress, perm), state) def __asdict__(self): """build a dictionary containing the archive contents""" # get the names of all directories in the directory keys = self._keydict() # get the values return dict((key,self.__getitem__(key)) for key in keys) #FIXME: missing __cmp__, __...__ def __eq__(self, y): try: if y.__module__ != self.__module__: return NotImplemented return self.__asdict__() == y.__asdict__() #XXX: faster than get? #if len(self) != len(y): return False #try: s = min(k for k in self if self.get(k) != y.get(k)) #except ValueError: s = [] #try: v = min(k for k in y if y.get(k) != self.get(k)) #except ValueError: v = [] #if s != v: return False #elif s == []: return True #return self[s] == y[v] except: return NotImplemented __eq__.__doc__ = dict.__eq__.__doc__ def __ne__(self, y): y = self.__eq__(y) return NotImplemented if y is NotImplemented else not y __ne__.__doc__ = dict.__ne__.__doc__ def __delitem__(self, key): try: memo = {key: None} self._rmdir(key) except: memo = {} memo.__delitem__(key) return __delitem__.__doc__ = dict.__delitem__.__doc__ def __getitem__(self, key): return self._lookup(key) __getitem__.__doc__ = dict.__getitem__.__doc__ def __repr__(self): return "dir_archive('%s', %s, cached=False)" % (self.name, self.__asdict__()) __repr__.__doc__ = dict.__repr__.__doc__ def __setitem__(self, key, value): self._store(key, value, input=False) # input=True also stores input return __setitem__.__doc__ = dict.__setitem__.__doc__ def clear(self): rmtree(self.__state__['id'], self=False, ignore_errors=True) return clear.__doc__ = dict.clear.__doc__ def copy(self, name=None): #XXX: always None? or allow other settings? "D.copy(name) -> a copy of D, with a new archive at the given name" if name is None: name = self.__state__['id'] else: #XXX: overwrite? shutil.copytree(self.__state__['id'], os.path.abspath(name)) adict = dir_archive(dirname=name, **self.state) #adict.update(self.__asdict__()) return adict def __drop__(self): """drop the associated database EXPERIMENTAL: Deleting the directory may not work due to permission issues. Caller may need to be connected as a superuser and database owner. """ self.clear() import os import shutil if os.path.exists(self.__state__['id']): shutil.rmtree(self.__state__['id']) return def fromkeys(self, *args): #XXX: build a dict (not an archive)? return dict.fromkeys(*args) fromkeys.__doc__ = dict.fromkeys.__doc__ def get(self, key, value=None): try: return self.__getitem__(key) except: return value get.__doc__ = dict.get.__doc__ def __contains__(self, key): _dir = self._getdir(key) return os.path.exists(_dir) __contains__.__doc__ = dict.__contains__.__doc__ def __iter__(self): return iter(self._keydict().keys()) __iter__.__doc__ = dict.__iter__.__doc__ def keys(self): return KeysView(self) #XXX: show keys not dict keys.__doc__ = dict.keys.__doc__ def items(self): return ItemsView(self) #XXX: show items not dict items.__doc__ = dict.items.__doc__ def values(self): return ValuesView(self) #XXX: show values not dict values.__doc__ = dict.values.__doc__ def popkeys(self, keys, *value): """ D.popkeys(k[,d]) -> v, remove specified keys and return corresponding values. If key in keys is not found, d is returned if given, otherwise KeyError is raised.""" if not hasattr(keys, '__iter__'): return self.pop(keys, *value) if len(value): return [self.pop(k, *value) for k in keys] memo = self._keydict() # 'shadow' dict for desired error behavior [memo.pop(k) for k in keys] return [self.pop(k) for k in keys] def pop(self, key, *value): #XXX: or make DEAD ? try: memo = {key: self.__getitem__(key)} self._rmdir(key) except: memo = {} res = memo.pop(key, *value) return res pop.__doc__ = dict.pop.__doc__ def popitem(self): key = self.__iter__() try: key = next(key) except StopIteration: raise KeyError("popitem(): dictionary is empty") return (key, self.pop(key)) popitem.__doc__ = dict.popitem.__doc__ def setdefault(self, key, *value): res = self.get(key, *value) self.__setitem__(key, res) return res setdefault.__doc__ = dict.setdefault.__doc__ def update(self, adict, **kwds): if hasattr(adict,'__asdict__'): adict = adict.__asdict__() memo = {} memo.update(adict, **kwds) #XXX: could be better ? for (key,val) in memo.items(): self.__setitem__(key,val) return update.__doc__ = dict.update.__doc__ def __len__(self): return len(self._lsdir()) def _fname(self, key): "generate suitable filename for a given key" # special handling for pickles; enable non-strings (however 1=='1') try: ispickle = key.startswith(PROTO) and key.endswith(STOP) except: ispickle = False #FIXME: protocol 0,1 don't startwith(PROTO) key = hash(key, 'md5') if ispickle else str(key) #XXX: always hash? return key.replace('-','_') ##XXX: below probably fails on windows, and could be huge... use 'md5' #return repr(key)[1:-1] if ispickle else str(key) # or repr? def _mkdir(self, key): "create results subdirectory corresponding to given key" key = self._fname(key) try: return mkdir(PREFIX+key, root=self.__state__['id'], mode=self.__state__['permissions']) except OSError: # then directory already exists return self._getdir(key) def _getdir(self, key): "get results directory name corresponding to given key" key = self._fname(key) return os.path.join(self.__state__['id'], PREFIX+key) def _rmdir(self, key): "remove results subdirectory corresponding to given key" rmtree(self._getdir(key), self=True, ignore_errors=True) return def _lsdir(self): "get a list of subdirectories in the root directory" return walk(self.__state__['id'],patterns=PREFIX+'*',recurse=False,folders=True,files=False,links=False) def _hasinput(self, root): "check if results subdirectory has stored input file" return bool(walk(root,patterns=self._args,recurse=False,folders=False,files=True,links=False)) def _getkey(self, root): "get key given a results subdirectory name" key = os.path.basename(root)[2:] return self._lookup(key,input=True) if self._hasinput(root) else key def _keydict(self): "get a dict of subdirectories in the root directory, with dummy values" keys = self._lsdir() return dict((self._getkey(key),None) for key in keys) #FIXME: dict((i,self._getkey(key)) for i,key in enumerate(keys)) def _reverse_lookup(self, args): #XXX: guaranteed 1-to-1 mapping? "get subdirectory name from args" d = {} for key in iter(self._keydict()): try: if args == self._lookup(key, input=True): d[args] = None #XXX: unnecessarily memory intensive? break except KeyError: continue # throw KeyError(args) if key not found del d[args] return key def _lookup(self, key, input=False): "get input or output from subdirectory name" _dir = self._getdir(key) if self.__state__['serialized']: _file = self._args if input else self._file _file = os.path.join(_dir, _file) try: if self.__state__['fast']: #XXX: enable override of 'mode' ? memo = _pickle.load(_file, mmap_mode=self.__state__['memmode']) else: protocol = self.__state__['protocol'] if type(protocol) is str: #XXX: assumes json pik,mode = json,'r' else: pik,mode = dill,'rb' with open(_file, mode) as f: memo = pik.load(f) except: #XXX: should only catch the appropriate exceptions memo = None raise KeyError(key) #raise OSError("error reading directory for '%s'" % key) else: import tempfile base = os.path.basename(_dir) #XXX: PREFIX+key root = os.path.realpath(self.__state__['id']) name = tempfile.mktemp(prefix="_____", dir="").replace("-","_") _arg = ".__args__" if input else "" string = "from %s%s import memo as %s; sys.modules.pop('%s')" % (base, _arg, name, base) try: sys.path.insert(0, root) exec(string, globals()) #FIXME: unsafe, potential name conflict memo = globals().get(name)# None) #XXX: error if not found? globals().pop(name, None) except: #XXX: should only catch the appropriate exceptions raise KeyError(key) #raise OSError("error reading directory for '%s'" % key) finally: sys.path.remove(root) return memo def _store(self, key, value, input=False): "store output (and possibly input) in a subdirectory" _key = TEMP+hash(random(), 'md5') # create an input file when key is not suitable directory name if self._fname(key) != key: input=True #XXX: errors if protocol=0,1? # create a temporary directory, and dump the results try: _file = os.path.join(self._mkdir(_key), self._file) if input: _args = os.path.join(self._getdir(_key), self._args) if self.__state__['serialized']: protocol = self.__state__['protocol'] if self.__state__['fast']: protocol = None if type(protocol) is str else protocol compression = self.__state__['compression'] _pickle.dump(value, _file, compress=compression, protocol=protocol) if input: _pickle.dump(key, _args, compress=compression, protocol=protocol) else: if type(protocol) is str: #XXX: assumes json pik,mode,kwd = json,'w',{} else: #XXX: byref? pik,mode,kwd = dill,'wb',{'protocol':protocol} with open(_file, mode) as f: pik.dump(value, f, **kwd) if input: with open(_args, mode) as f: pik.dump(key, f, **kwd) else: # try to get an import for the object try: memo = getimportable(value, alias='memo', byname=False) except AttributeError: #XXX: HACKY... get classes by name memo = getimportable(value, alias='memo') #XXX: class instances and such fail... abuse pickle here? from .tools import _b with open(_file, 'wb') as f: f.write(_b(memo)) if input: try: memo = getimportable(key, alias='memo', byname=False) except AttributeError: memo = getimportable(key, alias='memo') from .tools import _b with open(_args, 'wb') as f: f.write(_b(memo)) except OSError: "failed to populate directory for '%s'" % str(key) # move the results to the proper place try: #XXX: possible permissions issues here self._rmdir(key) #XXX: 'key' must be a suitable dir name os.renames(self._getdir(_key), self._getdir(key)) # except TypeError: #XXX: catch key that isn't converted to safe filename # "error in populating directory for '%s'" % str(key) except OSError: #XXX: if rename fails, may need cleanup (_rmdir ?) "error in populating directory for '%s'" % str(key) def _get_args(self): if self.__state__['serialized']: if type(self.__state__['protocol']) is str \ and not self.__state__['fast']: return 'input.json' else: return 'input.pkl' return '__args__.py' def _get_file(self): if self.__state__['serialized']: if type(self.__state__['protocol']) is str \ and not self.__state__['fast']: return 'output.json' else: return 'output.pkl' return '__init__.py' def _set_file(self, file): raise NotImplementedError("cannot set attribute '_file'") # interface def __get_name(self): return os.path.basename(self.__state__['id']) def __archive(self, archive): raise ValueError("cannot set new archive") name = property(__get_name, __archive) _file = property(_get_file, _set_file) _args = property(_get_args, _set_file) pass class file_archive(archive): """dictionary-style interface to a file""" def __init__(self, filename=None, serialized=True, **kwds): # False """initialize a file with a synchronized dictionary interface Args: filename (str, default='memo.pkl'): path of the file archive serialized (bool, default=True): save python objects in pickled file protocol (int, default=DEFAULT_PROTOCOL): pickling protocol """ #FIXME: (needs doc) if protocol='json', use the json serializer protocol = kwds.get('protocol', None) if filename is None: #XXX: need better default filename? if serialized: filename = 'memo.json' if type(protocol) is str else 'memo.pkl' else: filename = 'memo.py' elif not serialized and not filename.endswith(('.py','.pyc','.pyo','.pyd')): filename = filename+'.py' # set state self.__state__ = { 'id': filename, 'serialized': serialized, 'protocol': protocol } #XXX: add 'cloud' option? if not os.path.exists(filename): self.__save__({}) return def __reduce__(self): fname = self.__state__['id'] serial = self.__state__['serialized'] state = {'__state__': self.__state__} return (self.__class__, (fname, serial), state) def __asdict__(self): """build a dictionary containing the archive contents""" filename = self.__state__['id'] if self.__state__['serialized']: protocol = self.__state__['protocol'] if type(protocol) is str: pik,mode = json,'r' else: pik,mode = dill,'rb' try: with open(filename, mode) as f: memo = pik.load(f) except: memo = {} #raise OSError("error reading file archive %s" % filename) else: import tempfile file = os.path.basename(filename) root = os.path.realpath(filename).rstrip(file)[:-1] curdir = os.path.realpath(os.curdir) if file.endswith(('.py','.pyc','.pyo','.pyd')): file = file.rsplit('.',1)[0] name = tempfile.mktemp(prefix="_____", dir="").replace("-","_") os.chdir(root) string = "from %s import memo as %s; sys.modules.pop('%s')" % (file, name, file) try: exec(string, globals()) #FIXME: unsafe, potential name conflict memo = globals().get(name, {}) #XXX: error if not found ? globals().pop(name, None) except: #XXX: should only catch appropriate exceptions memo = {} #raise OSError("error reading file archive %s" % filename) finally: os.chdir(curdir) return memo def __save__(self, memo=None): """create an archive from the given dictionary""" if memo == None: return filename = self.__state__['id'] _filename = os.path.join(os.path.dirname(os.path.abspath(filename)), TEMP+hash(random(), 'md5')) # create a temporary file, and dump the results try: if self.__state__['serialized']: protocol = self.__state__['protocol'] if type(protocol) is str: #XXX: assumes 'json' pik,mode,kwd = json,'w',{} else: #XXX: byref=True ? pik,mode,kwd = dill,'wb',{'protocol':protocol} with open(_filename, mode) as f: pik.dump(memo, f, **kwd) else: #XXX: likely_import for each item in dict... ? from .tools import _b open(_filename, 'wb').write(_b('memo = %s' % repr(memo))) except OSError: "failed to populate file for %s" % str(filename) # move the results to the proper place try: os.remove(filename) except: pass try: os.renames(_filename, filename) except OSError: "error in populating %s" % str(filename) return #FIXME: missing __cmp__, __...__ def __eq__(self, y): try: if y.__module__ != self.__module__: return NotImplemented return self.__asdict__() == y.__asdict__() #XXX: faster than get? except: return NotImplemented __eq__.__doc__ = dict.__eq__.__doc__ def __ne__(self, y): y = self.__eq__(y) return NotImplemented if y is NotImplemented else not y __ne__.__doc__ = dict.__ne__.__doc__ def __delitem__(self, key): memo = self.__asdict__() memo.__delitem__(key) self.__save__(memo) return __delitem__.__doc__ = dict.__delitem__.__doc__ def __getitem__(self, key): memo = self.__asdict__() return memo[key] __getitem__.__doc__ = dict.__getitem__.__doc__ def __repr__(self): return "file_archive('%s', %s, cached=False)" % (self.name, self.__asdict__()) __repr__.__doc__ = dict.__repr__.__doc__ def __setitem__(self, key, value): memo = self.__asdict__() memo[key] = value self.__save__(memo) return __setitem__.__doc__ = dict.__setitem__.__doc__ def clear(self): self.__save__({}) return clear.__doc__ = dict.clear.__doc__ def copy(self, name=None): #XXX: always None? or allow other settings? "D.copy(name) -> a copy of D, with a new archive at the given name" filename = self.__state__['id'] if name is None: name = filename else: shutil.copy2(filename, name) #XXX: overwrite? adict = file_archive(filename=name, **self.state) #adict.update(self.__asdict__()) return adict def __drop__(self): """drop the associated database EXPERIMENTAL: Deleting the file may not work due to permission issues. Caller may need to be connected as a superuser and database owner. """ self.clear() import os if os.path.exists(self.__state__['id']): os.remove(self.__state__['id']) return def fromkeys(self, *args): #XXX: build a dict (not an archive)? return dict.fromkeys(*args) fromkeys.__doc__ = dict.fromkeys.__doc__ def get(self, key, value=None): memo = self.__asdict__() return memo.get(key, value) get.__doc__ = dict.get.__doc__ def __contains__(self, key): return key in self.__asdict__() __contains__.__doc__ = dict.__contains__.__doc__ def __iter__(self): return iter(self.__asdict__().keys()) __iter__.__doc__ = dict.__iter__.__doc__ def keys(self): return KeysView(self) #XXX: show keys not dict keys.__doc__ = dict.keys.__doc__ def items(self): return ItemsView(self) #XXX: show items not dict items.__doc__ = dict.items.__doc__ def values(self): return ValuesView(self) #XXX: show values not dict values.__doc__ = dict.values.__doc__ def popkeys(self, keys, *value): """ D.popkeys(k[,d]) -> v, remove specified keys and return corresponding values. If key in keys is not found, d is returned if given, otherwise KeyError is raised.""" if not hasattr(keys, '__iter__'): return self.pop(keys, *value) memo = self.__asdict__() res = [memo.pop(k, *value) for k in keys] self.__save__(memo) return res def pop(self, key, *value): memo = self.__asdict__() res = memo.pop(key, *value) self.__save__(memo) return res pop.__doc__ = dict.pop.__doc__ def popitem(self): memo = self.__asdict__() res = memo.popitem() self.__save__(memo) return res popitem.__doc__ = dict.popitem.__doc__ def setdefault(self, key, *value): res = self.__asdict__().get(key, *value) self.__setitem__(key, res) return res setdefault.__doc__ = dict.setdefault.__doc__ def update(self, adict, **kwds): if hasattr(adict,'__asdict__'): adict = adict.__asdict__() memo = self.__asdict__() memo.update(adict, **kwds) self.__save__(memo) return update.__doc__ = dict.update.__doc__ def __len__(self): return len(self.__asdict__()) # interface def __get_name(self): return os.path.basename(self.__state__['id']) def __archive(self, archive): raise ValueError("cannot set new archive") name = property(__get_name, __archive) pass def _sqlname(name): """parse database name and table name from given name string name: a string of the form 'databaseurl?table=tablename' """ key = '?table=' if name is None: db, table = None, None # name=None elif name.startswith((key,'table=')): # name='table=memo' db, table = None, name.lstrip('?').lstrip('table').lstrip('=') elif name.count('/'): # name='sqlite:///' db, table = name.split(key,1) if name.count(key) else (name, None) else: db, table = None, name # name='memo' return (db, table) if sql: #FIXME: serialized throws RecursionError... but r'\x80' is valid (so is '80') # however, '\x80' and u'\x80' and b'\x80' are not valid (also not 80) # NOTE: if sql == False: 80, u'\x80', and b'\\x80' are also VALID class sql_archive(archive): """dictionary-style interface to a sql database""" def __init__(self, database=None, **kwds): """initialize a sql database with a synchronized dictionary interface Connect to an existing database, or initialize a new database, at the selected database url. For example, to use a sqlite database 'foo.db' in the current directory, database='sqlite:///foo.db'. To use a mysql database 'foo' on localhost, database='mysql://user:pass@localhost/foo'. For postgresql, use database='postgresql://user:pass@localhost/foo'. When connecting to sqlite, the default database is ':memory:'; otherwise, the default database is 'defaultdb'. Allows keyword options for database configuration, such as connection pooling. Args: database (str, default=None): database url (see above note) serialized (bool, default=True): save objects as pickled strings protocol (int, default=DEFAULT_PROTOCOL): pickling protocol """ #FIXME: (needs doc) if protocol='json', use the json serializer __import_sql__() # create database, if doesn't exist if database is None: database = 'sqlite:///:memory:' elif database == 'sqlite:///': database = 'sqlite:///:memory:' _database = database try: url, dbname = database.rsplit('/', 1) except ValueError: # only dbname given url, dbname = 'sqlite://', database _database = "%s/%s" % (url,dbname) if url.endswith(":/") or dbname == '': # then no dbname was given url = _database dbname = 'defaultdb' _database = "%s/%s" % (url,dbname) # set state kwds.pop('id',None) self.__state__ = { #XXX: add 'cloud' option? 'serialized': bool(kwds.pop('serialized', True)), 'id': _database, 'protocol': kwds.pop('protocol', dill.DEFAULT_PROTOCOL), # preserve other settings (for copy) 'config': kwds.pop('config', kwds.copy()) } #XXX: _engine and _metadata (and _key and _val) also __state__ ? # get engine kwds['future'] = True # 1.4 & 2.0 if dbname == ':memory:': self._engine = sql.create_engine(url, **kwds) elif _database.startswith('sqlite'): self._engine = sql.create_engine(_database, **kwds) else: self._engine = sql.create_engine(url, future=True) #XXX: **kwds ? try: self._conn = self._engine.connect() if _database.startswith('postgres'): self._conn.connection.connection.set_isolation_level(0) self._conn.execute(sql.text("CREATE DATABASE %s;" % dbname)) self._conn.commit() except Exception: self._conn = None finally: if _database.startswith('postgres'): self._conn.connection.connection.set_isolation_level(1) try: if self._conn is None: self._conn = self._engine.connect() self._conn.execute(sql.text("USE %s;" % dbname)) self._conn.commit() except Exception: pass self._engine = sql.create_engine(_database, **kwds) self._conn = self._engine.connect() # table internals self._metadata = sql.MetaData() self._key = 'Kkeyqwg907' # primary key name self._val = 'Kvalmol142' # object storage name # discover all tables #FIXME: with matching self._key keys = self._keys() [self._mktable(key) for key in keys] #self._metadata.create_all(self._engine) return def __drop__(self, **kwds): """drop the associated database EXPERIMENTAL: For certain database engines, this may not work due to permission issues. Caller may need to be connected as a superuser and database owner. """ _database = self.__state__['id'] url, dbname = _database.rsplit('/', 1) self._engine = sql.create_engine(url, future=True) # 1.4 & 2.0 try: self._conn = self._engine.connect() if _database.startswith('postgres'): # these two commands require superuser privs self._conn.execute(sql.text("update pg_database set datallowconn = 'false' WHERE datname = '%s';" % dbname)) self._conn.commit() self._conn.execute(sql.text("SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '%s';" % dbname)) # 'pid' used in postgresql >= 9.2 self._conn.commit() self._conn.connection.connection.set_isolation_level(0) self._conn.execute(sql.text("DROP DATABASE %s;" % dbname)) # must be db owner self._conn.commit() if _database.startswith('postgres'): self._conn.connection.connection.set_isolation_level(1) except Exception: dbpath = _database.split('///')[-1] if os.path.exists(dbpath): # else fail silently os.remove(dbpath) self._metadata = self._engine = self._conn = None # self.__state__['table']=None return def __asdict__(self): """build a dictionary containing the archive contents""" keys = self._keys() return dict((key,self.__getitem__(key)) for key in keys) #FIXME: missing __cmp__, __...__ def __eq__(self, y): try: if y.__module__ != self.__module__: return NotImplemented return self.__asdict__() == y.__asdict__() #XXX: faster than get? except: return NotImplemented __eq__.__doc__ = dict.__eq__.__doc__ def __ne__(self, y): y = self.__eq__(y) return NotImplemented if y is NotImplemented else not y __ne__.__doc__ = dict.__ne__.__doc__ def __delitem__(self, key): table = self._gettable(key) self._metadata.remove(table) table.drop(self._engine) #XXX: optionally delete data ? return __delitem__.__doc__ = dict.__delitem__.__doc__ def __getitem__(self, key): #XXX: value is table['key','key']; slow? table = self._gettable(key) query = sql.select(table).where(table.c[self._key] == self._key)#XXX: slow? row = self._conn.execute(query).fetchone() if row is None: raise RuntimeError("primary key for '%s' not found" % key) return row._mapping[self._val] __getitem__.__doc__ = dict.__getitem__.__doc__ def __repr__(self): return "sql_archive('%s', %s, cached=False)" % (self.name, self.__asdict__()) __repr__.__doc__ = dict.__repr__.__doc__ def __setitem__(self, key, value): #XXX: _setkey is part of _mktable value = {self._val: value} try: table = self._gettable(key) # KeyError if table doesn't exist query = table.update().where(table.c[self._key] == self._key) values = value except KeyError: table = self._mktable(key) query = table.insert() values = {self._key: self._key} values.update(value) self._conn.execute(query.values(**values)) self._conn.commit() return __setitem__.__doc__ = dict.__setitem__.__doc__ def clear(self): #self._metadata.drop_all() for key in self._keys(): try: self.__delitem__(key) #XXX: optionally delete data ? except: pass #XXX: don't catch ? return clear.__doc__ = dict.clear.__doc__ def copy(self, name=None): #XXX: always None? or allow other settings? "D.copy(name) -> a copy of D, with a new archive at the given name" if name is None: name = self.name else: pass #FIXME: copy database/table instead of do update below #FIXME: should reference, not copy adict = sql_archive(database=name, **self.state) adict.update(self.__asdict__()) return adict def fromkeys(self, *args): #XXX: build a dict (not an archive)? return dict.fromkeys(*args) fromkeys.__doc__ = dict.fromkeys.__doc__ def get(self, key, value=None): try: _value = self.__getitem__(key) except KeyError: _value = value return _value get.__doc__ = dict.get.__doc__ def __contains__(self, key): return key in self._keys() __contains__.__doc__ = dict.__contains__.__doc__ def __iter__(self): return iter(self._keys()) __iter__.__doc__ = dict.__iter__.__doc__ def keys(self): return KeysView(self) #XXX: show keys not dict keys.__doc__ = dict.keys.__doc__ def items(self): return ItemsView(self) #XXX: show items not dict items.__doc__ = dict.items.__doc__ def values(self): return ValuesView(self) #XXX: show values not dict values.__doc__ = dict.values.__doc__ def popkeys(self, keys, *value): """ D.popkeys(k[,d]) -> v, remove specified keys and return corresponding values. If key in keys is not found, d is returned if given, otherwise KeyError is raised.""" if not hasattr(keys, '__iter__'): return self.pop(keys, *value) if len(value): return [self.pop(k, *value) for k in keys] memo = self.fromkeys(self._keys()) # 'shadow' dict [memo.pop(k) for k in keys] return [self.pop(k) for k in keys] def pop(self, key, *value): try: memo = {key: self.__getitem__(key)} self.__delitem__(key) except: memo = {} res = memo.pop(key, *value) return res pop.__doc__ = dict.pop.__doc__ def popitem(self): key = self.__iter__() try: key = next(key) except StopIteration: raise KeyError("popitem(): dictionary is empty") return (key, self.pop(key)) popitem.__doc__ = dict.popitem.__doc__ def setdefault(self, key, *value): res = self.get(key, *value) self.__setitem__(key, res) return res setdefault.__doc__ = dict.setdefault.__doc__ def update(self, adict, **kwds): if hasattr(adict,'__asdict__'): adict = adict.__asdict__() memo = {} memo.update(adict, **kwds) #XXX: could be better ? for (key,val) in memo.items(): self.__setitem__(key,val) return update.__doc__ = dict.update.__doc__ def __len__(self): return len(self._keys()) def _mktable(self, key): "create table corresponding to given key" try: return self._gettable(key, meta=True) # table exists except KeyError: table = key # table doesn't exist in metadata # prepare table types #XXX: do in __init__ ? keytype = sql.String(255) if self.__state__['serialized']: proto = self.__state__['protocol'] if type(proto) is str: #XXX: assumes 'json' valtype = sql.PickleType(pickler=json) else: valtype = sql.PickleType(protocol=proto, pickler=dill) else: valtype = sql.Text() # create table, if doesn't exist table = sql.Table(table, self._metadata, sql.Column(self._key, keytype, primary_key=True), sql.Column(self._val, valtype) ) # initialize self._metadata.create_all(self._engine) return table def _gettable(self, key, meta=False): "get table corresponding to given key" table = str(key) if meta: return self._metadata.tables[table] # otherwise, look at all the tables in the database if table in self._keys(): return self._mktable(table) # if you are here... raise a KeyError tables = {} return tables[table] def _keys(self, meta=False): "get a list of tables in the database" #FIXME: with matching self._key if meta: return self._metadata.tables.keys() # look at all the tables in the database inspector = sql.inspect(self._engine) names = inspector.get_table_names() names = [str(name) for name in names] # clean up metadata by removing stale tables tables = set(self._metadata.tables.keys()) - set(names) #XXX: slow? tables = [self._gettable(key, meta=True) for key in tables] [self._metadata.remove(key) for key in tables] return names def _tables(self, meta=False): "get a dict of tables in the database" if meta: return self._metadata.tables # otherwise, look at all the tables in the database keys = self._keys() return dict((key,self._mktable(key)) for key in keys) #XXX: immutable def _primary(self, key): #XXX: faster if value is table['key'].name ? "get table primary key corresponding to given key" table = self._gettable(key) return table.c[self._key] # interface def __get_state(self): state = self.__state__.copy() state.update(state.pop('config',{})) return state def __archive(self, archive): raise ValueError("cannot set new archive") state = property(__get_state, __archive) pass class sqltable_archive(archive): """dictionary-style interface to a sql database table""" def __init__(self, database=None, table=None, **kwds): """initialize a sql database with a synchronized dictionary interface Connect to an existing database, or initialize a new database, at the selected database url. For example, to use a sqlite database 'foo.db' in the current directory, database='sqlite:///foo.db'. To use a mysql database 'foo' on localhost, database='mysql://user:pass@localhost/foo'. For postgresql, use database='postgresql://user:pass@localhost/foo'. When connecting to sqlite, the default database is ':memory:'; otherwise, the default database is 'defaultdb'. Allows keyword options for database configuration, such as connection pooling. Args: database (str, default=None): database url (see above note) table (str, default='memo'): name of the associated database table serialized (bool, default=True): save objects as pickled strings protocol (int, default=DEFAULT_PROTOCOL): pickling protocol """ #FIXME: (needs doc) if protocol='json', use the json serializer __import_sql__() if table is None: table = 'memo' #XXX: better random unique id ? # create database, if doesn't exist if database is None: database = 'sqlite:///:memory:' elif database == 'sqlite:///': database = 'sqlite:///:memory:' _database = database try: url, dbname = _database.rsplit('/', 1) except ValueError: # only dbname given url, dbname = 'sqlite://', _database _database = "%s/%s" % (url,dbname) if url.endswith(":/") or dbname == '': # then no dbname was given url = _database dbname = 'defaultdb' _database = "%s/%s" % (url,dbname) # set state kwds.pop('id',None) kwds.pop('root',None) self.__state__ = { #XXX: add 'cloud' option? 'serialized': bool(kwds.pop('serialized', True)), 'root': _database, 'id': table, 'protocol': kwds.pop('protocol', dill.DEFAULT_PROTOCOL), # preserve other settings (for copy) 'config': kwds.pop('config', kwds.copy()) } #XXX: _engine and _metadata (and _key and _val) also __state__ ? # get engine kwds['future'] = True # 1.4 & 2.0 if dbname == ':memory:': self._engine = sql.create_engine(url, **kwds) elif _database.startswith('sqlite'): self._engine = sql.create_engine(_database, **kwds) else: self._engine = sql.create_engine(url, future=True) #XXX: **kwds ? try: self._conn = self._engine.connect() if _database.startswith('postgres'): self._conn.connection.connection.set_isolation_level(0) self._conn.execute(sql.text("CREATE DATABASE %s;" % dbname)) self._conn.commit() except Exception: self._conn = None finally: if _database.startswith('postgres'): self._conn.connection.connection.set_isolation_level(1) try: if self._conn is None: self._conn = self._engine.connect() self._conn.execute(sql.text("USE %s;" % dbname)) self._conn.commit() except Exception: pass self._engine = sql.create_engine(_database, **kwds) self._conn = self._engine.connect() # prepare to create table self._metadata = sql.MetaData() self._key = 'Kkey' # primary key name self._val = 'Kval' # object storage name keytype = sql.String(255) #XXX: other better fixed size? if self.__state__['serialized']: proto = self.__state__['protocol'] if type(proto) is str: #XXX: assumes 'json' valtype = sql.PickleType(pickler=json) else: valtype = sql.PickleType(protocol=proto, pickler=dill) else: valtype = sql.Text() #XXX: String(255) or BLOB() ??? # create table, if doesn't exist if isinstance(table, str): #XXX: better str-variants ? or no if ? table = sql.Table(table, self._metadata, sql.Column(self._key, keytype, primary_key=True), sql.Column(self._val, valtype) ) self._key = table.c[self._key] self.__state__['id'] = table # initialize self._metadata.create_all(self._engine) return def __drop__(self, **kwds): """drop the database table EXPERIMENTAL: also drop the associated database. For certain database engines, this may not work due to permission issues. Caller may need to be connected as a superuser and database owner. To drop associated database, use __drop__(database=True) """ if not bool(kwds.get('database', False)): self.__state__['id'].drop(self._engine) #XXX: or delete data ? self._metadata.remove(self.__state__['id']) self._metadata = self._engine = self.__state__['id'] = None return _database = self.__state__['root'] url, dbname = _database.rsplit('/', 1) self._engine = sql.create_engine(url, future=True) # 1.4 & 2.0 try: self._conn = self._engine.connect() if _database.startswith('postgres'): # these two commands require superuser privs self._conn.execute(sql.text("update pg_database set datallowconn = 'false' WHERE datname = '%s';" % dbname)) self._conn.commit() self._conn.execute(sql.text("SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '%s';" % dbname)) # 'pid' used in postgresql >= 9.2 self._conn.commit() self._conn.connection.connection.set_isolation_level(0) self._conn.execute(sql.text("DROP DATABASE %s;" % dbname)) # must be db owner self._conn.commit() if _database.startswith('postgres'): self._conn.connection.connection.set_isolation_level(1) except Exception: dbpath = _database.split('///')[-1] if os.path.exists(dbpath): # else fail silently os.remove(dbpath) self._metadata = self._engine = self._conn = self.__state__['id'] = None return def __len__(self): from sqlalchemy import orm session = orm.sessionmaker(bind=self._engine, future=True)() # 1.4 & 2.0 return int(session.query(self.__state__['id']).count()) def __contains__(self, key): query = sql.select(self._key).where(self._key == key) row = self._conn.execute(query).fetchone() return row is not None __contains__.__doc__ = dict.__contains__.__doc__ def __setitem__(self, key, value): value = {self._val: value} #XXX: force into single item dict...? table = self.__state__['id'] if key in self: values = value query = table.update().where(self._key == key) else: values = {self._key.name: key} values.update(value) query = table.insert() self._conn.execute(query.values(**values)) self._conn.commit() return __setitem__.__doc__ = dict.__setitem__.__doc__ #FIXME: missing __cmp__, __...__ def __eq__(self, y): try: if y.__module__ != self.__module__: return NotImplemented return self.__asdict__() == y.__asdict__() #XXX: faster than get? #if len(self) != len(y): return False #try: s = min(k for k in self if self.get(k) != y.get(k)) #except ValueError: s = [] #try: v = min(k for k in y if y.get(k) != self.get(k)) #except ValueError: v = [] #if s != v: return False #elif s == []: return True #return self[s] == y[v] except: return NotImplemented __eq__.__doc__ = dict.__eq__.__doc__ def __ne__(self, y): y = self.__eq__(y) return NotImplemented if y is NotImplemented else not y __ne__.__doc__ = dict.__ne__.__doc__ def __delitem__(self, key): try: self.pop(key) #FIXME: faster without value lookup except KeyError: memo = {} memo.__delitem__(key) return __delitem__.__doc__ = dict.__delitem__.__doc__ def __getitem__(self, key): query = sql.select(self.__state__['id']).where(self._key == key) row = self._conn.execute(query).fetchone() if row is None: raise KeyError(key) return row._mapping[self._val] __getitem__.__doc__ = dict.__getitem__.__doc__ def __iter__(self): #XXX: should be dictionary-keyiterator query = sql.select(self._key) result = self._conn.execute(query) for row in result: yield row[0] __iter__.__doc__ = dict.__iter__.__doc__ def get(self, key, value=None): query = sql.select(self.__state__['id']).where(self._key == key) row = self._conn.execute(query).fetchone() if row != None: _value = row._mapping[self._val] else: _value = value return _value get.__doc__ = dict.get.__doc__ def clear(self): query = self.__state__['id'].delete() self._conn.execute(query) self._conn.commit() return clear.__doc__ = dict.clear.__doc__ #def insert(self, d): #XXX: don't allow this method, or hide ? # query = self.__state__['id'].insert(d) # self._conn.execute(query) # self._conn.commit() # return def copy(self, name=None): #XXX: always None? or allow other settings? "D.copy(name) -> a copy of D, with a new archive at the given name" if name is None: name = self.name else: pass #FIXME: copy database/table instead of do update below db,table = _sqlname(name) #FIXME: should reference, not copy adict = sqltable_archive(database=db, table=table, **self.state) adict.update(self.__asdict__()) return adict def fromkeys(self, *args): #XXX: build a dict (not an archive)? return dict.fromkeys(*args) fromkeys.__doc__ = dict.fromkeys.__doc__ def __asdict__(self): """build a dictionary containing the archive contents""" return dict(self.items()) def __repr__(self): return "sqltable_archive('%s' %s, cached=False)" % (self.name, self.__asdict__()) __repr__.__doc__ = dict.__repr__.__doc__ def keys(self): return KeysView(self) #XXX: show keys not dict def items(self): return ItemsView(self) #XXX: show keys not dict def values(self): return ValuesView(self) #XXX: show keys not dict keys.__doc__ = dict.keys.__doc__ items.__doc__ = dict.items.__doc__ values.__doc__ = dict.values.__doc__ def popkeys(self, keys, *value): """ D.popkeys(k[,d]) -> v, remove specified keys and return corresponding values. If key in keys is not found, d is returned if given, otherwise KeyError is raised.""" if not hasattr(keys, '__iter__'): return self.pop(keys, *value) if len(value): return [self.pop(k, *value) for k in keys] memo = self.fromkeys(self.keys()) # 'shadow' dict [memo.pop(k) for k in keys] return [self.pop(k) for k in keys] def pop(self, key, *value): L = len(value) if L > 1: raise TypeError("pop expected at most 2 arguments, got %s" % str(L+1)) query = sql.select(self.__state__['id']).where(self._key == key) row = self._conn.execute(query).fetchone() if row != None: _value = row._mapping[self._val] else: if not L: raise KeyError(key) _value = value[0] query = sql.delete(self.__state__['id']).where(self._key == key) self._conn.execute(query) self._conn.commit() return _value pop.__doc__ = dict.pop.__doc__ def popitem(self): key = self.__iter__() try: key = next(key) except StopIteration: raise KeyError("popitem(): dictionary is empty") return (key, self.pop(key)) popitem.__doc__ = dict.popitem.__doc__ def setdefault(self, key, *value): L = len(value) if L > 1: raise TypeError("setvalue expected at most 2 arguments, got %s" % str(L+1)) query = sql.select(self.__state__['id']).where(self._key == key) row = self._conn.execute(query).fetchone() if row != None: _value = row._mapping[self._val] else: if not L: _value = None else: _value = value[0] self.__setitem__(key, _value) return _value setdefault.__doc__ = dict.setdefault.__doc__ def update(self, adict, **kwds): if hasattr(adict,'__asdict__'): adict = adict.__asdict__() elif hasattr(adict, 'copy'): adict = adict.copy() else: adict = dict(adict) adict.update(**kwds) [self.__setitem__(k,v) for (k,v) in adict.items()] return #XXX: should do the above all at once, and more efficiently update.__doc__ = dict.update.__doc__ # interface def __get_name(self): return "%s?table=%s" % (self.__state__['root'], self.__state__['id']) def __get_state(self): state = self.__state__.copy() db,table = _sqlname(self.name) state.update(state.pop('config',{})) state.update({'root': db, 'id': table}) return state def __archive(self, archive): raise ValueError("cannot set new archive") name = property(__get_name, __archive) state = property(__get_state, __archive) pass else: class sqltable_archive(archive): #XXX: requires UTF-8 key; #FIXME: use sqlite3.dbapi2 """dictionary-style interface to a sql database table""" def __init__(self, database=None, table=None, **kwds): #serialized """initialize a sql database with a synchronized dictionary interface Connect to an existing database, or initialize a new database, at the selected database url. For example, to use a sqlite database 'foo.db' in the current directory, database='sqlite:///foo.db'. To use a mysql or postgresql database, sqlalchemy must be installed. When connecting to sqlite, the default database is ':memory:'. Storable values are limited to strings, integers, floats, and other basic objects. To store functions, classes, and similar constructs, sqlalchemy must be installed. Args: database (str, default=None): database url (see above note) table (str, default='memo'): name of the associated database table """ import sqlite3 as db if table is None: table = 'memo' # create database, if doesn't exist if database is None: database = 'sqlite:///:memory:' elif database == 'sqlite:///': database = 'sqlite:///:memory:' _database = database if not _database.startswith('sqlite:///'): if _database.count(':')+_database.count('/'): raise ValueError("install sqlalchemy for non-sqlite database support") _database = 'sqlite:///'+_database dbname = _database.split('sqlite:///')[-1] # set state kwds.pop('id',None) kwds.pop('root',None) kwds.pop('serialized', True) # 'serialized' is not available kwds.pop('protocol', None) # 'protocol' is not available self.__state__ = { 'serialized': False, 'root': _database, 'id': table, 'protocol': None, # preserve other settings (for copy) 'config': kwds.pop('config', kwds.copy()) } #XXX: _engine and _metadata (and _key and _val) also __state__ ? # create table, if doesn't exist self._conn = db.connect(dbname) self._engine = self._conn.cursor() sql = "create table if not exists %s(argstr, fval)" % table self._engine.execute(sql) # compatibility self._metadata = None self._key = 'Kkey' self._val = 'Kval' return def __drop__(self, **kwds): """drop the database table EXPERIMENTAL: also drop the associated database. For certain database engines, this may not work due to permission issues. Caller may need to be connected as a superuser and database owner. To drop associated database, use __drop__(database=True) """ if not bool(kwds.get('database', False)): self._engine.executescript('drop table if exists %s;' % self.__state__['id']) self._engine = self._conn = self.__state__['id'] = None return _database = self.__state__['root'] try: dbname = _database.lstrip('sqlite:///') conn = db.connect(':memory:') conn.execute("DROP DATABASE %s;" % dbname) #FIXME: always fails except Exception: dbpath = _database.split('///')[-1] if os.path.exists(dbpath): # else fail silently os.remove(dbpath) self._engine = self._conn = self.__state__['id'] = None return def __len__(self): return len(self.__asdict__()) def __contains__(self, key): return bool(self._select_key_items(key)) __contains__.__doc__ = dict.__contains__.__doc__ def __setitem__(self, key, value): #XXX: maintains 'history' of values sql = "insert into %s values(?,?)" % self.__state__['id'] self._engine.execute(sql, (key,value)) self._conn.commit() return __setitem__.__doc__ = dict.__setitem__.__doc__ #FIXME: missing __cmp__, __...__ def __eq__(self, y): try: if y.__module__ != self.__module__: return NotImplemented return self.__asdict__() == y.__asdict__() #XXX: faster than get? #if len(self) != len(y): return False #try: s = min(k for k in self if self.get(k) != y.get(k)) #except ValueError: s = [] #try: v = min(k for k in y if y.get(k) != self.get(k)) #except ValueError: v = [] #if s != v: return False #elif s == []: return True #return self[s] == y[v] except: return NotImplemented __eq__.__doc__ = dict.__eq__.__doc__ def __ne__(self, y): y = self.__eq__(y) return NotImplemented if y is NotImplemented else not y __ne__.__doc__ = dict.__ne__.__doc__ def __delitem__(self, key): try: self.pop(key) #FIXME: faster without value lookup except KeyError: memo = {} memo.__delitem__(key) return __delitem__.__doc__ = dict.__delitem__.__doc__ def __getitem__(self, key): res = self._select_key_items(key) if res: return res[-1][-1] # always get the last one raise KeyError(key) __getitem__.__doc__ = dict.__getitem__.__doc__ def __iter__(self): #XXX: should be dictionary-keyiterator sql = "select argstr from %s" % self.__state__['id'] return (k[-1] for k in set(self._engine.execute(sql))) __iter__.__doc__ = dict.__iter__.__doc__ def get(self, key, value=None): res = self._select_key_items(key) if res: value = res[-1][-1] return value get.__doc__ = dict.get.__doc__ def clear(self): [self.pop(k) for k in self.keys()] # better delete table, add empty ? return clear.__doc__ = dict.clear.__doc__ def copy(self, name=None): #XXX: always None? or allow other settings? "D.copy(name) -> a copy of D, with a new archive at the given name" if name is None: name = self.name else: pass #FIXME: copy database/table instead of do update below db,table = _sqlname(name) #FIXME: should reference, not copy adict = sqltable_archive(database=db, table=table, **self.state) adict.update(self.__asdict__()) return adict def fromkeys(self, *args): #XXX: build a dict (not an archive)? return dict.fromkeys(*args) fromkeys.__doc__ = dict.fromkeys.__doc__ def __asdict__(self): """build a dictionary containing the archive contents""" sql = "select * from %s" % self.__state__['id'] res = self._engine.execute(sql) d = {} [d.update({k:v}) for (k,v) in res] # always get the last one return d def __repr__(self): return "sqltable_archive('%s' %s, cached=False)" % (self.name, self.__asdict__()) __repr__.__doc__ = dict.__repr__.__doc__ def keys(self): return KeysView(self) #XXX: show keys not dict def items(self): return ItemsView(self) #XXX: show keys not dict def values(self): return ValuesView(self) #XXX: show keys not dict keys.__doc__ = dict.keys.__doc__ items.__doc__ = dict.items.__doc__ values.__doc__ = dict.values.__doc__ def popkeys(self, keys, *value): """ D.popkeys(k[,d]) -> v, remove specified keys and return corresponding values. If key in keys is not found, d is returned if given, otherwise KeyError is raised.""" if not hasattr(keys, '__iter__'): return self.pop(keys, *value) if len(value): return [self.pop(k, *value) for k in keys] memo = self.fromkeys(self.keys()) # 'shadow' dict [memo.pop(k) for k in keys] return [self.pop(k) for k in keys] def pop(self, key, *value): L = len(value) if L > 1: raise TypeError("pop expected at most 2 arguments, got %s" % str(L+1)) res = self._select_key_items(key) if res: _value = res[-1][-1] else: if not L: raise KeyError(key) _value = value[0] sql = "delete from %s where argstr = ?" % self.__state__['id'] self._engine.execute(sql, (key,)) self._conn.commit() return _value pop.__doc__ = dict.pop.__doc__ def popitem(self): key = self.__iter__() try: key = next(key) except StopIteration: raise KeyError("popitem(): dictionary is empty") return (key, self.pop(key)) popitem.__doc__ = dict.popitem.__doc__ def setdefault(self, key, *value): L = len(value) if L > 1: raise TypeError("setvalue expected at most 2 arguments, got %s" % str(L+1)) res = self._select_key_items(key) if res: _value = res[-1][-1] else: if not L: _value = None else: _value = value[0] self.__setitem__(key, _value) return _value setdefault.__doc__ = dict.setdefault.__doc__ def update(self, adict, **kwds): if hasattr(adict,'__asdict__'): adict = adict.__asdict__() elif hasattr(adict, 'copy'): adict = adict.copy() else: adict = dict(adict) adict.update(**kwds) [self.__setitem__(k,v) for (k,v) in adict.items()] return update.__doc__ = dict.update.__doc__ def _select_key_items(self, key): '''Return a tuple of (key, value) pairs that match the specified key''' sql = "select * from %s where argstr = ?" % self.__state__['id'] return tuple(self._engine.execute(sql, (key,))) # interface def __get_name(self): return "%s?table=%s" % (self.__state__['root'], self.__state__['id']) def __get_state(self): state = self.__state__.copy() state.update(state.pop('config',{})) return state def __archive(self, archive): raise ValueError("cannot set new archive") name = property(__get_name, __archive) state = property(__get_state, __archive) pass sql_archive = sqltable_archive #XXX: or NotImplemented ? if hdf: class hdf_archive(archive): """dictionary-style interface to a hdf5 file""" def __init__(self, filename=None, serialized=True, **kwds): """initialize a hdf5 file with a synchronized dictionary interface Args: filename (str, default='memo.hdf5'): path of the file archive serialized (bool, default=True): pickle saved python objects protocol (int, default=DEFAULT_PROTOCOL): pickling protocol meta (bool, default=False): store in root metadata (not in dataset) """ #FIXME: (needs doc) if protocol='json', use the json serializer __import_hdf__() if filename is None: filename = 'memo.hdf5' elif not filename.endswith(('.hdf5','.hdf','.h5')): filename = filename+'.hdf5' # set state meta = kwds.get('meta', False) self.__state__ = { 'id': filename, 'serialized': serialized, 'protocol': kwds.get('protocol', 0 if meta else None), 'meta': meta } #XXX: add 'cloud' option? if not os.path.exists(filename): self.__save__({}) return def __reduce__(self): fname = self.__state__['id'] serial = self.__state__['serialized'] state = {'__state__': self.__state__} return (self.__class__, (fname, serial), state) def _attrs(self, file): return file.attrs if self.__state__['meta'] else file def _loadkey(self, key): # get a key from the archive 'convert from a key stored in the HDF file' #key = key.encode() if type(key) is str else key pik = json if type(self.__state__['protocol']) is str else dill return pik.loads(key) def _loadval(self, value): # get a value from the archive 'convert from a value stored in the HDF file' pik = json if type(self.__state__['protocol']) is str else dill if self.__state__['meta']: return pik.loads(value.tobytes()) if self.__state__['serialized'] else value return pik.loads(value[0].tobytes()) if self.__state__['serialized'] else value[()] #XXX: correct for arrays? def _dumpkey(self, key): # lookup a key in the archive 'convert to a key stored in the HDF file' if type(self.__state__['protocol']) is str: return json.dumps(key).encode() return dill.dumps(key, protocol=0) def _dumpval(self, value): # lookup a value in the archive 'convert to a value stored in the HDF file' if self.__state__['serialized']: protocol = self.__state__['protocol'] #XXX: fix at 0? if type(protocol) is str: value = json.dumps(value).encode() else: #XXX: don't use void for protocol = 0? void = hdf.version.numpy.void #XXX: better import numpy? value = void(dill.dumps(value, protocol=protocol)) return value if self.__state__['meta'] else [value] return value #XXX: or [value]? (so no scalars) def __asdict__(self): """build a dictionary containing the archive contents""" filename = self.__state__['id'] f = None try: memo = {} f = hdf.File(filename, 'r') _f = self._attrs(f) for k,v in f.items(): memo[self._loadkey(k.encode())] = self._loadval(v) except TypeError: # we have an unhashable type memo = {} 'unhashable type' except: #XXX: should only catch appropriate exceptions memo = {} #raise OSError("error reading file archive %s" % filename) finally: if f is not None: f.close() return memo def __save__(self, memo=None, new=True): """create an archive from the given dictionary""" if memo == None: return filename = self.__state__['id'] _filename = os.path.join(os.path.dirname(os.path.abspath(filename)), TEMP+hash(random(), 'md5')) if new else filename # create a temporary file, and dump the results f = None try: f = hdf.File(_filename, 'w' if new else 'a') for k,v in memo.items(): #self._attrs(f).update({self._dumpkey(k): self._dumpval(v)}) _f = self._attrs(f) _k = self._dumpkey(k) _f.pop(_k,None) _f[_k] = self._dumpval(v) except OSError: "failed to populate file for %s" % str(filename) finally: if f is not None: f.close() if not new: return # move the results to the proper place try: os.remove(filename) except: pass try: os.renames(_filename, filename) except OSError: "error in populating %s" % str(filename) return #FIXME: missing __cmp__, __...__ def __eq__(self, y): try: if y.__module__ != self.__module__: return NotImplemented return self.__asdict__() == y.__asdict__() #XXX: faster than get? except: return NotImplemented __eq__.__doc__ = dict.__eq__.__doc__ def __ne__(self, y): y = self.__eq__(y) return NotImplemented if y is NotImplemented else not y __ne__.__doc__ = dict.__ne__.__doc__ def __delitem__(self, key): filename = self.__state__['id'] f = None try: f = hdf.File(filename, 'a') self._attrs(f).__delitem__(self._dumpkey(key)) except: #XXX: should only catch appropriate exceptions raise KeyError(key) #raise OSError("error reading file archive %s" % filename) finally: if f is not None: f.close() return __delitem__.__doc__ = dict.__delitem__.__doc__ def __getitem__(self, key): filename = self.__state__['id'] f = None try: f = hdf.File(filename, 'r') val = self._loadval(self._attrs(f)[self._dumpkey(key)]) except: #XXX: should only catch appropriate exceptions raise KeyError(key) #raise OSError("error reading file archive %s" % filename) finally: if f is not None: f.close() return val __getitem__.__doc__ = dict.__getitem__.__doc__ def __repr__(self): return "hdf_archive('%s', %s, cached=False)" % (self.name, self.__asdict__()) __repr__.__doc__ = dict.__repr__.__doc__ def __setitem__(self, key, value): filename = self.__state__['id'] f = None try: f = hdf.File(filename, 'a') #self._attrs(f).update({self._dumpkey(key): self._dumpval(value)}) _f = self._attrs(f) _k = self._dumpkey(key) _f.pop(_k,None) _f[_k] = self._dumpval(value) except KeyError: #XXX: should only catch appropriate exceptions raise KeyError(key) #raise OSError("error reading file archive %s" % filename) finally: if f is not None: f.close() return __setitem__.__doc__ = dict.__setitem__.__doc__ def clear(self): self.__save__({}, new=True) return clear.__doc__ = dict.clear.__doc__ def copy(self, name=None): #XXX: always None? or allow other settings? "D.copy(name) -> a copy of D, with a new archive at the given name" filename = self.__state__['id'] if name is None: name = filename else: shutil.copy2(filename, name) #XXX: overwrite? adict = hdf_archive(filename=name, **self.state) #adict.update(self.__asdict__()) return adict def __drop__(self): """drop the associated database EXPERIMENTAL: Deleting the file may not work due to permission issues. Caller may need to be connected as a superuser and database owner. """ self.clear() import os if os.path.exists(self.__state__['id']): os.remove(self.__state__['id']) return def fromkeys(self, *args): #XXX: build a dict (not an archive)? return dict.fromkeys(*args) fromkeys.__doc__ = dict.fromkeys.__doc__ def get(self, key, value=None): try: _value = self.__getitem__(key) except KeyError: _value = value return _value get.__doc__ = dict.get.__doc__ def keys(self): return KeysView(self) #XXX: show keys not dict keys.__doc__ = dict.keys.__doc__ def values(self): return ValuesView(self) #XXX: show values not dict values.__doc__ = dict.values.__doc__ def __contains__(self, key): filename = self.__state__['id'] f = None try: f = hdf.File(filename, 'r') has_key = self._dumpkey(key) in self._attrs(f) except KeyError: #XXX: should only catch appropriate exceptions raise KeyError(key) #raise OSError("error reading file archive %s" % filename) finally: if f is not None: f.close() return has_key __contains__.__doc__ = dict.__contains__.__doc__ def __iter__(self): return iter(self.keys()) __iter__.__doc__ = dict.__iter__.__doc__ def items(self): return ItemsView(self) #XXX: show items not dict items.__doc__ = dict.items.__doc__ def popkeys(self, keys, *value): """ D.popkeys(k[,d]) -> v, remove specified keys and return corresponding values. If key in keys is not found, d is returned if given, otherwise KeyError is raised.""" if not hasattr(keys, '__iter__'): return self.pop(keys, *value) if len(value): return [self.pop(k, *value) for k in keys] memo = self.fromkeys(self.keys()) # 'shadow' dict [memo.pop(k) for k in keys] return [self.pop(k) for k in keys] #XXX: should open file once def pop(self, key, *value): value = (self._dumpval(val) for val in value) filename = self.__state__['id'] f = None try: f = hdf.File(filename, 'a') val = self._loadval(self._attrs(f).pop(self._dumpkey(key), *value)) except KeyError: #XXX: should only catch appropriate exceptions raise KeyError(key) #raise OSError("error reading file archive %s" % filename) finally: if f is not None: f.close() return val pop.__doc__ = dict.pop.__doc__ def popitem(self): filename = self.__state__['id'] f = None try: f = hdf.File(filename, 'a') key,val = self._attrs(f).popitem() key,val = self._loadkey(key),self._loadval(val) except KeyError: #XXX: should only catch appropriate exceptions d = {} d.popitem() #raise OSError("error reading file archive %s" % filename) finally: if f is not None: f.close() return key,val popitem.__doc__ = dict.popitem.__doc__ def setdefault(self, key, *value): res = self.get(key, *value) self.__setitem__(key, res) return res setdefault.__doc__ = dict.setdefault.__doc__ def update(self, adict, **kwds): if hasattr(adict,'__asdict__'): adict = adict.__asdict__() memo = {} memo.update(adict, **kwds) self.__save__(memo, new=False) return update.__doc__ = dict.update.__doc__ def __len__(self): filename = self.__state__['id'] f = None try: f = hdf.File(filename, 'r') _len = len(self._attrs(f)) except: #XXX: should only catch appropriate exceptions raise OSError("error reading file archive %s" % filename) finally: if f is not None: f.close() return _len # interface def __get_name(self): return os.path.basename(self.__state__['id']) def __archive(self, archive): raise ValueError("cannot set new archive") name = property(__get_name, __archive) pass class hdfdir_archive(archive): """dictionary-style interface to a folder of hdf5 files""" def __init__(self, dirname=None, serialized=True, **kwds): """initialize a hdf5 file with a synchronized dictionary interface Args: dirname (str, default='memo'): path of the archive root directory serialized (bool, default=True): pickle saved python objects permissions (octal, default=0o775): read/write permission indicator protocol (int, default=DEFAULT_PROTOCOL): pickling protocol meta (bool, default=False): store in root metadata (not in dataset) """ #FIXME: (needs doc) if protocol='json', use the json serializer __import_hdf__() if dirname is None: #FIXME: default root as /tmp or something better dirname = 'memo' #FIXME: need better default # set state meta = kwds.get('meta', False) self.__state__ = { 'id': dirname, 'serialized': serialized, 'permissions': kwds.get('permissions', None), 'protocol': kwds.get('protocol', 0 if meta else None), 'meta': meta } #XXX: add 'cloud' option? try: self.__state__['id'] = mkdir(dirname, mode=self.__state__['permissions']) except OSError: # then directory already exists self.__state__['id'] = os.path.abspath(dirname) return def __reduce__(self): dirname = self.name serial = self.__state__['serialized'] state = {'__state__': self.__state__} return (self.__class__, (dirname, serial), state) def __asdict__(self): """build a dictionary containing the archive contents""" # get the names of all directories in the directory keys = self._keydict() # get the values return dict((key,self.__getitem__(key)) for key in keys) #FIXME: missing __cmp__, __...__ def __eq__(self, y): try: if y.__module__ != self.__module__: return NotImplemented return self.__asdict__() == y.__asdict__() #XXX: faster than get? except: return NotImplemented __eq__.__doc__ = dict.__eq__.__doc__ def __ne__(self, y): y = self.__eq__(y) return NotImplemented if y is NotImplemented else not y __ne__.__doc__ = dict.__ne__.__doc__ def __delitem__(self, key): try: memo = {key: None} self._rmdir(key) except: memo = {} memo.__delitem__(key) return __delitem__.__doc__ = dict.__delitem__.__doc__ def __getitem__(self, key): return self._lookup(key) __getitem__.__doc__ = dict.__getitem__.__doc__ def __repr__(self): return "hdfdir_archive('%s', %s, cached=False)" % (self.name, self.__asdict__()) __repr__.__doc__ = dict.__repr__.__doc__ def __setitem__(self, key, value): self._store(key, value, input=False) # input=True also stores input return __setitem__.__doc__ = dict.__setitem__.__doc__ def clear(self): rmtree(self.__state__['id'], self=False, ignore_errors=True) return clear.__doc__ = dict.clear.__doc__ def copy(self, name=None): #XXX: always None? or allow other settings? "D.copy(name) -> a copy of D, with a new archive at the given name" if name is None: name = self.__state__['id'] else: #XXX: overwrite? shutil.copytree(self.__state__['id'], os.path.abspath(name)) adict = hdfdir_archive(dirname=name, **self.state) #adict.update(self.__asdict__()) return adict def __drop__(self): """drop the associated database EXPERIMENTAL: Deleting the directory may not work due to permission issues. Caller may need to be connected as a superuser and database owner. """ self.clear() import os import shutil if os.path.exists(self.__state__['id']): shutil.rmtree(self.__state__['id']) return def fromkeys(self, *args): #XXX: build a dict (not an archive)? return dict.fromkeys(*args) fromkeys.__doc__ = dict.fromkeys.__doc__ def get(self, key, value=None): try: return self.__getitem__(key) except: return value get.__doc__ = dict.get.__doc__ def __contains__(self, key): _dir = self._getdir(key) return os.path.exists(_dir) __contains__.__doc__ = dict.__contains__.__doc__ def __iter__(self): return iter(self._keydict().keys()) __iter__.__doc__ = dict.__iter__.__doc__ def keys(self): return KeysView(self) #XXX: show keys not dict keys.__doc__ = dict.keys.__doc__ def items(self): return ItemsView(self) #XXX: show items not dict items.__doc__ = dict.items.__doc__ def values(self): return ValuesView(self) #XXX: show values not dict values.__doc__ = dict.values.__doc__ def popkeys(self, keys, *value): """ D.popkeys(k[,d]) -> v, remove specified keys and return corresponding values. If key in keys is not found, d is returned if given, otherwise KeyError is raised.""" if not hasattr(keys, '__iter__'): return self.pop(keys, *value) if len(value): return [self.pop(k, *value) for k in keys] memo = self._keydict() # 'shadow' dict for desired error behavior [memo.pop(k) for k in keys] return [self.pop(k) for k in keys] def pop(self, key, *value): #XXX: or make DEAD ? try: memo = {key: self.__getitem__(key)} self._rmdir(key) except: memo = {} res = memo.pop(key, *value) return res pop.__doc__ = dict.pop.__doc__ def popitem(self): key = self.__iter__() try: key = next(key) except StopIteration: raise KeyError("popitem(): dictionary is empty") return (key, self.pop(key)) popitem.__doc__ = dict.popitem.__doc__ def setdefault(self, key, *value): res = self.get(key, *value) self.__setitem__(key, res) return res setdefault.__doc__ = dict.setdefault.__doc__ def update(self, adict, **kwds): if hasattr(adict,'__asdict__'): adict = adict.__asdict__() memo = {} memo.update(adict, **kwds) #XXX: could be better ? for (key,val) in memo.items(): self.__setitem__(key,val) return update.__doc__ = dict.update.__doc__ def __len__(self): return len(self._lsdir()) def _fname(self, key): "generate suitable filename for a given key" # special handling for pickles; enable non-strings (however 1=='1') try: ispickle = key.startswith(PROTO) and key.endswith(STOP) except: ispickle = False #FIXME: protocol 0,1 don't startwith(PROTO) key = hash(key, 'md5') if ispickle else str(key) #XXX: always hash? return key.replace('-','_') #XXX: special handling in ispickle for protocol=json? ##XXX: below probably fails on windows, and could be huge... use 'md5' #return repr(key)[1:-1] if ispickle else str(key) # or repr? def _mkdir(self, key): "create results subdirectory corresponding to given key" key = self._fname(key) try: return mkdir(PREFIX+key, root=self.__state__['id'], mode=self.__state__['permissions']) except OSError: # then directory already exists return self._getdir(key) def _getdir(self, key): "get results directory name corresponding to given key" key = self._fname(key) return os.path.join(self.__state__['id'], PREFIX+key) def _rmdir(self, key): "remove results subdirectory corresponding to given key" rmtree(self._getdir(key), self=True, ignore_errors=True) return def _lsdir(self): "get a list of subdirectories in the root directory" return walk(self.__state__['id'],patterns=PREFIX+'*',recurse=False,folders=True,files=False,links=False) def _hasinput(self, root): "check if results subdirectory has stored input file" return bool(walk(root,patterns=self._args,recurse=False,folders=False,files=True,links=False)) def _getkey(self, root): "get key given a results subdirectory name" key = os.path.basename(root)[2:] return self._lookup(key,input=True) if self._hasinput(root) else key def _keydict(self): "get a dict of subdirectories in the root directory, with dummy values" keys = self._lsdir() return dict((self._getkey(key),None) for key in keys) #FIXME: dict((i,self._getkey(key)) for i,key in enumerate(keys)) def _reverse_lookup(self, args): #XXX: guaranteed 1-to-1 mapping? "get subdirectory name from args" d = {} for key in iter(self._keydict()): try: if args == self._lookup(key, input=True): d[args] = None #XXX: unnecessarily memory intensive? break except KeyError: continue # throw KeyError(args) if key not found del d[args] return key #NOTE: all above methods are virtually identical to those in dir_archive #NOTE: all below methods are similar, and could be merged to dir_archive def _lookup(self, key, input=False): "get input or output from subdirectory name" _dir = self._getdir(key) _file = self._args if input else self._file _file = os.path.join(_dir, _file) try: adict = {'serialized':self.__state__['serialized'],\ 'protocol':self.__state__['protocol'],\ 'meta':self.__state__['meta'], 'cached':False} #XXX: assumes one entry per file; ...could use name as key? #XXX: alternately, could store {key:value} (i.e. use one file)? memo = tuple(hdf_archive(_file, **adict).__asdict__().values())[0] #memo = next(iter(hdf_archive(_file, **adict).values())) except: #XXX: should only catch the appropriate exceptions memo = None #FIXME: not sure if _lookup should delete a key in all cases #FIXME: (maybe only delete key when it's new, but fails) #self._rmdir(key) # don't leave a keyfile on disk raise KeyError(key) #raise OSError("error reading directory for '%s'" % key) return memo def _store(self, key, value, input=False): "store output (and possibly input) in a subdirectory" _key = TEMP+hash(random(), 'md5') # create an input file when key is not suitable directory name if self._fname(key) != key: input=True #XXX: errors if protocol=0,1? # create a temporary directory, and dump the results try: _file = os.path.join(self._mkdir(_key), self._file) if input: _args = os.path.join(self._getdir(_key), self._args) adict = {'serialized':self.__state__['serialized'],\ 'protocol':self.__state__['protocol'],\ 'meta':self.__state__['meta']} #XXX: assumes one entry per file; ...could use name as key? memo = hdf_archive(_file, **adict) memo[None] = value if input: memo = hdf_archive(_args, **adict) memo[None] = key except (OSError,TypeError): "failed to populate directory for '%s'" % str(key) # move the results to the proper place try: #XXX: possible permissions issues here self._rmdir(key) #XXX: 'key' must be a suitable dir name os.renames(self._getdir(_key), self._getdir(key)) # except TypeError: #XXX: catch key that isn't converted to safe filename # "error in populating directory for '%s'" % str(key) except OSError: #XXX: if rename fails, may need cleanup (_rmdir ?) "error in populating directory for '%s'" % str(key) def _get_args(self): return 'input.hdf5' def _get_file(self): return 'output.hdf5' def _set_file(self, file): raise NotImplementedError("cannot set attribute '_file'") # interface def __get_name(self): return os.path.basename(self.__state__['id']) def __archive(self, archive): raise ValueError("cannot set new archive") name = property(__get_name, __archive) _file = property(_get_file, _set_file) _args = property(_get_args, _set_file) pass else: class hdf_archive(archive): """dictionary-style interface to a hdf5 file""" def __init__(self, *args, **kwds): import h5py pass class hdfdir_archive(archive): """dictionary-style interface to a folder of hdf5 files""" def __init__(self, *args, **kwds): import h5py pass # backward compatibility archive_dict = cache db_archive = sqltable_archive # EOF uqfoundation-klepto-69cd6ce/klepto/_cache.py000066400000000000000000001525441455531556400213310ustar00rootroot00000000000000#!/usr/bin/env python # # code inspired by Raymond Hettinger's LFU and LRU cache decorators # on http://code.activestate.com/recipes/498245-lru-and-lfu-cache-decorators # and subsequent forks as well as the version available in python3.3 # # Author: Mike McKerns (mmckerns @caltech and @uqfoundation) # Copyright (c) 2013-2016 California Institute of Technology. # Copyright (c) 2016-2024 The Uncertainty Quantification Foundation. # License: 3-clause BSD. The full license text is available at: # - https://github.com/uqfoundation/klepto/blob/master/LICENSE """ a selection of caching decorators """ from functools import update_wrapper, partial from klepto.archives import cache as archive_dict from klepto.keymaps import hashmap from klepto.tools import CacheInfo from klepto.rounding import deep_round, simple_round from ._inspect import _keygen __all__ = ['no_cache','inf_cache','lfu_cache',\ 'lru_cache','mru_cache','rr_cache'] class Counter(dict): 'Mapping where default values are zero' def __missing__(self, key): return 0 #XXX: what about caches that expire due to time, calls, etc... #XXX: check the impact of not serializing by default, and hashmap by default class no_cache(object): """empty (NO) cache decorator. Unlike other cache decorators, this decorator does not cache. It is a dummy that collects statistics and conforms to the caching interface. This decorator takes an integer tolerance 'tol', equal to the number of decimal places to which it will round off floats, and a bool 'deep' for whether the rounding on inputs will be 'shallow' or 'deep'. Note that rounding is not applied to the calculation of new results, but rather as a simple form of cache interpolation. For example, with tol=0 and a cached value for f(3.0), f(3.1) will lookup f(3.0) in the cache while f(3.6) will store a new value; however if tol=1, both f(3.1) and f(3.6) will store new values. maxsize = maximum cache size [fixed at maxsize=0] cache = storage hashmap (default is {}) keymap = cache key encoder (default is keymaps.hashmap(flat=True)) ignore = function argument names and indicies to 'ignore' (default is None) tol = integer tolerance for rounding (default is None) deep = boolean for rounding depth (default is False, i.e. 'shallow') purge = boolean for purge cache to archive at maxsize (fixed at True) If *keymap* is given, it will replace the hashing algorithm for generating cache keys. Several hashing algorithms are available in 'keymaps'. The default keymap requires arguments to the cached function to be hashable. If the keymap retains type information, then arguments of different types will be cached separately. For example, f(3.0) and f(3) will be treated as distinct calls with distinct results. Cache typing has a memory penalty, and may also be ignored by some 'keymaps'. Here, the keymap is only used to look up keys in an associated archive. If *ignore* is given, the keymap will ignore the arguments with the names and/or positional indicies provided. For example, if ignore=(0,), then the key generated for f(1,2) will be identical to that of f(3,2) or f(4,2). If ignore=('y',), then the key generated for f(x=3,y=4) will be identical to that of f(x=3,y=0) or f(x=3,y=10). If ignore=('*','**'), all varargs and varkwds will be 'ignored'. Ignored arguments never trigger a recalculation (they only trigger cache lookups), and thus are 'ignored'. When caching class methods, it may be useful to ignore=('self',). View cache statistics (hit, miss, load, maxsize, size) with f.info(). Clear the cache and statistics with f.clear(). Replace the cache archive with f.archive(obj). Load from the archive with f.load(), and dump from the cache to the archive with f.dump(). """ def __init__(self, maxsize=0, cache=None, keymap=None, ignore=None, tol=None, deep=False, purge=True): #if maxsize is not 0: raise ValueError('maxsize cannot be set') maxsize = 0 #XXX: allow maxsize to be given but ignored ? purge = True #XXX: allow purge to be given but ignored ? if cache is None: cache = archive_dict() elif type(cache) is dict: cache = archive_dict(cache) if keymap is None: keymap = hashmap(flat=True) if ignore is None: ignore = tuple() if deep: rounded = deep_round else: rounded = simple_round #else: rounded = shallow_round #FIXME: slow @rounded(tol) def rounded_args(*args, **kwds): return (args, kwds) # set state self.__state__ = { 'maxsize': maxsize, 'cache': cache, 'keymap': keymap, 'ignore': ignore, 'roundargs': rounded_args, 'tol': tol, 'deep': deep, 'purge': purge, } return def __call__(self, user_function): #cache = dict() # mapping of args to results stats = [0, 0, 0] # make statistics updateable non-locally HIT, MISS, LOAD = 0, 1, 2 # names for the stats fields _len = len # localize the global len() function #lock = RLock() # linkedlist updates aren't threadsafe maxsize = self.__state__['maxsize'] cache = self.__state__['cache'] keymap = self.__state__['keymap'] ignore = self.__state__['ignore'] rounded_args = self.__state__['roundargs'] def wrapper(*args, **kwds): _args, _kwds = rounded_args(*args, **kwds) _args, _kwds = _keygen(user_function, ignore, *_args, **_kwds) key = keymap(*_args, **_kwds) # look in archive if cache.archived(): cache.load(key) try: result = cache[key] cache.clear() stats[LOAD] += 1 except KeyError: # if not found, then compute result = user_function(*args, **kwds) cache[key] = result stats[MISS] += 1 # purge cache if _len(cache) > maxsize: #XXX: better: if cache is cache.archive ? if cache.archived(): cache.dump() cache.clear() return result def archive(obj): """Replace the cache archive""" if isinstance(obj, archive_dict): cache.archive = obj.archive else: cache.archive = obj def key(*args, **kwds): """Get the cache key for the given *args,**kwds""" _args, _kwds = rounded_args(*args, **kwds) _args, _kwds = _keygen(user_function, ignore, *_args, **_kwds) return keymap(*_args, **_kwds) def lookup(*args, **kwds): """Get the stored value for the given *args,**kwds""" _args, _kwds = rounded_args(*args, **kwds) _args, _kwds = _keygen(user_function, ignore, *_args, **_kwds) return cache[keymap(*_args, **_kwds)] def __get_cache(): """Get the cache""" return cache def __get_mask(): """Get the (ignore) mask""" return ignore def __get_keymap(): """Get the keymap""" return keymap def clear(keepstats=False): """Clear the cache and statistics""" if not keepstats: stats[:] = [0, 0, 0] def info(): """Report cache statistics""" return CacheInfo(stats[HIT], stats[MISS], stats[LOAD], maxsize, len(cache)) # interface wrapper.__wrapped__ = user_function #XXX: better is handle to key_function=keygen(ignore)(user_function) ? wrapper.info = info wrapper.clear = clear wrapper.load = cache.load wrapper.dump = cache.dump wrapper.archive = archive wrapper.archived = cache.archived wrapper.key = key wrapper.lookup = lookup wrapper.__cache__ = __get_cache wrapper.__mask__ = __get_mask wrapper.__map__ = __get_keymap #wrapper._queue = None #XXX return update_wrapper(wrapper, user_function) def __get__(self, obj, objtype): """support instance methods""" return partial(self.__call__, obj) def __reduce__(self): cache = self.__state__['cache'] keymap = self.__state__['keymap'] ignore = self.__state__['ignore'] tol = self.__state__['tol'] deep = self.__state__['deep'] return (self.__class__, (0, cache, keymap, ignore, tol, deep, True)) class inf_cache(object): """infinitely-growing (INF) cache decorator. This decorator memoizes a function's return value each time it is called. If called later with the same arguments, the cached value is returned, and not re-evaluated. This cache will grow without bound. To avoid memory issues, it is suggested to frequently dump and clear the cache. This decorator takes an integer tolerance 'tol', equal to the number of decimal places to which it will round off floats, and a bool 'deep' for whether the rounding on inputs will be 'shallow' or 'deep'. Note that rounding is not applied to the calculation of new results, but rather as a simple form of cache interpolation. For example, with tol=0 and a cached value for f(3.0), f(3.1) will lookup f(3.0) in the cache while f(3.6) will store a new value; however if tol=1, both f(3.1) and f(3.6) will store new values. maxsize = maximum cache size [fixed at maxsize=None] cache = storage hashmap (default is {}) keymap = cache key encoder (default is keymaps.hashmap(flat=True)) ignore = function argument names and indicies to 'ignore' (default is None) tol = integer tolerance for rounding (default is None) deep = boolean for rounding depth (default is False, i.e. 'shallow') purge = boolean for purge cache to archive at maxsize (fixed at False) If *keymap* is given, it will replace the hashing algorithm for generating cache keys. Several hashing algorithms are available in 'keymaps'. The default keymap requires arguments to the cached function to be hashable. If the keymap retains type information, then arguments of different types will be cached separately. For example, f(3.0) and f(3) will be treated as distinct calls with distinct results. Cache typing has a memory penalty, and may also be ignored by some 'keymaps'. If *ignore* is given, the keymap will ignore the arguments with the names and/or positional indicies provided. For example, if ignore=(0,), then the key generated for f(1,2) will be identical to that of f(3,2) or f(4,2). If ignore=('y',), then the key generated for f(x=3,y=4) will be identical to that of f(x=3,y=0) or f(x=3,y=10). If ignore=('*','**'), all varargs and varkwds will be 'ignored'. Ignored arguments never trigger a recalculation (they only trigger cache lookups), and thus are 'ignored'. When caching class methods, it may be useful to ignore=('self',). View cache statistics (hit, miss, load, maxsize, size) with f.info(). Clear the cache and statistics with f.clear(). Replace the cache archive with f.archive(obj). Load from the archive with f.load(), and dump from the cache to the archive with f.dump(). """ def __init__(self, maxsize=None, cache=None, keymap=None, ignore=None, tol=None, deep=False, purge=False): #if maxsize is not None: raise ValueError('maxsize cannot be set') maxsize = None #XXX: allow maxsize to be given but ignored ? purge = False #XXX: allow purge to be given but ignored ? if cache is None: cache = archive_dict() elif type(cache) is dict: cache = archive_dict(cache) if keymap is None: keymap = hashmap(flat=True) if ignore is None: ignore = tuple() if deep: rounded = deep_round else: rounded = simple_round #else: rounded = shallow_round #FIXME: slow @rounded(tol) def rounded_args(*args, **kwds): return (args, kwds) # set state self.__state__ = { 'maxsize': maxsize, 'cache': cache, 'keymap': keymap, 'ignore': ignore, 'roundargs': rounded_args, 'tol': tol, 'deep': deep, 'purge': purge, } return def __call__(self, user_function): #cache = dict() # mapping of args to results stats = [0, 0, 0] # make statistics updateable non-locally HIT, MISS, LOAD = 0, 1, 2 # names for the stats fields #_len = len # localize the global len() function #lock = RLock() # linkedlist updates aren't threadsafe maxsize = self.__state__['maxsize'] cache = self.__state__['cache'] keymap = self.__state__['keymap'] ignore = self.__state__['ignore'] rounded_args = self.__state__['roundargs'] def wrapper(*args, **kwds): _args, _kwds = rounded_args(*args, **kwds) _args, _kwds = _keygen(user_function, ignore, *_args, **_kwds) key = keymap(*_args, **_kwds) try: # get cache entry result = cache[key] stats[HIT] += 1 except KeyError: # if not in cache, look in archive if cache.archived(): cache.load(key) try: result = cache[key] stats[LOAD] += 1 except KeyError: # if not found, then compute result = user_function(*args, **kwds) cache[key] = result stats[MISS] += 1 return result def archive(obj): """Replace the cache archive""" if isinstance(obj, archive_dict): cache.archive = obj.archive else: cache.archive = obj def key(*args, **kwds): """Get the cache key for the given *args,**kwds""" _args, _kwds = rounded_args(*args, **kwds) _args, _kwds = _keygen(user_function, ignore, *_args, **_kwds) return keymap(*_args, **_kwds) def lookup(*args, **kwds): """Get the stored value for the given *args,**kwds""" _args, _kwds = rounded_args(*args, **kwds) _args, _kwds = _keygen(user_function, ignore, *_args, **_kwds) return cache[keymap(*_args, **_kwds)] def __get_cache(): """Get the cache""" return cache def __get_mask(): """Get the (ignore) mask""" return ignore def __get_keymap(): """Get the keymap""" return keymap def clear(keepstats=False): """Clear the cache and statistics""" cache.clear() if not keepstats: stats[:] = [0, 0, 0] def info(): """Report cache statistics""" return CacheInfo(stats[HIT], stats[MISS], stats[LOAD], maxsize, len(cache)) # interface wrapper.__wrapped__ = user_function #XXX: better is handle to key_function=keygen(ignore)(user_function) ? wrapper.info = info wrapper.clear = clear wrapper.load = cache.load wrapper.dump = cache.dump wrapper.archive = archive wrapper.archived = cache.archived wrapper.key = key wrapper.lookup = lookup wrapper.__cache__ = __get_cache wrapper.__mask__ = __get_mask wrapper.__map__ = __get_keymap #wrapper._queue = None #XXX return update_wrapper(wrapper, user_function) def __get__(self, obj, objtype): """support instance methods""" return partial(self.__call__, obj) def __reduce__(self): cache = self.__state__['cache'] keymap = self.__state__['keymap'] ignore = self.__state__['ignore'] tol = self.__state__['tol'] deep = self.__state__['deep'] return (self.__class__, (None, cache, keymap, ignore, tol, deep, False)) class lfu_cache(object): """least-frequenty-used (LFU) cache decorator. This decorator memoizes a function's return value each time it is called. If called later with the same arguments, the cached value is returned, and not re-evaluated. To avoid memory issues, a maximum cache size is imposed. For caches with an archive, the full cache dumps to archive upon reaching maxsize. For caches without an archive, the LFU algorithm manages the cache. Caches with an archive will use the latter behavior when 'purge' is False. This decorator takes an integer tolerance 'tol', equal to the number of decimal places to which it will round off floats, and a bool 'deep' for whether the rounding on inputs will be 'shallow' or 'deep'. Note that rounding is not applied to the calculation of new results, but rather as a simple form of cache interpolation. For example, with tol=0 and a cached value for f(3.0), f(3.1) will lookup f(3.0) in the cache while f(3.6) will store a new value; however if tol=1, both f(3.1) and f(3.6) will store new values. maxsize = maximum cache size cache = storage hashmap (default is {}) keymap = cache key encoder (default is keymaps.hashmap(flat=True)) ignore = function argument names and indicies to 'ignore' (default is None) tol = integer tolerance for rounding (default is None) deep = boolean for rounding depth (default is False, i.e. 'shallow') purge = boolean for purge cache to archive at maxsize (default is False) If *maxsize* is None, this cache will grow without bound. If *keymap* is given, it will replace the hashing algorithm for generating cache keys. Several hashing algorithms are available in 'keymaps'. The default keymap requires arguments to the cached function to be hashable. If the keymap retains type information, then arguments of different types will be cached separately. For example, f(3.0) and f(3) will be treated as distinct calls with distinct results. Cache typing has a memory penalty, and may also be ignored by some 'keymaps'. If *ignore* is given, the keymap will ignore the arguments with the names and/or positional indicies provided. For example, if ignore=(0,), then the key generated for f(1,2) will be identical to that of f(3,2) or f(4,2). If ignore=('y',), then the key generated for f(x=3,y=4) will be identical to that of f(x=3,y=0) or f(x=3,y=10). If ignore=('*','**'), all varargs and varkwds will be 'ignored'. Ignored arguments never trigger a recalculation (they only trigger cache lookups), and thus are 'ignored'. When caching class methods, it may be useful to ignore=('self',). View cache statistics (hit, miss, load, maxsize, size) with f.info(). Clear the cache and statistics with f.clear(). Replace the cache archive with f.archive(obj). Load from the archive with f.load(), and dump from the cache to the archive with f.dump(). See: http://en.wikipedia.org/wiki/Cache_algorithms#Least_Frequently_Used """ def __new__(cls, *args, **kwds): maxsize = kwds.get('maxsize', -1) if maxsize == 0: return no_cache(*args, **kwds) if maxsize is None: return inf_cache(*args, **kwds) return object.__new__(cls) def __init__(self, maxsize=100, cache=None, keymap=None, ignore=None, tol=None, deep=False, purge=False): if maxsize is None or maxsize == 0: return if cache is None: cache = archive_dict() elif type(cache) is dict: cache = archive_dict(cache) if keymap is None: keymap = hashmap(flat=True) if ignore is None: ignore = tuple() if deep: rounded = deep_round else: rounded = simple_round #else: rounded = shallow_round #FIXME: slow @rounded(tol) def rounded_args(*args, **kwds): return (args, kwds) # set state self.__state__ = { 'maxsize': maxsize, 'cache': cache, 'keymap': keymap, 'ignore': ignore, 'roundargs': rounded_args, 'tol': tol, 'deep': deep, 'purge': purge, } return def __call__(self, user_function): from heapq import nsmallest from operator import itemgetter #cache = dict() # mapping of args to results use_count = Counter() # times each key has been accessed stats = [0, 0, 0] # make statistics updateable non-locally HIT, MISS, LOAD = 0, 1, 2 # names for the stats fields _len = len # localize the global len() function #lock = RLock() # linkedlist updates aren't threadsafe maxsize = self.__state__['maxsize'] cache = self.__state__['cache'] keymap = self.__state__['keymap'] ignore = self.__state__['ignore'] rounded_args = self.__state__['roundargs'] purge = self.__state__['purge'] def wrapper(*args, **kwds): _args, _kwds = rounded_args(*args, **kwds) _args, _kwds = _keygen(user_function, ignore, *_args, **_kwds) key = keymap(*_args, **_kwds) try: # get cache entry result = cache[key] use_count[key] += 1 stats[HIT] += 1 except KeyError: # if not in cache, look in archive if cache.archived(): cache.load(key) try: result = cache[key] use_count[key] += 1 stats[LOAD] += 1 except KeyError: # if not found, then compute result = user_function(*args, **kwds) cache[key] = result use_count[key] += 1 stats[MISS] += 1 # purge cache if _len(cache) > maxsize: #XXX: better: if cache is cache.archive ? if cache.archived() and purge: cache.dump() cache.clear() use_count.clear() else: # purge least frequent cache entries for k, _ in nsmallest(max(2, maxsize // 10), iter(use_count.items()), key=itemgetter(1)): if cache.archived(): cache.dump(k) try: del cache[k] except KeyError: pass #FIXME: possible less purged use_count.pop(k, None) return result def archive(obj): """Replace the cache archive""" if isinstance(obj, archive_dict): cache.archive = obj.archive else: cache.archive = obj def key(*args, **kwds): """Get the cache key for the given *args,**kwds""" _args, _kwds = rounded_args(*args, **kwds) _args, _kwds = _keygen(user_function, ignore, *_args, **_kwds) return keymap(*_args, **_kwds) def lookup(*args, **kwds): """Get the stored value for the given *args,**kwds""" _args, _kwds = rounded_args(*args, **kwds) _args, _kwds = _keygen(user_function, ignore, *_args, **_kwds) return cache[keymap(*_args, **_kwds)] def __get_cache(): """Get the cache""" return cache def __get_mask(): """Get the (ignore) mask""" return ignore def __get_keymap(): """Get the keymap""" return keymap def clear(keepstats=False): """Clear the cache and statistics""" cache.clear() use_count.clear() if not keepstats: stats[:] = [0, 0, 0] def info(): """Report cache statistics""" return CacheInfo(stats[HIT], stats[MISS], stats[LOAD], maxsize, len(cache)) # interface wrapper.__wrapped__ = user_function #XXX: better is handle to key_function=keygen(ignore)(user_function) ? wrapper.info = info wrapper.clear = clear wrapper.load = cache.load wrapper.dump = cache.dump wrapper.archive = archive wrapper.archived = cache.archived wrapper.key = key wrapper.lookup = lookup wrapper.__cache__ = __get_cache wrapper.__mask__ = __get_mask wrapper.__map__ = __get_keymap #wrapper._queue = use_count #XXX return update_wrapper(wrapper, user_function) def __get__(self, obj, objtype): """support instance methods""" return partial(self.__call__, obj) def __reduce__(self): maxsize = self.__state__['maxsize'] cache = self.__state__['cache'] keymap = self.__state__['keymap'] ignore = self.__state__['ignore'] tol = self.__state__['tol'] deep = self.__state__['deep'] purge = self.__state__['purge'] return (self.__class__, (maxsize, cache, keymap, ignore, tol, deep, purge)) class lru_cache(object): """least-recently-used (LRU) cache decorator. This decorator memoizes a function's return value each time it is called. If called later with the same arguments, the cached value is returned, and not re-evaluated. To avoid memory issues, a maximum cache size is imposed. For caches with an archive, the full cache dumps to archive upon reaching maxsize. For caches without an archive, the LRU algorithm manages the cache. Caches with an archive will use the latter behavior when 'purge' is False. This decorator takes an integer tolerance 'tol', equal to the number of decimal places to which it will round off floats, and a bool 'deep' for whether the rounding on inputs will be 'shallow' or 'deep'. Note that rounding is not applied to the calculation of new results, but rather as a simple form of cache interpolation. For example, with tol=0 and a cached value for f(3.0), f(3.1) will lookup f(3.0) in the cache while f(3.6) will store a new value; however if tol=1, both f(3.1) and f(3.6) will store new values. maxsize = maximum cache size cache = storage hashmap (default is {}) keymap = cache key encoder (default is keymaps.hashmap(flat=True)) ignore = function argument names and indicies to 'ignore' (default is None) tol = integer tolerance for rounding (default is None) deep = boolean for rounding depth (default is False, i.e. 'shallow') purge = boolean for purge cache to archive at maxsize (default is False) If *maxsize* is None, this cache will grow without bound. If *keymap* is given, it will replace the hashing algorithm for generating cache keys. Several hashing algorithms are available in 'keymaps'. The default keymap requires arguments to the cached function to be hashable. If the keymap retains type information, then arguments of different types will be cached separately. For example, f(3.0) and f(3) will be treated as distinct calls with distinct results. Cache typing has a memory penalty, and may also be ignored by some 'keymaps'. If *ignore* is given, the keymap will ignore the arguments with the names and/or positional indicies provided. For example, if ignore=(0,), then the key generated for f(1,2) will be identical to that of f(3,2) or f(4,2). If ignore=('y',), then the key generated for f(x=3,y=4) will be identical to that of f(x=3,y=0) or f(x=3,y=10). If ignore=('*','**'), all varargs and varkwds will be 'ignored'. Ignored arguments never trigger a recalculation (they only trigger cache lookups), and thus are 'ignored'. When caching class methods, it may be useful to ignore=('self',). View cache statistics (hit, miss, load, maxsize, size) with f.info(). Clear the cache and statistics with f.clear(). Replace the cache archive with f.archive(obj). Load from the archive with f.load(), and dump from the cache to the archive with f.dump(). See: http://en.wikipedia.org/wiki/Cache_algorithms#Least_Recently_Used """ def __new__(cls, *args, **kwds): maxsize = kwds.get('maxsize', -1) if maxsize == 0: return no_cache(*args, **kwds) if maxsize is None: return inf_cache(*args, **kwds) return object.__new__(cls) def __init__(self, maxsize=100, cache=None, keymap=None, ignore=None, tol=None, deep=False, purge=False): if maxsize is None or maxsize == 0: return if cache is None: cache = archive_dict() elif type(cache) is dict: cache = archive_dict(cache) if keymap is None: keymap = hashmap(flat=True) if ignore is None: ignore = tuple() if deep: rounded = deep_round else: rounded = simple_round #else: rounded = shallow_round #FIXME: slow @rounded(tol) def rounded_args(*args, **kwds): return (args, kwds) # set state self.__state__ = { 'maxsize': maxsize, 'cache': cache, 'keymap': keymap, 'ignore': ignore, 'roundargs': rounded_args, 'tol': tol, 'deep': deep, 'purge': purge, } return def __call__(self, user_function): from collections import deque from itertools import filterfalse #cache = dict() # mapping of args to results queue = deque() # order that keys have been used refcount = Counter() # times each key is in the queue sentinel = object() # marker for looping around the queue stats = [0, 0, 0] # make statistics updateable non-locally HIT, MISS, LOAD = 0, 1, 2 # names for the stats fields _len = len # localize the global len() function #lock = RLock() # linkedlist updates aren't threadsafe maxsize = self.__state__['maxsize'] cache = self.__state__['cache'] keymap = self.__state__['keymap'] ignore = self.__state__['ignore'] rounded_args = self.__state__['roundargs'] purge = self.__state__['purge'] maxqueue = maxsize * 10 #XXX: settable? confirm this works as expected # lookup optimizations (ugly but fast) queue_append, queue_popleft = queue.append, queue.popleft queue_appendleft, queue_pop = queue.appendleft, queue.pop def wrapper(*args, **kwds): _args, _kwds = rounded_args(*args, **kwds) _args, _kwds = _keygen(user_function, ignore, *_args, **_kwds) key = keymap(*_args, **_kwds) try: # get cache entry result = cache[key] # record recent use of this key queue_append(key) refcount[key] += 1 stats[HIT] += 1 except KeyError: # if not in cache, look in archive if cache.archived(): cache.load(key) try: result = cache[key] # record recent use of this key queue_append(key) refcount[key] += 1 stats[LOAD] += 1 except KeyError: # if not found, then compute result = user_function(*args, **kwds) cache[key] = result # record recent use of this key queue_append(key) refcount[key] += 1 stats[MISS] += 1 # purge cache if _len(cache) > maxsize: #XXX: better: if cache is cache.archive ? if cache.archived() and purge: cache.dump() cache.clear() queue.clear() refcount.clear() else: # purge least recently used cache entry key = queue_popleft() refcount[key] -= 1 while refcount[key]: key = queue_popleft() refcount[key] -= 1 if cache.archived(): cache.dump(key) try: del cache[key] except KeyError: pass #FIXME: possible none purged refcount.pop(key, None) # periodically compact the queue by eliminating duplicate keys # while preserving order of most recent access if _len(queue) > maxqueue: refcount.clear() queue_appendleft(sentinel) for key in filterfalse(refcount.__contains__, iter(queue_pop, sentinel)): queue_appendleft(key) refcount[key] = 1 return result def archive(obj): """Replace the cache archive""" if isinstance(obj, archive_dict): cache.archive = obj.archive else: cache.archive = obj def key(*args, **kwds): """Get the cache key for the given *args,**kwds""" _args, _kwds = rounded_args(*args, **kwds) _args, _kwds = _keygen(user_function, ignore, *_args, **_kwds) return keymap(*_args, **_kwds) def lookup(*args, **kwds): """Get the stored value for the given *args,**kwds""" _args, _kwds = rounded_args(*args, **kwds) _args, _kwds = _keygen(user_function, ignore, *_args, **_kwds) return cache[keymap(*_args, **_kwds)] def __get_cache(): """Get the cache""" return cache def __get_mask(): """Get the (ignore) mask""" return ignore def __get_keymap(): """Get the keymap""" return keymap def clear(keepstats=False): """Clear the cache and statistics""" cache.clear() queue.clear() refcount.clear() if not keepstats: stats[:] = [0, 0, 0] def info(): """Report cache statistics""" return CacheInfo(stats[HIT], stats[MISS], stats[LOAD], maxsize, len(cache)) # interface wrapper.__wrapped__ = user_function #XXX: better is handle to key_function=keygen(ignore)(user_function) ? wrapper.info = info wrapper.clear = clear wrapper.load = cache.load wrapper.dump = cache.dump wrapper.archive = archive wrapper.archived = cache.archived wrapper.key = key wrapper.lookup = lookup wrapper.__cache__ = __get_cache wrapper.__mask__ = __get_mask wrapper.__map__ = __get_keymap #wrapper._queue = queue #XXX return update_wrapper(wrapper, user_function) def __get__(self, obj, objtype): """support instance methods""" return partial(self.__call__, obj) def __reduce__(self): maxsize = self.__state__['maxsize'] cache = self.__state__['cache'] keymap = self.__state__['keymap'] ignore = self.__state__['ignore'] tol = self.__state__['tol'] deep = self.__state__['deep'] purge = self.__state__['purge'] return (self.__class__, (maxsize, cache, keymap, ignore, tol, deep, purge)) class mru_cache(object): """most-recently-used (MRU) cache decorator. This decorator memoizes a function's return value each time it is called. If called later with the same arguments, the cached value is returned, and not re-evaluated. To avoid memory issues, a maximum cache size is imposed. For caches with an archive, the full cache dumps to archive upon reaching maxsize. For caches without an archive, the MRU algorithm manages the cache. Caches with an archive will use the latter behavior when 'purge' is False. This decorator takes an integer tolerance 'tol', equal to the number of decimal places to which it will round off floats, and a bool 'deep' for whether the rounding on inputs will be 'shallow' or 'deep'. Note that rounding is not applied to the calculation of new results, but rather as a simple form of cache interpolation. For example, with tol=0 and a cached value for f(3.0), f(3.1) will lookup f(3.0) in the cache while f(3.6) will store a new value; however if tol=1, both f(3.1) and f(3.6) will store new values. maxsize = maximum cache size cache = storage hashmap (default is {}) keymap = cache key encoder (default is keymaps.hashmap(flat=True)) ignore = function argument names and indicies to 'ignore' (default is None) tol = integer tolerance for rounding (default is None) deep = boolean for rounding depth (default is False, i.e. 'shallow') purge = boolean for purge cache to archive at maxsize (default is False) If *maxsize* is None, this cache will grow without bound. If *keymap* is given, it will replace the hashing algorithm for generating cache keys. Several hashing algorithms are available in 'keymaps'. The default keymap requires arguments to the cached function to be hashable. If the keymap retains type information, then arguments of different types will be cached separately. For example, f(3.0) and f(3) will be treated as distinct calls with distinct results. Cache typing has a memory penalty, and may also be ignored by some 'keymaps'. If *ignore* is given, the keymap will ignore the arguments with the names and/or positional indicies provided. For example, if ignore=(0,), then the key generated for f(1,2) will be identical to that of f(3,2) or f(4,2). If ignore=('y',), then the key generated for f(x=3,y=4) will be identical to that of f(x=3,y=0) or f(x=3,y=10). If ignore=('*','**'), all varargs and varkwds will be 'ignored'. Ignored arguments never trigger a recalculation (they only trigger cache lookups), and thus are 'ignored'. When caching class methods, it may be useful to ignore=('self',). View cache statistics (hit, miss, load, maxsize, size) with f.info(). Clear the cache and statistics with f.clear(). Replace the cache archive with f.archive(obj). Load from the archive with f.load(), and dump from the cache to the archive with f.dump(). See: http://en.wikipedia.org/wiki/Cache_algorithms#Most_Recently_Used """ def __new__(cls, *args, **kwds): maxsize = kwds.get('maxsize', -1) if maxsize == 0: return no_cache(*args, **kwds) if maxsize is None: return inf_cache(*args, **kwds) return object.__new__(cls) def __init__(self, maxsize=100, cache=None, keymap=None, ignore=None, tol=None, deep=False, purge=False): if maxsize is None or maxsize == 0: return if cache is None: cache = archive_dict() elif type(cache) is dict: cache = archive_dict(cache) if keymap is None: keymap = hashmap(flat=True) if ignore is None: ignore = tuple() if deep: rounded = deep_round else: rounded = simple_round #else: rounded = shallow_round #FIXME: slow @rounded(tol) def rounded_args(*args, **kwds): return (args, kwds) # set state self.__state__ = { 'maxsize': maxsize, 'cache': cache, 'keymap': keymap, 'ignore': ignore, 'roundargs': rounded_args, 'tol': tol, 'deep': deep, 'purge': purge, } return def __call__(self, user_function): from collections import deque #cache = dict() # mapping of args to results queue = deque() # order that keys have been used stats = [0, 0, 0] # make statistics updateable non-locally HIT, MISS, LOAD = 0, 1, 2 # names for the stats fields _len = len # localize the global len() function #lock = RLock() # linkedlist updates aren't threadsafe maxsize = self.__state__['maxsize'] cache = self.__state__['cache'] keymap = self.__state__['keymap'] ignore = self.__state__['ignore'] rounded_args = self.__state__['roundargs'] purge = self.__state__['purge'] # lookup optimizations (ugly but fast) queue_append, queue_popleft = queue.append, queue.popleft queue_appendleft, queue_pop = queue.appendleft, queue.pop def wrapper(*args, **kwds): _args, _kwds = rounded_args(*args, **kwds) _args, _kwds = _keygen(user_function, ignore, *_args, **_kwds) key = keymap(*_args, **_kwds) try: # get cache entry result = cache[key] try: queue.remove(key) except ValueError: pass stats[HIT] += 1 except KeyError: # if not in cache, look in archive if cache.archived(): cache.load(key) try: result = cache[key] stats[LOAD] += 1 except KeyError: # if not found, then compute result = user_function(*args, **kwds) cache[key] = result stats[MISS] += 1 # purge cache if _len(cache) > maxsize: #XXX: better: if cache is cache.archive ? if cache.archived() and purge: cache.dump() cache.clear() queue.clear() else: # purge most recently used cache entry k = queue_pop() if cache.archived(): cache.dump(k) try: del cache[k] except KeyError: pass #FIXME: possible none purged # record recent use of this key queue_append(key) return result def archive(obj): """Replace the cache archive""" if isinstance(obj, archive_dict): cache.archive = obj.archive else: cache.archive = obj def key(*args, **kwds): """Get the cache key for the given *args,**kwds""" _args, _kwds = rounded_args(*args, **kwds) _args, _kwds = _keygen(user_function, ignore, *_args, **_kwds) return keymap(*_args, **_kwds) def lookup(*args, **kwds): """Get the stored value for the given *args,**kwds""" _args, _kwds = rounded_args(*args, **kwds) _args, _kwds = _keygen(user_function, ignore, *_args, **_kwds) return cache[keymap(*_args, **_kwds)] def __get_cache(): """Get the cache""" return cache def __get_mask(): """Get the (ignore) mask""" return ignore def __get_keymap(): """Get the keymap""" return keymap def clear(keepstats=False): """Clear the cache and statistics""" cache.clear() queue.clear() if not keepstats: stats[:] = [0, 0, 0] def info(): """Report cache statistics""" return CacheInfo(stats[HIT], stats[MISS], stats[LOAD], maxsize, len(cache)) # interface wrapper.__wrapped__ = user_function #XXX: better is handle to key_function=keygen(ignore)(user_function) ? wrapper.info = info wrapper.clear = clear wrapper.load = cache.load wrapper.dump = cache.dump wrapper.archive = archive wrapper.archived = cache.archived wrapper.key = key wrapper.lookup = lookup wrapper.__cache__ = __get_cache wrapper.__mask__ = __get_mask wrapper.__map__ = __get_keymap #wrapper._queue = queue #XXX return update_wrapper(wrapper, user_function) def __get__(self, obj, objtype): """support instance methods""" return partial(self.__call__, obj) def __reduce__(self): maxsize = self.__state__['maxsize'] cache = self.__state__['cache'] keymap = self.__state__['keymap'] ignore = self.__state__['ignore'] tol = self.__state__['tol'] deep = self.__state__['deep'] purge = self.__state__['purge'] return (self.__class__, (maxsize, cache, keymap, ignore, tol, deep, purge)) class rr_cache(object): """random-replacement (RR) cache decorator. This decorator memoizes a function's return value each time it is called. If called later with the same arguments, the cached value is returned, and not re-evaluated. To avoid memory issues, a maximum cache size is imposed. For caches with an archive, the full cache dumps to archive upon reaching maxsize. For caches without an archive, the RR algorithm manages the cache. Caches with an archive will use the latter behavior when 'purge' is False. This decorator takes an integer tolerance 'tol', equal to the number of decimal places to which it will round off floats, and a bool 'deep' for whether the rounding on inputs will be 'shallow' or 'deep'. Note that rounding is not applied to the calculation of new results, but rather as a simple form of cache interpolation. For example, with tol=0 and a cached value for f(3.0), f(3.1) will lookup f(3.0) in the cache while f(3.6) will store a new value; however if tol=1, both f(3.1) and f(3.6) will store new values. maxsize = maximum cache size cache = storage hashmap (default is {}) keymap = cache key encoder (default is keymaps.hashmap(flat=True)) ignore = function argument names and indicies to 'ignore' (default is None) tol = integer tolerance for rounding (default is None) deep = boolean for rounding depth (default is False, i.e. 'shallow') purge = boolean for purge cache to archive at maxsize (default is False) If *maxsize* is None, this cache will grow without bound. If *keymap* is given, it will replace the hashing algorithm for generating cache keys. Several hashing algorithms are available in 'keymaps'. The default keymap requires arguments to the cached function to be hashable. If the keymap retains type information, then arguments of different types will be cached separately. For example, f(3.0) and f(3) will be treated as distinct calls with distinct results. Cache typing has a memory penalty, and may also be ignored by some 'keymaps'. If *ignore* is given, the keymap will ignore the arguments with the names and/or positional indicies provided. For example, if ignore=(0,), then the key generated for f(1,2) will be identical to that of f(3,2) or f(4,2). If ignore=('y',), then the key generated for f(x=3,y=4) will be identical to that of f(x=3,y=0) or f(x=3,y=10). If ignore=('*','**'), all varargs and varkwds will be 'ignored'. Ignored arguments never trigger a recalculation (they only trigger cache lookups), and thus are 'ignored'. When caching class methods, it may be useful to ignore=('self',). View cache statistics (hit, miss, load, maxsize, size) with f.info(). Clear the cache and statistics with f.clear(). Replace the cache archive with f.archive(obj). Load from the archive with f.load(), and dump from the cache to the archive with f.dump(). http://en.wikipedia.org/wiki/Cache_algorithms#Random_Replacement """ def __new__(cls, *args, **kwds): maxsize = kwds.get('maxsize', -1) if maxsize == 0: return no_cache(*args, **kwds) if maxsize is None: return inf_cache(*args, **kwds) return object.__new__(cls) def __init__(self, maxsize=100, cache=None, keymap=None, ignore=None, tol=None, deep=False, purge=False): if maxsize is None or maxsize == 0: return if cache is None: cache = archive_dict() elif type(cache) is dict: cache = archive_dict(cache) if keymap is None: keymap = hashmap(flat=True) if ignore is None: ignore = tuple() if deep: rounded = deep_round else: rounded = simple_round #else: rounded = shallow_round #FIXME: slow @rounded(tol) def rounded_args(*args, **kwds): return (args, kwds) # set state self.__state__ = { 'maxsize': maxsize, 'cache': cache, 'keymap': keymap, 'ignore': ignore, 'roundargs': rounded_args, 'tol': tol, 'deep': deep, 'purge': purge, } return def __call__(self, user_function): #cache = dict() # mapping of args to results stats = [0, 0, 0] # make statistics updateable non-locally HIT, MISS, LOAD = 0, 1, 2 # names for the stats fields _len = len # localize the global len() function #lock = RLock() # linkedlist updates aren't threadsafe maxsize = self.__state__['maxsize'] cache = self.__state__['cache'] keymap = self.__state__['keymap'] ignore = self.__state__['ignore'] rounded_args = self.__state__['roundargs'] purge = self.__state__['purge'] def wrapper(*args, **kwds): from random import choice #XXX: biased? _args, _kwds = rounded_args(*args, **kwds) _args, _kwds = _keygen(user_function, ignore, *_args, **_kwds) key = keymap(*_args, **_kwds) try: # get cache entry result = cache[key] stats[HIT] += 1 except KeyError: # if not in cache, look in archive if cache.archived(): cache.load(key) try: result = cache[key] stats[LOAD] += 1 except KeyError: # if not found, then compute result = user_function(*args, **kwds) cache[key] = result stats[MISS] += 1 # purge cache if _len(cache) > maxsize: #XXX: better: if cache is cache.archive ? if cache.archived() and purge: cache.dump() cache.clear() else: # purge random cache entry key = choice(list(cache.keys())) if cache.archived(): cache.dump(key) try: del cache[key] except KeyError: pass #FIXME: possible none purged return result def archive(obj): """Replace the cache archive""" if isinstance(obj, archive_dict): cache.archive = obj.archive else: cache.archive = obj def key(*args, **kwds): """Get the cache key for the given *args,**kwds""" _args, _kwds = rounded_args(*args, **kwds) _args, _kwds = _keygen(user_function, ignore, *_args, **_kwds) return keymap(*_args, **_kwds) def lookup(*args, **kwds): """Get the stored value for the given *args,**kwds""" _args, _kwds = rounded_args(*args, **kwds) _args, _kwds = _keygen(user_function, ignore, *_args, **_kwds) return cache[keymap(*_args, **_kwds)] def __get_cache(): """Get the cache""" return cache def __get_mask(): """Get the (ignore) mask""" return ignore def __get_keymap(): """Get the keymap""" return keymap def clear(keepstats=False): """Clear the cache and statistics""" cache.clear() if not keepstats: stats[:] = [0, 0, 0] def info(): """Report cache statistics""" return CacheInfo(stats[HIT], stats[MISS], stats[LOAD], maxsize, len(cache)) # interface wrapper.__wrapped__ = user_function #XXX: better is handle to key_function=keygen(ignore)(user_function) ? wrapper.info = info wrapper.clear = clear wrapper.load = cache.load wrapper.dump = cache.dump wrapper.archive = archive wrapper.archived = cache.archived wrapper.key = key wrapper.lookup = lookup wrapper.__cache__ = __get_cache wrapper.__mask__ = __get_mask wrapper.__map__ = __get_keymap #wrapper._queue = None #XXX return update_wrapper(wrapper, user_function) def __get__(self, obj, objtype): """support instance methods""" return partial(self.__call__, obj) def __reduce__(self): maxsize = self.__state__['maxsize'] cache = self.__state__['cache'] keymap = self.__state__['keymap'] ignore = self.__state__['ignore'] tol = self.__state__['tol'] deep = self.__state__['deep'] purge = self.__state__['purge'] return (self.__class__, (maxsize, cache, keymap, ignore, tol, deep, purge)) if __name__ == '__main__': import dill #@no_cache(10, tol=0) #@inf_cache(10, tol=0) #@lfu_cache(10, tol=0) #@lru_cache(10, tol=0) #@mru_cache(10, tol=0) @rr_cache(10, tol=0) def squared(x): return x**2 res = squared(10) assert res == dill.loads(dill.dumps(squared))(10) # EOF uqfoundation-klepto-69cd6ce/klepto/_inspect.py000066400000000000000000000503251455531556400217250ustar00rootroot00000000000000#!/usr/bin/env python # # Author: Mike McKerns (mmckerns @caltech and @uqfoundation) # Copyright (c) 2013-2016 California Institute of Technology. # Copyright (c) 2016-2024 The Uncertainty Quantification Foundation. # License: 3-clause BSD. The full license text is available at: # - https://github.com/uqfoundation/klepto/blob/master/LICENSE #FIXME: klepto's caches ignore names/index, however ignore should be in keymap import inspect from klepto.tools import IS_PYPY def signature(func, variadic=True, markup=True, safe=False): """get the input signature of a function func: the function to inspect variadic: if True, also return names of (*args, **kwds) used in func markup: if True, show a "!" before any 'unsettable' parameters safe: if True, return (None,None,None,None) instead of throwing an error Returns a tuple of variable names and a dict of keywords with defaults. If variadic=True, additionally return names of func's (*args, **kwds). Python functions, methods, lambdas, and partials can be inspected. If safe=False, non-python functions (e.g. builtins) will raise an error. For partials, 'fixed' args correspond to positional arguments given in when the partial was defined. Partials have 'unsettalble' parameters, where, these parameters may be given as input but will throw errors. If markup=True, 'unsettable' parameters are denoted by a prepended '!'. For example: >>> def bar(x,y,z,a=1,b=2,*args): ... return x+y+z+a+b ... >>> signature(bar) (('x', 'y', 'z', 'a', 'b'), {'a': 1, 'b': 2}, 'args', '') >>> >>> # a partial with a 'fixed' x, thus x is 'unsettable' as a keyword >>> p = partial(bar, 0) >>> signature(p) (('y', 'z', 'a', 'b'), {'a': 1, '!x': 0, 'b': 2}, 'args', '') >>> p(0,1) 4 >>> p(0,1,2,3,4,5) 6 >>> >>> # a partial where y is 'unsettable' as a positional argument >>> p = partial(bar, y=10) >>> signature(p) (('x', '!y', 'z', 'a', 'b'), {'a': 1, 'y': 10, 'b': 2}, 'args', '') >>> p(0,1,2) Traceback (most recent call last): File "", line 1, in TypeError: bar() got multiple values for keyword argument 'y' >>> p(0,z=2) 15 >>> p(0,y=1,z=2) 6 >>> >>> # a partial with a 'fixed' x, and positionally 'unsettable' b >>> p = partial(bar, 0,b=10) >>> signature(p) (('y', 'z', 'a', '!b'), {'a': 1, '!x': 0, 'b': 10}, 'args', '') >>> >>> # apply some options that reduce information content >>> signature(p, markup=False) (('y', 'z', 'a', 'b'), {'a': 1, 'b': 10}, 'args', '') >>> signature(p, markup=False, variadic=False) (('y', 'z', 'a', 'b'), {'a': 1, 'b': 10}) """ TINY_FAIL = None,None #XXX: or (),{} ? LONG_FAIL = None,None,None,None #XXX: or (),{},'','' ? if safe and inspect.isbuiltin(func) and not IS_PYPY: return LONG_FAIL if variadic else TINY_FAIL #"""fixed: if True, include any 'fixed' args in returned keywords""" # maybe it's less confusing to tie 'fixed' to 'markup'... so do that. fixed = markup identified = False if not inspect.ismethod(func) and not inspect.isfunction(func): try: # then it could be a partial... p_args = func.args # list of default arg values p_kwds = func.keywords or {} # dict of default kwd values func = func.func identified = True except AttributeError: if hasattr(func, '__call__') and not hasattr(func, '__name__'): func = func.__call__ # treat callable instance as __call__ else: #XXX: anything else to try? No? Give up. pass if not identified: p_args = () p_kwds = {} FULL_ARGS = hasattr(inspect, 'getfullargspec') try: if FULL_ARGS: arg_spec = inspect.getfullargspec(func) else: arg_spec = inspect.getargspec(func) except TypeError: if safe: return LONG_FAIL if variadic else TINY_FAIL raise TypeError('%r is not a Python function' % func) if hasattr(arg_spec, 'args'): arg_names = arg_spec.args # list of input variable names arg_defaults = arg_spec.defaults # list of kwd default values arg_varargs = arg_spec.varargs # name of *args if FULL_ARGS: # name of **kwds arg_keywords = getattr(arg_spec, 'varkw') or {} arg_kwdefault = getattr(arg_spec, 'kwonlydefaults') or {} else: arg_keywords = arg_spec.keywords arg_kwdefault = {} else: arg_names, arg_varargs, arg_keywords, arg_defaults = arg_spec arg_kwdefault = {} if not arg_defaults or not arg_names: defaults = {} explicit = tuple(arg_names) or () else: defaults = dict(zip(arg_names[-len(arg_defaults):],arg_defaults)) explicit = tuple(arg_names) or () # always return all names #explicit = tuple(arg_names[:-len(arg_defaults)]) # only return args # for a partial, the first p_args are now at fixed values _fixed = dict(zip(arg_names[:len(p_args)],p_args)) # deal with the stupid case that the partial always fails errors = [i for i in _fixed if i in p_kwds] if errors: if safe: return LONG_FAIL if variadic else TINY_FAIL raise TypeError("%s() got multiple values for keyword argument '%s'" % (func.__name__,errors[0])) # the above could fail if taking a partial of a partial # include any keyword-only defaults defaults.update(arg_kwdefault) # for a partial, arguments given in p_kwds have new defaults defaults.update(p_kwds) if markup: X = '!' else: X = '' # remove args 'fixed' by the partial; prepend 'unsettable' args with '!' explicit = tuple(X+i if i in p_kwds else i for i in explicit \ if i not in _fixed) if fixed: #defaults.update(_fixed) defaults = dict((k,v) for (k,v) in defaults.items() if k not in _fixed) defaults.update(dict((X+k,v) for (k,v) in _fixed.items())) if inspect.ismethod(func) and func.__self__: # then it's a bound method explicit = explicit[1:] #XXX: correct to remove 'self' ? if variadic: varargs = arg_varargs or '' varkwds = arg_keywords or '' return explicit, defaults, varargs, varkwds return explicit, defaults import sys def isvalid(func, *args, **kwds): """check if func(*args,**kwds) is a valid call for function 'func' returns True if valid, returns False if an error is thrown""" try: validate(func, *args, **kwds) return True except: error = sys.exc_info()[1] # check for the special case of builtins, etc if str(error).endswith('is not a Python function'): #return None # None distinguishes from False, as "I don't know" try: # probably inexpensive, so just try evaluating it... (yikes?) func(*args, **kwds) return True except: pass return False def validate(func, *args, **kwds): """validate a function's arguments and keywords against the call signature Raises an exception when args and kwds do not match the call signature. If valid args and kwds are provided, "None" is returned. NOTE: 'validate' does not call f(*args,**kwds), instead checks *args,**kwds against the call signature of func. Thus, 'validate' will fail when called to inspect builtins and other non-python functions.""" named, defaults, hasargs, haskwds = signature(func) # if it's a partial, set func = func.func identified = False if not inspect.ismethod(func) and not inspect.isfunction(func): try: # then it could be a partial... p_args = func.args # list of default arg values p_kwds = func.keywords or {} # dict of default kwd values p_named,p_defaults = signature(func.func, markup=False, variadic=False) func = func.func p_required = set(p_named) - set(p_defaults) identified = True except AttributeError: if hasattr(func, '__call__') and not hasattr(func, '__name__'): func = func.__call__ # treat callable instance as __call__ else: #XXX: anything else to try? No? Give up. pass if not identified: p_args = p_named = () p_kwds = p_defaults = {} p_required = set() # get bad args/kwds from markup bad_args = set(i.strip('!') for i in named if i.startswith('!')) bad_kwds = set(i.strip('!') for i in defaults if i.startswith('!')) # strip markup named, defaults = strip_markup(named, defaults) # FAIL if partial built for **kwds, but **kwds not used in func.func p_varkwds = set(p_kwds) - bad_kwds - bad_args if p_varkwds and not haskwds: raise TypeError("%s() got an unexpected keyword argument '%s'" % (func.__name__,p_varkwds.pop())) # FAIL if partial built for *args, but *args not used in func.func p_varargs = max(0, len(p_args) - len(p_required)) if p_varargs and not hasargs: raise TypeError("%s() takes at most %d arguments (%d given)" % (func.__name__, len(p_named), len(p_args)+len(args)+len(kwds))) # get any varargs; FAIL if func doesn't take varargs var_args = args[len(named):] if var_args and not hasargs: var_kwds = set(kwds) - set(named) raise TypeError("%s() takes at most %d arguments (%d given)" % (func.__name__, len(named)+len(p_args), len(p_args)+len(args)+len(kwds))) # check any varkwds; FAIL if func doesn't take varkwds var_kwds = set(kwds) - set(named) if var_kwds and not haskwds: raise TypeError("%s() got an unexpected keyword argument '%s'" % (func.__name__,var_kwds.pop())) # get user_args as a dict args_kwds = dict(zip(named,args)) # check if user has given one of the bad_args or bad_kwds bad_args = bad_args.intersection(args_kwds) if bad_args: raise TypeError("%s() got multiple values for keyword argument '%s'" % (func.__name__, bad_args.pop())) bad_kwds = bad_kwds.intersection(kwds) if bad_kwds: raise TypeError("%s() got multiple values for keyword argument '%s'" % (func.__name__, bad_kwds.pop())) # check for duplicates; FAIL if anything is defined twice duplicates = set(args_kwds).intersection(kwds) if duplicates: raise TypeError("%s() got multiple values for keyword argument '%s'" % (func.__name__, duplicates.pop())) # get names of required args required = set(named) - set(defaults) # mixin defaults defaults.update(kwds) # now there are no duplicates, build a dict of all known names/values defaults.update(args_kwds) # check if all required are provided; FAIL if any required are missing provided = required.intersection(defaults) _required = len(required) if len(provided) < _required: p_bad = len(p_args) + len(set(named).intersection(p_kwds)) #p_bad = len(bad_args) + len(bad_kwds) _required = max(len(p_required), _required) provided = len(provided) + p_bad raise TypeError("%s() takes at least %d arguments (%d given)" % (func.__name__, _required, provided)) # if you are here, then success! return None #XXX: better return (args, kwds.copy()) ? from klepto.keymaps import keymap as kleptokeymap from klepto.rounding import simple_round, deep_round def keygen(*ignored, **kwds): """decorator for generating a cache key for a given function ignored: names and/or indicies of the function's arguments to 'ignore' tol: integer tolerance for rounding (default is None) deep: boolean for rounding depth (default is False, i.e. 'shallow') The decorator accepts integers (for the index of positional args to ignore, or strings (the names of the kwds to ignore). A cache key is returned, built with the registered keymap. Ignored arguments are stored in the keymap with a value of klepto.NULL. Note that for class methods, it may be useful to ignore 'self'. The decorated function will gain new methods for working with cache keys - call: __call__ the function with the most recently provided arguments - valid: True if the most recently provided arguments are valid - key: get the cache key for the most recently provided arguments - keymap: get the registered keymap [default: klepto.keymaps.keymap] - register: register a new keymap The function is not evaluated until the 'call' method is called. Both generating the key and checking for validity avoid calling the function by inspecting the function's input signature. The default keymap is keymaps.keymap(flat=True). Alternate keymaps can be set with the 'register' method on the decorated function.""" # returns (*varargs, **kwds) where all info in kwds except varargs # however, special cases (builtins, etc) return (*args, **kwds) _map = kwds.get('keymap', None) if _map is None: _map = kleptokeymap() tol = kwds.get('tol', None) deep = kwds.get('deep', False) if deep: rounded = deep_round else: rounded = simple_round # enable rounding @rounded(tol) def rounded_args(*args, **kwds): return (args, kwds) def dec(f): _args = [(),{}] _keymap = [_map] #[kleptokeymap()] def last_args(): "get the most recently provided (*args, **kwds)" return _args[0],_args[1] def func(*args, **kwds): _args[0] = args _args[1] = kwds _map = _keymap[0] args,kwds = rounded_args(*args, **kwds) args,kwds = _keygen(f, ignored, *args, **kwds) return _map(*args, **kwds) def call(): "call func with the most recently provided (*args, **kwds)" ar,kw = last_args() return f(*ar,**kw) def valid(): "check if func(*args, **kwds) is valid (without calling the function)" ar,kw = last_args() return isvalid(f,*ar,**kw) #XXX: better validate? (raises errors) def key(): "get cache 'key' for most recently provided (*args, **kwds)" ar,kw = last_args() _map = _keymap[0] ar,kw = rounded_args(*ar, **kw) ar,kw = _keygen(f, ignored, *ar, **kw) #XXX: better lookup saved key? return _map(*ar, **kw) def register(mapper): "register a new keymap instance" if isinstance(mapper, type): mapper = mapper() if not isinstance(mapper, kleptokeymap): raise TypeError("'%s' is not a klepto keymap instance" % getattr(mapper,'__name__',mapper)) _keymap[0] = mapper return def keymap(): "get the registered keymap instance" return _keymap[0] func.__ignored__ = ignored func.__func__ = f func.__args__ = last_args func.call = call func.valid = valid func.key = key func.keymap = keymap func.register = register return func return dec ################################### def strip_markup(names, defaults): """strip markup ('!') from function argument names and defaults""" names = tuple(i.strip('!') for i in names) defaults = dict((k,v) for (k,v) in defaults.items() if not k.startswith('!')) return names, defaults class _Null(object): """build a stub object for the NULL singleton""" def __repr__(self): return "NULL" NULL = _Null() #try: # from collections import OrderedDict as odict #except ImportError: # #XXX: adds a new dependency # from ordereddict import OrderedDict as odict from copy import copy def _keygen(func, ignored, *args, **kwds): """generate a 'key' from the (*args,**kwds) suitable for use in caching func is the function being called ignored is the list of names and/or indicies to ignore args and kwds are func's input arguments and keywords returns the archive 'key' -- does not call the function ignored can include names (e.g. 'x','y'), indicies (e.g. 0,1), or '*','**'. if '*' in ignored, all varargs are ignored. Similarly for '**' and varkwds.` Note that for class methods, it may be useful to ignore 'self'. """ # hard-wire cross-populate names and indicies to True # crossref = True # hard-wire discover and apply function defaults to True defaults = True # hard-wire that keygen is 'safe' (doesn't throw errors from signature) safe = True # get variable names and defaults from func signature explicitly_named,user_kwds = signature(func,markup=False,variadic=False, safe=safe) # start off with user_args as the user provided args user_args = copy(args) # if safe and signature failed, return unmolested *args, **kwds if explicitly_named is None and user_kwds is None: return user_args, kwds.copy() # mix-in the function's defaults to the user provided kwds if defaults: user_kwds.update(kwds) else: # don't apply the function defaults (why, you wouldn't, I don't know) user_kwds = kwds.copy() # decompose the list of things to ignore to names and indicies if isinstance(ignored, (str,int)): ignored = [ignored] index_to_ignore = set(i for i in ignored if isinstance(i,int)) names_to_ignore = set(i for i in ignored if isinstance(i,str)) # if ignore self, remove self instead of NULL it if inspect.isfunction(func): try: # this is a pretty good filter that: user_args[0] is self _bound = getattr(user_args[0], func.__name__) _self = getattr(_bound, '__self__') assert _self == user_args[0] except: _bound = None if _bound and explicitly_named[0] in ignored: user_args = user_args[1:] # remove 'self' instance user_kwds.pop(explicitly_named[0], None) #XXX: unnecessary? explicitly_named = explicitly_named[1:] # remove 'self' name #XXX: hopefully, this doesn't mess up arg counting and other stuff # remove markers for ignoring all varagrs and all varkwds varargs_to_ignore = '*' in names_to_ignore varkwds_to_ignore = '**' in names_to_ignore names_to_ignore -= set(['*','**']) # var_index_to_ignore = {i for i in index_to_ignore if i >= len(explicitly_named)} # fix_index_to_ignore = index_to_ignore - var_index_to_ignore # fix_names_to_ignore = {i for i in names_to_ignore if i in explicitly_named} # var_names_to_ignore = names_to_ignore - fix_names_to_ignore - set(['*','**']) # cross-populate names_to_ignore and index_to_ignore for explicitly_named names_index = dict(enumerate(explicitly_named)) _index = set(i for (i,k) in names_index.items() if k in names_to_ignore) _names = set(k for (i,k) in names_index.items() if i in index_to_ignore) names_to_ignore = names_to_ignore.union(_names) index_to_ignore = index_to_ignore.union(_index) # NULL out the ignored args (and also drop not in user_args) #XXX: better if user_args always include NAMES/INDEX from ignored? NO. user_args = tuple(NULL if i in index_to_ignore else k for i,k in enumerate(user_args)) # if ignoring *args, clip off all args that are varargs if varargs_to_ignore: user_args = user_args[:len(explicitly_named)] # NULL out the ignored kwds (also drop not in user_kwds + explicitly_named) #XXX: better if user_kwds always include NAMES/INDEX from ignored? MAYBE. #user_kwds.update(dict([(k,NULL) for k in names_to_ignore])) #(see above) _keys = tuple(user_kwds.keys()) + explicitly_named user_kwds.update(dict([(k,NULL) for k in names_to_ignore if k in _keys])) # if ignoring **kwds, then pop all not in explicitly_named if varkwds_to_ignore: [user_kwds.pop(k) for k in kwds if k not in explicitly_named] # NULL out args that are NULL'ed as kwds, and vice-versa # if crossref: # inputs = odict(zip(explicitly_named,user_args)) # vararg = user_args[len(explicitly_named):] # user_kwds.update([(k,v) for (k,v) in inputs.items() if v == NULL and k in user_kwds]) # inputs.update(dict((k,v) for (k,v) in user_kwds.items() if v == NULL and k in inputs)) # user_args = tuple(inputs.values()) + vararg # transfer all from user_args to user_kwds, except for any varargs user_kwds.update(dict(zip(explicitly_named,user_args))) #XXX: if double-defined, prefer value in args #user_kwds.update(dict((k,v) for (k,v) in zip(explicitly_named,user_args) if k not in user_kwds)) #XXX: if double-defined, prefer value in kwds user_args = user_args[len(explicitly_named):] return user_args, user_kwds # EOF uqfoundation-klepto-69cd6ce/klepto/_pickle.py000066400000000000000000000367431455531556400215370ustar00rootroot00000000000000#!/usr/bin/env python # # Author: Gael Varoquaux # Copyright (c) 2009 Gael Varoquaux # License: BSD Style, 3 clauses. # Forked by: Mike McKerns (December 2013) # Author: Mike McKerns (mmckerns @caltech and @uqfoundation) # Copyright (c) 2013-2016 California Institute of Technology. # Copyright (c) 2016-2024 The Uncertainty Quantification Foundation. # License: 3-clause BSD. The full license text is available at: # - https://github.com/uqfoundation/klepto/blob/master/LICENSE """ Utilities for fast persistence of big data, with optional compression. """ import traceback import os import zlib import warnings import dill # add some dill magic to pickle import pickle try: _basestring = basestring except NameError: _basestring = str from io import BytesIO Unpickler = pickle._Unpickler Pickler = pickle._Pickler def asbytes(s): if isinstance(s, bytes): return s return s.encode('latin1') _MEGA = 2 ** 20 _MAX_LEN = len(hex(2 ** 64)) # To detect file types _ZFILE_PREFIX = asbytes('ZF') ############################################################################### # Compressed file with Zlib def _read_magic(file_handle): """ Utility to check the magic signature of a file identifying it as a Zfile """ magic = file_handle.read(len(_ZFILE_PREFIX)) # Pickling needs file-handles at the beginning of the file file_handle.seek(0) return magic def read_zfile(file_handle): """Read the z-file and return the content as a string Z-files are raw data compressed with zlib used internally for persistence. Backward compatibility is not guaranteed. Do not use for external purposes. """ file_handle.seek(0) assert _read_magic(file_handle) == _ZFILE_PREFIX, \ "File does not have the right magic" length = file_handle.read(len(_ZFILE_PREFIX) + _MAX_LEN) length = length[len(_ZFILE_PREFIX):] length = int(length, 16) # We use the known length of the data to tell Zlib the size of the # buffer to allocate. data = zlib.decompress(file_handle.read(), 15, length) assert len(data) == length, ( "Incorrect data length while decompressing %s." "The file could be corrupted." % file_handle) return data def write_zfile(file_handle, data, compress=1): """Write the data in the given file as a Z-file. Z-files are raw data compressed with zlib used internally for persistence. Backward compatibility is not guarantied. Do not use for external purposes. """ file_handle.write(_ZFILE_PREFIX) length = hex(len(data)) # Store the length of the data file_handle.write(asbytes(length.ljust(_MAX_LEN))) file_handle.write(zlib.compress(asbytes(data), compress)) ############################################################################### # Utility objects for persistence. class NDArrayWrapper(object): """ An object to be persisted instead of numpy arrays. The only thing this object does, is to carry the filename in which the array has been persisted, and the array subclass. """ def __init__(self, filename, subclass): "Store the useful information for later" self.filename = filename self.subclass = subclass def read(self, unpickler): "Reconstruct the array" filename = os.path.join(unpickler._dirname, self.filename) # Load the array from the disk if unpickler.np.__version__ >= '1.3': array = unpickler.np.load(filename, mmap_mode=unpickler.mmap_mode) else: # Numpy does not have mmap_mode before 1.3 array = unpickler.np.load(filename) # Reconstruct subclasses. This does not work with old # versions of numpy if (hasattr(array, '__array_prepare__') and not self.subclass in (unpickler.np.ndarray, unpickler.np.memmap)): # We need to reconstruct another subclass new_array = unpickler.np.core.multiarray._reconstruct( self.subclass, (0,), 'b') new_array.__array_prepare__(array) array = new_array return array #def __reduce__(self): # return None class ZNDArrayWrapper(NDArrayWrapper): """An object to be persisted instead of numpy arrays. This object store the Zfile filename in which the data array has been persisted, and the meta information to retrieve it. The reason that we store the raw buffer data of the array and the meta information, rather than array representation routine (tostring) is that it enables us to use completely the strided model to avoid memory copies (a and a.T store as fast). In addition saving the heavy information separately can avoid creating large temporary buffers when unpickling data with large arrays. """ def __init__(self, filename, init_args, state): "Store the useful information for later" self.filename = filename self.state = state self.init_args = init_args def read(self, unpickler): "Reconstruct the array from the meta-information and the z-file" # Here we a simply reproducing the unpickling mechanism for numpy # arrays filename = os.path.join(unpickler._dirname, self.filename) array = unpickler.np.core.multiarray._reconstruct(*self.init_args) data = read_zfile(open(filename, 'rb')) state = self.state + (data,) array.__setstate__(state) return array ############################################################################### # Pickler classes class NumpyPickler(Pickler): """A pickler to persist of big data efficiently. The main features of this object are: * persistence of numpy arrays in separate .npy files, for which I/O is fast. * optional compression using Zlib, with a special care on avoid temporaries. """ def __init__(self, filename, compress=0, cache_size=10, protocol=None): self._filename = filename self._filenames = [filename, ] self.cache_size = cache_size self.compress = compress if not self.compress: self.file = open(filename, 'wb') else: self.file = BytesIO() if protocol is None: protocol = dill.DEFAULT_PROTOCOL #NOTE: is self.proto # Count the number of npy files that we have created: self._npy_counter = 0 Pickler.__init__(self, self.file, protocol=protocol) # delayed import of numpy, to avoid tight coupling try: import numpy as np except ImportError: np = None self.np = np def _write_array(self, array, filename): if not self.compress: self.np.save(filename, array) container = NDArrayWrapper(os.path.basename(filename), type(array)) else: filename += '.z' # Efficient compressed storage: # The meta data is stored in the container, and the core # numerics in a z-file _, init_args, state = array.__reduce__() # the last entry of 'state' is the data itself with open(filename, 'wb') as zfile: write_zfile(zfile, state[-1], compress=self.compress) state = state[:-1] container = ZNDArrayWrapper(os.path.basename(filename), init_args, state) return container, filename def save(self, obj): """ Subclass the save method, to save ndarray subclasses in npy files, rather than pickling them. Of course, this is a total abuse of the Pickler class. """ if self.np is not None and type(obj) in (self.np.ndarray, self.np.matrix, self.np.memmap): size = obj.size * obj.itemsize if self.compress and size < self.cache_size * _MEGA: # When compressing, as we are not writing directly to the # disk, it is more efficient to use standard pickling if type(obj) is self.np.memmap: # Pickling doesn't work with memmaped arrays obj = self.np.asarray(obj) #FIXME: really? test this... return Pickler.save(self, obj) self._npy_counter += 1 try: filename = '%s_%02i.npy' % (self._filename, self._npy_counter) # This converts the array in a container obj, filename = self._write_array(obj, filename) self._filenames.append(filename) except: self._npy_counter -= 1 # XXX: We should have a logging mechanism print('Failed to save %s to .npy file:\n%s' % ( type(obj), traceback.format_exc())) return Pickler.save(self, obj) def close(self): if self.compress: with open(self._filename, 'wb') as zfile: write_zfile(zfile, self.file.getvalue(), self.compress) class NumpyUnpickler(Unpickler): """A subclass of the Unpickler to unpickle our numpy pickles. """ dispatch = Unpickler.dispatch.copy() def __init__(self, filename, file_handle, mmap_mode=None): self._filename = os.path.basename(filename) self._dirname = os.path.dirname(filename) self.mmap_mode = mmap_mode self.file_handle = self._open_pickle(file_handle) Unpickler.__init__(self, self.file_handle) try: import numpy as np except ImportError: np = None self.np = np def _open_pickle(self, file_handle): return file_handle def load_build(self): """ This method is called to set the state of a newly created object. We capture it to replace our place-holder objects, NDArrayWrapper, by the array we are interested in. We replace them directly in the stack of pickler. """ Unpickler.load_build(self) if isinstance(self.stack[-1], NDArrayWrapper): if self.np is None: raise ImportError('Trying to unpickle an ndarray, ' "but numpy didn't import correctly") nd_array_wrapper = self.stack.pop() array = nd_array_wrapper.read(self) self.stack.append(array) # Be careful to register our new method. dispatch[pickle.BUILD[0]] = load_build class ZipNumpyUnpickler(NumpyUnpickler): """A subclass of our Unpickler to unpickle on the fly from compressed storage.""" def __init__(self, filename, file_handle): NumpyUnpickler.__init__(self, filename, file_handle, mmap_mode=None) def _open_pickle(self, file_handle): return BytesIO(read_zfile(file_handle)) ############################################################################### # Utility functions def dump(value, filename, compress=0, cache_size=100, protocol=None): """Fast persistence of an arbitrary Python object into a files, with dedicated storage for numpy arrays. Parameters ----------- value: any Python object The object to store to disk filename: string The name of the file in which it is to be stored compress: integer for 0 to 9, optional Optional compression level for the data. 0 is no compression. Higher means more compression, but also slower read and write times. Using a value of 3 is often a good compromise. See the notes for more details. cache_size: positive number, optional Fixes the order of magnitude (in megabytes) of the cache used for in-memory compression. Note that this is just an order of magnitude estimate and that for big arrays, the code will go over this value at dump and at load time. protocol: integer The value of the pickle protocol (see serializer documentation for valid values). Generally, 0 is the oldest, with increasing integers for newer protocols, and -1 a shortcut for the highest protocol. Returns ------- filenames: list of strings The list of file names in which the data is stored. If compress is false, each array is stored in a different file. See Also -------- load : corresponding loader Notes ----- Memmapping on load cannot be used for compressed files. Thus using compression can significantly slow down loading. In addition, compressed files take extra extra memory during dump and load. """ if compress is True: # By default, if compress is enabled, we want to be using 3 by # default compress = 3 if not isinstance(filename, _basestring): # People keep inverting arguments, and the resulting error is # incomprehensible raise ValueError( 'Second argument should be a filename, %s (type %s) was given' % (filename, type(filename)) ) try: pickler = NumpyPickler(filename, compress=compress, cache_size=cache_size, protocol=protocol) pickler.dump(value) pickler.close() finally: if 'pickler' in locals() and hasattr(pickler, 'file'): pickler.file.flush() pickler.file.close() return pickler._filenames def load(filename, mmap_mode=None): """Reconstruct a Python object from a file persisted with load. Parameters ----------- filename: string The name of the file from which to load the object mmap_mode: {None, 'r+', 'r', 'w+', 'c'}, optional If not None, the arrays are memory-mapped from the disk. This mode has not effect for compressed files. Note that in this case the reconstructed object might not longer match exactly the originally pickled object. Returns ------- result: any Python object The object stored in the file. See Also -------- dump : function to save an object Notes ----- This function can load numpy array files saved separately during the dump. If the mmap_mode argument is given, it is passed to np.load and arrays are loaded as memmaps. As a consequence, the reconstructed object might not match the original pickled object. Note that if the file was saved with compression, the arrays cannot be memmaped. """ file_handle = open(filename, 'rb') # We are careful to open the file handle early and keep it open to # avoid race-conditions on renames. That said, if data are stored in # companion files, moving the directory will create a race when # trying to access the companion files. if _read_magic(file_handle) == _ZFILE_PREFIX: if mmap_mode is not None: warnings.warn('file "%(filename)s" appears to be a zip, ' 'ignoring mmap_mode "%(mmap_mode)s" flag passed' % locals(), Warning, stacklevel=2) unpickler = ZipNumpyUnpickler(filename, file_handle=file_handle) else: unpickler = NumpyUnpickler(filename, file_handle=file_handle, mmap_mode=mmap_mode) try: obj = unpickler.load() finally: if hasattr(unpickler, 'file_handle'): unpickler.file_handle.close() return obj uqfoundation-klepto-69cd6ce/klepto/archives.py000066400000000000000000000261731455531556400217310ustar00rootroot00000000000000#!/usr/bin/env python # # Author: Mike McKerns (mmckerns @caltech and @uqfoundation) # Copyright (c) 2013-2016 California Institute of Technology. # Copyright (c) 2016-2024 The Uncertainty Quantification Foundation. # License: 3-clause BSD. The full license text is available at: # - https://github.com/uqfoundation/klepto/blob/master/LICENSE """ custom caching dict, which archives results to memory, file, or database """ from ._archives import cache, archive from ._archives import dict_archive as _dict_archive from ._archives import null_archive as _null_archive from ._archives import dir_archive as _dir_archive from ._archives import file_archive as _file_archive from ._archives import sql_archive as _sql_archive from ._archives import sqltable_archive as _sqltable_archive from ._archives import hdf_archive as _hdf_archive from ._archives import hdfdir_archive as _hdfdir_archive from ._archives import _sqlname, _from_frame, _to_frame __all__ = ['cache','dict_archive','null_archive','dir_archive',\ 'file_archive','sql_archive','sqltable_archive',\ 'hdf_archive','hdfdir_archive'] class dict_archive(_dict_archive): def __new__(dict_archive, name=None, dict=None, cached=True, **kwds): """initialize a dictionary with an in-memory dictionary archive backend Args: name (str, default=None): (optional) identifier string dict (dict, default={}): initial dictionary to seed the archive cached (bool, default=True): interact through an in-memory cache """ if dict is None: dict = {} archive = _dict_archive() archive.__state__['id'] = None if name is None else str(name) if cached: archive = cache(archive=archive) archive.update(dict) return archive @classmethod def from_frame(dict_archive, dataframe): try: dataframe = dataframe.copy() dataframe.columns.name = dict_archive.__name__ except AttributeError: pass return _from_frame(dataframe) pass class null_archive(_null_archive): def __new__(null_archive, name=None, dict=None, cached=True, **kwds): """initialize a dictionary with a permanently-empty archive backend Args: name (str, default=None): (optional) identifier string dict (dict, default={}): initial dictionary to seed the archive cached (bool, default=True): interact through an in-memory cache """ if dict is None: dict = {} archive = _null_archive() archive.__state__['id'] = None if name is None else str(name) if cached: archive = cache(archive=archive) archive.update(dict) return archive @classmethod def from_frame(null_archive, dataframe): try: dataframe = dataframe.copy() dataframe.columns.name = null_archive.__name__ except AttributeError: pass return _from_frame(dataframe) pass class dir_archive(_dir_archive): def __new__(dir_archive, name=None, dict=None, cached=True, **kwds): """initialize a dictionary with a file-folder archive backend Args: name (str, default='memo'): path of the archive root directory dict (dict, default={}): initial dictionary to seed the archive cached (bool, default=True): interact through an in-memory cache serialized (bool, default=True): save python objects in pickled files compression (int, default=0): compression level (0 to 9), 0 is None permissions (octal, default=0o775): read/write permission indicator memmode (str, default=None): mode, one of ``{None, 'r+', 'r', 'w+', 'c'}`` memsize (int, default=100): size (MB) of cache for in-memory compression protocol (int, default=DEFAULT_PROTOCOL): pickling protocol """ if dict is None: dict = {} archive = _dir_archive(name, **kwds) if cached: archive = cache(archive=archive) archive.update(dict) return archive @classmethod def from_frame(dir_archive, dataframe): try: dataframe = dataframe.copy() dataframe.columns.name = dir_archive.__name__ except AttributeError: pass return _from_frame(dataframe) pass class file_archive(_file_archive): def __new__(file_archive, name=None, dict=None, cached=True, **kwds): """initialize a dictionary with a single file archive backend Args: name (str, default='memo.pkl'): path of the file archive dict (dict, default={}): initial dictionary to seed the archive cached (bool, default=True): interact through an in-memory cache serialized (bool, default=True): save python objects in pickled file protocol (int, default=DEFAULT_PROTOCOL): pickling protocol """ if dict is None: dict = {} archive = _file_archive(name, **kwds) if cached: archive = cache(archive=archive) archive.update(dict) return archive @classmethod def from_frame(file_archive, dataframe): try: dataframe = dataframe.copy() dataframe.columns.name = file_archive.__name__ except AttributeError: pass return _from_frame(dataframe) pass class sqltable_archive(_sqltable_archive): def __new__(sqltable_archive, name=None, dict=None, cached=True, **kwds): """initialize a dictionary with a sql database table archive backend Connect to an existing database, or initialize a new database, at the selected database url. For example, to use a sqlite database 'foo.db' in the current directory, database='sqlite:///foo.db'. To use a mysql database 'foo' on localhost, database='mysql://user:pass@localhost/foo'. For postgresql, use database='postgresql://user:pass@localhost/foo'. When connecting to sqlite, the default database is ':memory:'; otherwise, the default database is 'defaultdb'. Connections should be given as database?table=tablename; for example, name='sqlite:///foo.db?table=bar'. If not provided, the default tablename is 'memo'. If sqlalchemy is not installed, storable values are limited to strings, integers, floats, and other basic objects. If sqlalchemy is installed, additional keyword options can provide database configuration, such as connection pooling. To use a mysql or postgresql database, sqlalchemy must be installed. Args: name (str, default=None): url for database table (see above note) dict (dict, default={}): initial dictionary to seed the archive cached (bool, default=True): interact through an in-memory cache serialized (bool, default=True): save objects as pickled strings protocol (int, default=DEFAULT_PROTOCOL): pickling protocol """ if dict is None: dict = {} db, table = _sqlname(name) archive = _sqltable_archive(db, table, **kwds) if cached: archive = cache(archive=archive) archive.update(dict) return archive @classmethod def from_frame(sqltable_archive, dataframe): try: dataframe = dataframe.copy() dataframe.columns.name = sqltable_archive.__name__ except AttributeError: pass return _from_frame(dataframe) pass class sql_archive(_sql_archive): def __new__(sql_archive, name=None, dict=None, cached=True, **kwds): """initialize a dictionary with a sql database archive backend Connect to an existing database, or initialize a new database, at the selected database url. For example, to use a sqlite database 'foo.db' in the current directory, database='sqlite:///foo.db'. To use a mysql database 'foo' on localhost, database='mysql://user:pass@localhost/foo'. For postgresql, use database='postgresql://user:pass@localhost/foo'. When connecting to sqlite, the default database is ':memory:'; otherwise, the default database is 'defaultdb'. If sqlalchemy is not installed, storable values are limited to strings, integers, floats, and other basic objects. If sqlalchemy is installed, additional keyword options can provide database configuration, such as connection pooling. To use a mysql or postgresql database, sqlalchemy must be installed. Args: name (str, default=None): database url (see above note) dict (dict, default={}): initial dictionary to seed the archive cached (bool, default=True): interact through an in-memory cache serialized (bool, default=True): save objects as pickled strings protocol (int, default=DEFAULT_PROTOCOL): pickling protocol """ if dict is None: dict = {} archive = _sql_archive(name, **kwds) if cached: archive = cache(archive=archive) archive.update(dict) return archive @classmethod def from_frame(sql_archive, dataframe): try: dataframe = dataframe.copy() dataframe.columns.name = sql_archive.__name__ except AttributeError: pass return _from_frame(dataframe) pass class hdfdir_archive(_hdfdir_archive): def __new__(hdfdir_archive, name=None, dict=None, cached=True, **kwds): """initialize a dictionary with a hdf5 file-folder archive backend Args: name (str, default='memo'): path of the archive root directory dict (dict, default={}): initial dictionary to seed the archive cached (bool, default=True): interact through an in-memory cache serialized (bool, default=True): pickle saved python objects permissions (octal, default=0o775): read/write permission indicator protocol (int, default=DEFAULT_PROTOCOL): pickling protocol meta (bool, default=False): store in root metadata (not in dataset) """ if dict is None: dict = {} archive = _hdfdir_archive(name, **kwds) if cached: archive = cache(archive=archive) archive.update(dict) return archive @classmethod def from_frame(hdfdir_archive, dataframe): try: dataframe = dataframe.copy() dataframe.columns.name = hdfdir_archive.__name__ except AttributeError: pass return _from_frame(dataframe) pass class hdf_archive(_hdf_archive): def __new__(hdf_archive, name=None, dict=None, cached=True, **kwds): """initialize a dictionary with a single hdf5 file archive backend Args: name (str, default='memo.hdf5'): path of the file archive dict (dict, default={}): initial dictionary to seed the archive cached (bool, default=True): interact through an in-memory cache serialized (bool, default=True): pickle saved python objects protocol (int, default=DEFAULT_PROTOCOL): pickling protocol meta (bool, default=False): store in root metadata (not in dataset) """ if dict is None: dict = {} archive = _hdf_archive(name, **kwds) if cached: archive = cache(archive=archive) archive.update(dict) return archive @classmethod def from_frame(hdf_archive, dataframe): try: dataframe = dataframe.copy() dataframe.columns.name = hdf_archive.__name__ except AttributeError: pass return _from_frame(dataframe) pass # EOF uqfoundation-klepto-69cd6ce/klepto/crypto.py000066400000000000000000000106551455531556400214430ustar00rootroot00000000000000#!/usr/bin/env python # # Author: Mike McKerns (mmckerns @caltech and @uqfoundation) # Copyright (c) 2013-2016 California Institute of Technology. # Copyright (c) 2016-2024 The Uncertainty Quantification Foundation. # License: 3-clause BSD. The full license text is available at: # - https://github.com/uqfoundation/klepto/blob/master/LICENSE import os import sys import hashlib import pkgutil import encodings as codecs __hash = hash def algorithms(): """return a tuple of available hash algorithms""" try: algs = tuple(hashlib.algorithms_available) except: algs = ('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512') return (None,) + algs def hash(object, algorithm=None): if algorithm is None: return __hash(object) return hashlib.new(algorithm, repr(object).encode()).hexdigest() hash.algorithms = algorithms hash.__doc__ = \ """cryptographic hashing algorithm: one of %s The default is algorithm=None, which uses python's 'hash'.""" % repr(algorithms()) def encodings(): """return a tuple of available encodings and string-like types""" try: algs = set([modname for importer, modname, ispkg in pkgutil.walk_packages(path=[os.path.dirname(codecs.__file__)], prefix='')]) except: algs = set() algs = algs.union(set(codecs.aliases.aliases.values())) try: #FIXME: essentially, a poor alias for python 3.x eval('unicode') #XXX: i.e. only allow unicode and bytes in python 2.x utype = ('unicode','bytes') except NameError: utype = tuple() if 'tactis' in algs: algs.remove('tactis') pop = [t for t in algs if t.endswith('_codec')] [algs.remove(t) for t in pop] #FIXME: giving up here for 3.x... # (any '*_codec' throws 'str' does not support the buffer interface) stype = ('str','repr') return (None,) + tuple(algs) + stype + utype def string(object, encoding=None, strict=True): """encode an object (as a string) strict: bool or None, for 'strictness' of the encoding encoding: one of the available string encodings or string-like types For encodings, such as 'utf-8', strict=True will raise an exception in the case of an encoding error, strict=None will ignore malformed data, and strict=False will replace malformed data with a suitable marker such as '?' or '\ufffd'. For string-like types, strict=True restricts the type casting to the list of types in klepto.crypto.encodings(). The default is encoding=None, which uses python's 'str'.""" if encoding is None: return str(object) try: if strict and encoding not in encodings(): raise NameError try: #FIXME: 'bytes' not quite right for python3.x return eval("%s(object)" % encoding) #XXX: safer is %s(repr(object)) except TypeError: # special case for bytes: object is a string return eval("%s(object, 'utf_8')" % encoding) except: if strict: strict = 'strict' elif strict is None: strict = 'ignore' else: strict = 'replace' return repr(object).encode(encoding, strict) string.encodings = encodings def serializers(): #FIXME: could be much smarter """return a tuple of string names of serializers""" serializers = (None, 'pickle', 'json', 'dill') from importlib import util as imp if imp.find_spec('cloudpickle'): serializers += ('cloudpickle',) if imp.find_spec('jsonpickle'): serializers += ('jsonpickle',) return serializers def pickle(object, serializer=None, **kwds): """pickle an object (to a string) serializer: name of pickler module with a 'dumps' method The default is serializer=None, which uses python's 'repr'. NOTE: any 'bad' kwds will cause all kwds to be ignored.""" if serializer is None: return repr(object) #XXX: better hex(id(object)) ? if not isinstance(serializer, type(os)): # if module, don't try to import it if not isinstance(serializer, str): raise TypeError("'%s' is not a module" % repr(serializer)) try: # is a string serializer = __import__(serializer) except: raise NameError("name '%s' is not defined" % serializer) # now serializer is a module, work with it try: return serializer.dumps(object, **kwds) except TypeError: return serializer.dumps(object) #XXX: better alternative behavior? pickle.serializers = serializers # EOF uqfoundation-klepto-69cd6ce/klepto/keymaps.py000066400000000000000000000356271455531556400216020ustar00rootroot00000000000000#!/usr/bin/env python # # Author: Mike McKerns (mmckerns @caltech and @uqfoundation) # Copyright (c) 2013-2016 California Institute of Technology. # Copyright (c) 2016-2024 The Uncertainty Quantification Foundation. # License: 3-clause BSD. The full license text is available at: # - https://github.com/uqfoundation/klepto/blob/master/LICENSE """ custom 'keymaps' for generating dictionary keys from function input signatures """ __all__ = ['SENTINEL','NOSENTINEL','keymap','hashmap','stringmap','picklemap'] class _Sentinel(object): """build a sentinel object for the SENTINEL singleton""" def __repr__(self): return "" class _NoSentinel(object): """build a sentinel object for the NOSENTINEL singleton""" def __repr__(self): return "" SENTINEL = _Sentinel() NOSENTINEL = _NoSentinel() # SENTINEL = object() # NOSENTINEL = (SENTINEL,) #XXX: use to indicate "don't use a sentinel" ? from copy import copy from klepto.crypto import hash, string, pickle def _stub_decoder(keymap=None): "generate a keymap decoder from information in the keymap stub" #FIXME: need to implement generalized inverse of keymap #HACK: only works in certain special cases if isinstance(keymap, (str, (u'').__class__)): from klepto.crypto import algorithms, encodings, serializers if keymap in serializers(): #FIXME: ignores all config options import importlib inv = lambda k: importlib.import_module(keymap).loads(k) elif keymap in algorithms() or encodings(): #FIXME always assumes repr inv = lambda k: eval(k) else: #FIXME: give up, ignore keymap inv = lambda k: k return inv kind = getattr(keymap, '__stub__', '') if kind in ('encoding', 'algorithm'): #FIXME always assumes repr inv = lambda k: eval(k) elif kind in ('serializer', ): #FIXME: ignores all config options if keymap.__type__ is None: inv = lambda k: eval(k) else: import importlib inv = lambda k: importlib.import_module(keymap.__type__).loads(k) else: #FIXME: give up, ignore keymap inv = lambda k: k return inv def __chain__(x, y): "chain two keymaps: calls 'x' then 'y' on object to produce y(x(object))" if x is None: x,y = y,x if y is None: f = lambda z: x(z) else: f = lambda z: y(x(z)) if y is None: msg = "" else: msg = "calls %s then %s" % (x,y) f.__doc__ = msg f.__inner__ = x f.__outer__ = y return f class keymap(object): """tool for converting a function's input signature to an unique key This keymap does not serialize objects, but does do some formatting. Since the keys are stored as raw objects, there is no information loss, and thus it is easy to recover the original input signature. However, to use an object as a key, the object must be hashable. """ def __init__(self, typed=False, flat=True, sentinel=NOSENTINEL, **kwds): '''initialize the key builder typed: if True, include type information in the key flat: if True, flatten the key to a sequence; if False, use (args, kwds) sentinel: marker for separating args and kwds in flattened keys This keymap stores function args and kwds as (args, kwds) if flat=False, or a flattened ``(*args, zip(**kwds))`` if flat=True. If typed, then include a tuple of type information (args, kwds, argstypes, kwdstypes) in the generated key. If a sentinel is given, the sentinel will be added to a flattened key to indicate the boundary between args, keys, argstypes, and kwdstypes. ''' self.typed = typed self.flat = flat self.sentinel = sentinel #self.__chain__ = __chain__(self, None) self.__inner__ = None self.__outer__ = None #self.__type__ = None #XXX: stuff breaks if this exists self.__stub__ = '' # some rare kwds that allow keymap customization try: self._fasttypes = (int,str,bytes,frozenset,type(None)) except NameError: self._fasttypes = (int,str,frozenset,type(None)) self._fasttypes = kwds.pop('fasttypes', set(self._fasttypes)) self._sorted = kwds.pop('sorted', sorted) self._tuple = kwds.pop('tuple', tuple) self._type = kwds.pop('type', type) self._len = kwds.pop('len', len) # the rest of the kwds are for customizaton of the encoder self._config = kwds.copy() return def __get_outer(self): "get 'outer' keymap" return self.__outer__ def __get_inner(self): "get 'nested' keymap, if one exists" #if self.__chain__.__outer__: # return self.__chain__.__inner__ #return None return self.__inner__ def __chain(self, map): "create a 'nested' keymap" raise NotImplementedError("Combine keymaps with '+'") def __repr__(self): msg = "%s(" % self.__class__.__name__ if self.typed != False: msg += 'typed=%s, ' % self.typed if self.flat != True: msg += 'flat=%s, ' % self.flat if self.sentinel != NOSENTINEL: msg += 'sentinel=%s, ' % self.sentinel if self.__stub__ != '' and self.__type__ is not None: msg += "%s='%s', " % (self.__stub__, self.__type__) #msg += 'inner=%s)' % bool(self.inner) if msg: msg = msg.rstrip().rstrip(',') if bool(self.inner): msg += ')*' else: msg += ')' return msg def __get_sentinel(self): if self._mark: return self._mark[0] return NOSENTINEL #XXX: or None? def __sentinel(self, mark): if mark != NOSENTINEL: self._mark = (mark,) else: self._mark = None def __call__(self, *args, **kwds): 'generate a key from optionally typed positional and keyword arguments' if self.flat: return self.encode(*args, **kwds) return self.encrypt(*args, **kwds) def encrypt(self, *args, **kwds): """use a non-flat scheme for generating a key""" key = (args, kwds) #XXX: pickles larger, but is simpler to unpack if self.typed: sorted_items = self._sorted(list(kwds.items())) key += (self._tuple(self._type(v) for v in args), \ self._tuple(self._type(v) for (k,v) in sorted_items)) # __chain__ if self.outer: return self.inner(key) return key def encode(self, *args, **kwds): """use a flattened scheme for generating a key""" key = args if kwds: sorted_items = self._sorted(list(kwds.items())) if self._mark: key += self._mark for item in sorted_items: key += item if self.typed: #XXX: 'mark' between each part, so easy to split if self._mark: key += self._mark key += self._tuple(self._type(v) for v in args) if kwds: if self._mark: key += self._mark key += self._tuple(self._type(v) for (k,v) in sorted_items) elif self._len(key) == 1 and self._type(key[0]) in self._fasttypes: key = key[0] # __chain__ if self.outer: return self.inner(key) return key def decrypt(self, key): """recover the stored value directly from a generated (non-flat) key""" raise NotImplementedError("Key decryption is not implemented") def decode(self, key): """recover the stored value directly from a generated (flattened) key""" raise NotImplementedError("Key decoding is not implemented") def dumps(self, obj, **kwds): """a more pickle-like interface for encoding a key""" return self.encode(obj, **kwds) def loads(self, key): """a more pickle-like interface for decoding a key""" return self.decode(key) def __add__(self, other): """concatenate two keymaps, to produce a new keymap""" if not isinstance(other, keymap): raise TypeError("can't concatenate '%s' and '%s' objects" % (self.__class__.__name__, other.__class__.__name__)) k = copy(other) #k.__chain__ = __chain__(self, k) k.__inner__ = copy(self) #XXX: or just... self ? k.__outer__ = copy(other) #XXX: or just... other ? return k # interface sentinel = property(__get_sentinel, __sentinel) inner = property(__get_inner, __chain) outer = property(__get_outer, __chain) pass class hashmap(keymap): """tool for converting a function's input signature to an unique key This keymap generates a hash for the given object. Not all objects are hashable, and generating a hash incurrs some information loss. Hashing is fast, however there is not a method to recover the input signature from a hash. """ #XXX: algorithm as first argument? easier to build, but less standard def __init__(self, typed=False, flat=True, sentinel=NOSENTINEL, **kwds): '''initialize the key builder typed: if True, include type information in the key flat: if True, flatten the key to a sequence; if False, use (args, kwds) sentinel: marker for separating args and kwds in flattened keys algorithm: string name of hashing algorithm [default: use python's hash] This keymap stores function args and kwds as (args, kwds) if flat=False, or a flattened ``(*args, zip(**kwds))`` if flat=True. If typed, then include a tuple of type information (args, kwds, argstypes, kwdstypes) in the generated key. If a sentinel is given, the sentinel will be added to a flattened key to indicate the boundary between args, keys, argstypes, and kwdstypes. Use kelpto.crypto.algorithms() to get the names of available hashing algorithms. ''' self.__type__ = kwds.pop('algorithm', None) keymap.__init__(self, typed=typed, flat=flat, sentinel=sentinel, **kwds) self.__stub__ = 'algorithm' #XXX: unnecessary if unified kwd return def encode(self, *args, **kwds): """use a flattened scheme for generating a key""" return hash(keymap.encode(self, *args, **kwds), algorithm=self.__type__, **self._config) def encrypt(self, *args, **kwds): """use a non-flat scheme for generating a key""" return hash(keymap.encrypt(self, *args, **kwds), algorithm=self.__type__, **self._config) class stringmap(keymap): """tool for converting a function's input signature to an unique key This keymap serializes objects by converting __repr__ to a string. Converting to a string is slower than hashing, however will produce a key in all cases. Some forms of archival storage, like a database, may require string keys. There is not a method to recover the input signature from a string key that works in all cases, however this is possible for any object where __repr__ effectively mimics __init__. """ #XXX: encoding as first argument? easier to build, but less standard def __init__(self, typed=False, flat=True, sentinel=NOSENTINEL, **kwds): '''initialize the key builder typed: if True, include type information in the key flat: if True, flatten the key to a sequence; if False, use (args, kwds) sentinel: marker for separating args and kwds in flattened keys encoding: string name of string encoding [default: use python's str] This keymap stores function args and kwds as (args, kwds) if flat=False, or a flattened ``(*args, zip(**kwds))`` if flat=True. If typed, then include a tuple of type information (args, kwds, argstypes, kwdstypes) in the generated key. If a sentinel is given, the sentinel will be added to a flattened key to indicate the boundary between args, keys, argstypes, and kwdstypes. Use kelpto.crypto.encodings() to get the names of available string encodings. ''' self.__type__ = kwds.pop('encoding', None) keymap.__init__(self, typed=typed, flat=flat, sentinel=sentinel, **kwds) self.__stub__ = 'encoding' #XXX: unnecessary if unified kwd return def encode(self, *args, **kwds): """use a flattened scheme for generating a key""" return string(keymap.encode(self, *args, **kwds), encoding=self.__type__, **self._config) def encrypt(self, *args, **kwds): """use a non-flat scheme for generating a key""" return string(keymap.encrypt(self, *args, **kwds), encoding=self.__type__, **self._config) class picklemap(keymap): """tool for converting a function's input signature to an unique key This keymap serializes objects by pickling the object. Serializing an object with pickle is relatively slower, however will reliably produce a unique key for all picklable objects. Also, pickling is a reversible operation, where the original input signature can be recovered from the generated key. """ #XXX: serializer as first argument? easier to build, but less standard def __init__(self, typed=False, flat=True, sentinel=NOSENTINEL, **kwds): '''initialize the key builder typed: if True, include type information in the key flat: if True, flatten the key to a sequence; if False, use (args, kwds) sentinel: marker for separating args and kwds in flattened keys serializer: string name of pickler [default: use python's repr] This keymap stores function args and kwds as (args, kwds) if flat=False, or a flattened ``(*args, zip(**kwds))`` if flat=True. If typed, then include a tuple of type information (args, kwds, argstypes, kwdstypes) in the generated key. If a sentinel is given, the sentinel will be added to a flattened key to indicate the boundary between args, keys, argstypes, and kwdstypes. Use kelpto.crypto.serializers() to get the names of available picklers. NOTE: the serializer kwd expects a object, and not a . ''' kwds['byref'] = kwds.get('byref',True) #XXX: for dill self.__type__ = kwds.pop('serializer', None) #XXX: better not convert __type__ to string, so don't __import__ ? if not isinstance(self.__type__, (str, type(None))): self.__type__ = self.__type__.__name__ keymap.__init__(self, typed=typed, flat=flat, sentinel=sentinel, **kwds) self.__stub__ = 'serializer' #XXX: unnecessary if unified kwd return def encode(self, *args, **kwds): """use a flattened scheme for generating a key""" return pickle(keymap.encode(self, *args, **kwds), serializer=self.__type__, **self._config) # separator=(',',':') for json def encrypt(self, *args, **kwds): """use a non-flat scheme for generating a key""" return pickle(keymap.encrypt(self, *args, **kwds), serializer=self.__type__, **self._config) # separator=(',',':') for json # EOF uqfoundation-klepto-69cd6ce/klepto/rounding.py000066400000000000000000000161101455531556400217400ustar00rootroot00000000000000#!/usr/bin/env python # # Author: Mike McKerns (mmckerns @caltech and @uqfoundation) # Copyright (c) 2013-2016 California Institute of Technology. # Copyright (c) 2016-2024 The Uncertainty Quantification Foundation. # License: 3-clause BSD. The full license text is available at: # - https://github.com/uqfoundation/klepto/blob/master/LICENSE """ decorators that provide rounding """ __all__ = ['deep_round', 'shallow_round', 'simple_round'] #FIXME: these seem *slow*... and a bit convoluted. Maybe rewrite as classes? unicode = str #PYTHON3 def deep_round_factory(tol): """helper function for deep_round (a factory for deep_round functions)""" from klepto.tools import isiterable def deep_round(*args, **kwds): argstype = type(args) _args = list(args) _kwds = kwds.copy() for i,j in enumerate(args): if isinstance(j, float): _args[i] = round(j, tol) # don't round int elif isinstance(j, (str, unicode, type(BaseException()))): continue elif isinstance(j, dict): _args[i] = deep_round(**j)[1] elif isiterable(j): #XXX: fails on the above, so don't iterate them jtype = type(j) _args[i] = jtype(deep_round(*j)[0]) for i,j in kwds.items(): if isinstance(j, float): _kwds[i] = round(j, tol) elif isinstance(j, (str, unicode, type(BaseException()))): continue elif isinstance(j, dict): _kwds[i] = deep_round(**j)[1] elif isiterable(j): #XXX: fails on the above, so don't iterate them jtype = type(j) _kwds[i] = jtype(deep_round(*j)[0]) return argstype(_args), _kwds return deep_round """ >>> deep_round = deep_round_factory(tol=0) #FIXME: convert to decorator !!! >>> deep_round([1.12,2,{'x':1.23, 'y':[4.56,5.67]}], x=set([11.22,44,'hi'])) (([1.0, 2, {'y': [5.0, 6.0], 'x': 1.0}],), {'x': set([11.0, 'hi', 44])}) """ class deep_round(object): """decorator for rounding a function's input argument and keywords to the given precision *tol*. This decorator always rounds to a floating point number. Rounding is done recursively for each element of all arguments and keywords. For example: >>> @deep_round(tol=1) ... def add(x,y): ... return x+y ... >>> add(2.54, 5.47) 8.0 >>> >>> # rounds each float, regardless of depth in an object >>> add([2.54, 'x'],[5.47, 'y']) [2.5, 'x', 5.5, 'y'] >>> >>> # rounds each float, regardless of depth in an object >>> add([2.54, 'x'],[5.47, [8.99, 'y']]) [2.5, 'x', 5.5, [9.0, 'y']] """ def __init__(self, tol=0): self.__round__ = deep_round_factory(tol) self.__round__.tol = tol return def __call__(self, f): def func(*args, **kwds): if self.__round__.tol is None: _args,_kwds = args,kwds else: _args,_kwds = self.__round__(*args, **kwds) return f(*_args, **_kwds) func.__wrapped__ = f #NOTE: attr missing after (un)pickling return func def __get__(self, obj, objtype): import functools return functools.partial(self.__call__, obj) def __reduce__(self): return (self.__class__, (self.__round__.tol,)) def simple_round_factory(tol): """helper function for simple_round (a factory for simple_round functions)""" def simple_round(*args, **kwds): argstype = type(args) _args = list(args) _kwds = kwds.copy() for i,j in enumerate(args): if isinstance(j, float): _args[i] = round(j, tol) # don't round int for i,j in kwds.items(): if isinstance(j, float): _kwds[i] = round(j, tol) return argstype(_args), _kwds return simple_round class simple_round(object): #NOTE: only rounds floats, nothing else """decorator for rounding a function's input argument and keywords to the given precision *tol*. This decorator always rounds to a floating point number. Rounding is only done for arguments or keywords that are floats. For example: >>> @simple_round(tol=1) ... def add(x,y): ... return x+y ... >>> add(2.54, 5.47) 8.0 >>> >>> # does not round elements of iterables, only rounds at the top-level >>> add([2.54, 'x'],[5.47, 'y']) [2.54, 'x', 5.4699999999999998, 'y'] >>> >>> # does not round elements of iterables, only rounds at the top-level >>> add([2.54, 'x'],[5.47, [8.99, 'y']]) [2.54, 'x', 5.4699999999999998, [8.9900000000000002, 'y']] """ def __init__(self, tol=0): self.__round__ = simple_round_factory(tol) self.__round__.tol = tol return def __call__(self, f): def func(*args, **kwds): if self.__round__.tol is None: _args,_kwds = args,kwds else: _args,_kwds = self.__round__(*args, **kwds) return f(*_args, **_kwds) func.__wrapped__ = f #NOTE: attr missing after (un)pickling return func def __get__(self, obj, objtype): import functools return functools.partial(self.__call__, obj) def __reduce__(self): return (self.__class__, (self.__round__.tol,)) def shallow_round_factory(tol): """helper function for shallow_round (a factory for shallow_round functions)""" def around(iterable, tol): if isinstance(iterable, float): return round(iterable, tol) from klepto.tools import isiterable if not isiterable(iterable): return iterable itype = type(iterable) _iterable = list(iterable) for i,j in enumerate(iterable): if isinstance(j, float): _iterable[i] = round(j, tol) return itype(_iterable) def shallow_round(*args, **kwds): argstype = type(args) _args = list(args) _kwds = kwds.copy() for i,j in enumerate(args): try: jtype = type(j) _args[i] = jtype(around(j, tol)) except: pass for i,j in kwds.items(): try: jtype = type(j) _kwds[i] = jtype(around(j, tol)) except: pass return argstype(_args), _kwds return shallow_round class shallow_round(object): #NOTE: rounds floats, lists, arrays one level deep """decorator for rounding a function's input argument and keywords to the given precision *tol*. This decorator always rounds to a floating point number. Rounding is done recursively for each element of all arguments and keywords, however the rounding is shallow (a max of one level deep into each object). For example: >>> @shallow_round(tol=1) ... def add(x,y): ... return x+y ... >>> add(2.54, 5.47) 8.0 >>> >>> # rounds each float, at the top-level or first-level of each object. >>> add([2.54, 'x'],[5.47, 'y']) [2.5, 'x', 5.5, 'y'] >>> >>> # rounds each float, at the top-level or first-level of each object. >>> add([2.54, 'x'],[5.47, [8.99, 'y']]) [2.5, 'x', 5.5, [8.9900000000000002, 'y']] """ def __init__(self, tol=0): self.__round__ = shallow_round_factory(tol) self.__round__.tol = tol return def __call__(self, f): def func(*args, **kwds): if self.__round__.tol is None: _args,_kwds = args,kwds else: _args,_kwds = self.__round__(*args, **kwds) return f(*_args, **_kwds) func.__wrapped__ = f #NOTE: attr missing after (un)pickling return func def __get__(self, obj, objtype): import functools return functools.partial(self.__call__, obj) def __reduce__(self): return (self.__class__, (self.__round__.tol,)) # EOF uqfoundation-klepto-69cd6ce/klepto/safe.py000066400000000000000000001577121455531556400210470ustar00rootroot00000000000000#!/usr/bin/env python # code inspired by Raymond Hettinger's LFU and LRU cache decorators # on http://code.activestate.com/recipes/498245-lru-and-lfu-cache-decorators # and subsequent forks as well as the version available in python3.3 # # Author: Mike McKerns (mmckerns @caltech and @uqfoundation) # Copyright (c) 2013-2016 California Institute of Technology. # Copyright (c) 2016-2024 The Uncertainty Quantification Foundation. # License: 3-clause BSD. The full license text is available at: # - https://github.com/uqfoundation/klepto/blob/master/LICENSE """ 'safe' versions of selected caching decorators If a hashing error occurs, the cached function will be evaluated. """ from functools import update_wrapper, partial from klepto.archives import cache as archive_dict from klepto.keymaps import stringmap from klepto.tools import CacheInfo from klepto.rounding import deep_round, simple_round from ._inspect import _keygen __all__ = ['no_cache','inf_cache','lfu_cache',\ 'lru_cache','mru_cache','rr_cache'] class Counter(dict): 'Mapping where default values are zero' def __missing__(self, key): return 0 #XXX: what about caches that expire due to time, calls, etc... #XXX: check the impact of not serializing by default, and stringmap by default class no_cache(object): """'safe' version of the empty (NO) cache decorator. Unlike other cache decorators, this decorator does not cache. It is a dummy that collects statistics and conforms to the caching interface. This decorator takes an integer tolerance 'tol', equal to the number of decimal places to which it will round off floats, and a bool 'deep' for whether the rounding on inputs will be 'shallow' or 'deep'. Note that rounding is not applied to the calculation of new results, but rather as a simple form of cache interpolation. For example, with tol=0 and a cached value for f(3.0), f(3.1) will lookup f(3.0) in the cache while f(3.6) will store a new value; however if tol=1, both f(3.1) and f(3.6) will store new values. maxsize = maximum cache size [fixed at maxsize=0] cache = storage hashmap (default is {}) keymap = cache key encoder (default is keymaps.stringmap(flat=False)) ignore = function argument names and indicies to 'ignore' (default is None) tol = integer tolerance for rounding (default is None) deep = boolean for rounding depth (default is False, i.e. 'shallow') purge = boolean for purge cache to archive at maxsize (fixed at True) If *keymap* is given, it will replace the hashing algorithm for generating cache keys. Several hashing algorithms are available in 'keymaps'. The default keymap does not require arguments to the cached function to be hashable. If a hashing error occurs, the cached function will be evaluated. If the keymap retains type information, then arguments of different types will be cached separately. For example, f(3.0) and f(3) will be treated as distinct calls with distinct results. Cache typing has a memory penalty, and may also be ignored by some 'keymaps'. Here, the keymap is only used to look up keys in an associated archive. If *ignore* is given, the keymap will ignore the arguments with the names and/or positional indicies provided. For example, if ignore=(0,), then the key generated for f(1,2) will be identical to that of f(3,2) or f(4,2). If ignore=('y',), then the key generated for f(x=3,y=4) will be identical to that of f(x=3,y=0) or f(x=3,y=10). If ignore=('*','**'), all varargs and varkwds will be 'ignored'. Ignored arguments never trigger a recalculation (they only trigger cache lookups), and thus are 'ignored'. When caching class methods, it may be useful to ignore=('self',). View cache statistics (hit, miss, load, maxsize, size) with f.info(). Clear the cache and statistics with f.clear(). Replace the cache archive with f.archive(obj). Load from the archive with f.load(), and dump from the cache to the archive with f.dump(). """ def __init__(self, maxsize=0, cache=None, keymap=None, ignore=None, tol=None, deep=False, purge=True): #if maxsize is not 0: raise ValueError('maxsize cannot be set') maxsize = 0 #XXX: allow maxsize to be given but ignored ? purge = True #XXX: allow purge to be given but ignored ? if cache is None: cache = archive_dict() elif type(cache) is dict: cache = archive_dict(cache) if keymap is None: keymap = stringmap(flat=False) if ignore is None: ignore = tuple() if deep: rounded = deep_round else: rounded = simple_round #else: rounded = shallow_round #FIXME: slow @rounded(tol) def rounded_args(*args, **kwds): return (args, kwds) # set state self.__state__ = { 'maxsize': maxsize, 'cache': cache, 'keymap': keymap, 'ignore': ignore, 'roundargs': rounded_args, 'tol': tol, 'deep': deep, 'purge': purge, } return def __call__(self, user_function): #cache = dict() # mapping of args to results stats = [0, 0, 0] # make statistics updateable non-locally HIT, MISS, LOAD = 0, 1, 2 # names for the stats fields _len = len # localize the global len() function #lock = RLock() # linkedlist updates aren't threadsafe maxsize = self.__state__['maxsize'] cache = self.__state__['cache'] keymap = self.__state__['keymap'] ignore = self.__state__['ignore'] rounded_args = self.__state__['roundargs'] def wrapper(*args, **kwds): try: _args, _kwds = rounded_args(*args, **kwds) _args, _kwds = _keygen(user_function, ignore, *_args, **_kwds) key = keymap(*_args, **_kwds) except: #TypeError result = user_function(*args, **kwds) stats[MISS] += 1 return result # look in archive if cache.archived(): cache.load(key) try: result = cache[key] cache.clear() stats[LOAD] += 1 except KeyError: # if not found, then compute result = user_function(*args, **kwds) cache[key] = result stats[MISS] += 1 except: #TypeError: # unhashable key result = user_function(*args, **kwds) stats[MISS] += 1 # purge cache if _len(cache) > maxsize: #XXX: better: if cache is cache.archive ? if cache.archived(): cache.dump() cache.clear() return result def archive(obj): """Replace the cache archive""" if isinstance(obj, archive_dict): cache.archive = obj.archive else: cache.archive = obj def key(*args, **kwds): """Get the cache key for the given *args,**kwds""" _args, _kwds = rounded_args(*args, **kwds) _args, _kwds = _keygen(user_function, ignore, *_args, **_kwds) return keymap(*_args, **_kwds) def lookup(*args, **kwds): """Get the stored value for the given *args,**kwds""" _args, _kwds = rounded_args(*args, **kwds) _args, _kwds = _keygen(user_function, ignore, *_args, **_kwds) return cache[keymap(*_args, **_kwds)] def __get_cache(): """Get the cache""" return cache def __get_mask(): """Get the (ignore) mask""" return ignore def __get_keymap(): """Get the keymap""" return keymap def clear(keepstats=False): """Clear the cache and statistics""" if not keepstats: stats[:] = [0, 0, 0] def info(): """Report cache statistics""" return CacheInfo(stats[HIT], stats[MISS], stats[LOAD], maxsize, len(cache)) # interface wrapper.__wrapped__ = user_function #XXX: better is handle to key_function=keygen(ignore)(user_function) ? wrapper.info = info wrapper.clear = clear wrapper.load = cache.load wrapper.dump = cache.dump wrapper.archive = archive wrapper.archived = cache.archived wrapper.key = key wrapper.lookup = lookup wrapper.__cache__ = __get_cache wrapper.__mask__ = __get_mask wrapper.__map__ = __get_keymap #wrapper._queue = None #XXX return update_wrapper(wrapper, user_function) def __get__(self, obj, objtype): """support instance methods""" return partial(self.__call__, obj) def __reduce__(self): cache = self.__state__['cache'] keymap = self.__state__['keymap'] ignore = self.__state__['ignore'] tol = self.__state__['tol'] deep = self.__state__['deep'] return (self.__class__, (0, cache, keymap, ignore, tol, deep, True)) class inf_cache(object): """'safe' version of the infinitely-growing (INF) cache decorator. This decorator memoizes a function's return value each time it is called. If called later with the same arguments, the cached value is returned, and not re-evaluated. This cache will grow without bound. To avoid memory issues, it is suggested to frequently dump and clear the cache. This decorator takes an integer tolerance 'tol', equal to the number of decimal places to which it will round off floats, and a bool 'deep' for whether the rounding on inputs will be 'shallow' or 'deep'. Note that rounding is not applied to the calculation of new results, but rather as a simple form of cache interpolation. For example, with tol=0 and a cached value for f(3.0), f(3.1) will lookup f(3.0) in the cache while f(3.6) will store a new value; however if tol=1, both f(3.1) and f(3.6) will store new values. maxsize = maximum cache size [fixed at maxsize=None] cache = storage hashmap (default is {}) keymap = cache key encoder (default is keymaps.stringmap(flat=False)) ignore = function argument names and indicies to 'ignore' (default is None) tol = integer tolerance for rounding (default is None) deep = boolean for rounding depth (default is False, i.e. 'shallow') purge = boolean for purge cache to archive at maxsize (fixed at False) If *keymap* is given, it will replace the hashing algorithm for generating cache keys. Several hashing algorithms are available in 'keymaps'. The default keymap does not require arguments to the cached function to be hashable. If a hashing error occurs, the cached function will be evaluated. If the keymap retains type information, then arguments of different types will be cached separately. For example, f(3.0) and f(3) will be treated as distinct calls with distinct results. Cache typing has a memory penalty, and may also be ignored by some 'keymaps'. If *ignore* is given, the keymap will ignore the arguments with the names and/or positional indicies provided. For example, if ignore=(0,), then the key generated for f(1,2) will be identical to that of f(3,2) or f(4,2). If ignore=('y',), then the key generated for f(x=3,y=4) will be identical to that of f(x=3,y=0) or f(x=3,y=10). If ignore=('*','**'), all varargs and varkwds will be 'ignored'. Ignored arguments never trigger a recalculation (they only trigger cache lookups), and thus are 'ignored'. When caching class methods, it may be useful to ignore=('self',). View cache statistics (hit, miss, load, maxsize, size) with f.info(). Clear the cache and statistics with f.clear(). Replace the cache archive with f.archive(obj). Load from the archive with f.load(), and dump from the cache to the archive with f.dump(). """ def __init__(self, maxsize=None, cache=None, keymap=None, ignore=None, tol=None, deep=False, purge=False): #if maxsize is not None: raise ValueError('maxsize cannot be set') maxsize = None #XXX: allow maxsize to be given but ignored ? purge = False #XXX: allow purge to be given but ignored ? if cache is None: cache = archive_dict() elif type(cache) is dict: cache = archive_dict(cache) if keymap is None: keymap = stringmap(flat=False) if ignore is None: ignore = tuple() if deep: rounded = deep_round else: rounded = simple_round #else: rounded = shallow_round #FIXME: slow @rounded(tol) def rounded_args(*args, **kwds): return (args, kwds) # set state self.__state__ = { 'maxsize': maxsize, 'cache': cache, 'keymap': keymap, 'ignore': ignore, 'roundargs': rounded_args, 'tol': tol, 'deep': deep, 'purge': purge, } return def __call__(self, user_function): #cache = dict() # mapping of args to results stats = [0, 0, 0] # make statistics updateable non-locally HIT, MISS, LOAD = 0, 1, 2 # names for the stats fields #_len = len # localize the global len() function #lock = RLock() # linkedlist updates aren't threadsafe maxsize = self.__state__['maxsize'] cache = self.__state__['cache'] keymap = self.__state__['keymap'] ignore = self.__state__['ignore'] rounded_args = self.__state__['roundargs'] def wrapper(*args, **kwds): try: _args, _kwds = rounded_args(*args, **kwds) _args, _kwds = _keygen(user_function, ignore, *_args, **_kwds) key = keymap(*_args, **_kwds) except: #TypeError result = user_function(*args, **kwds) stats[MISS] += 1 return result try: # get cache entry result = cache[key] stats[HIT] += 1 except KeyError: # if not in cache, look in archive if cache.archived(): cache.load(key) try: result = cache[key] stats[LOAD] += 1 except KeyError: # if not found, then compute result = user_function(*args, **kwds) cache[key] = result stats[MISS] += 1 except: #TypeError: # unhashable key result = user_function(*args, **kwds) stats[MISS] += 1 return result def archive(obj): """Replace the cache archive""" if isinstance(obj, archive_dict): cache.archive = obj.archive else: cache.archive = obj def key(*args, **kwds): """Get the cache key for the given *args,**kwds""" _args, _kwds = rounded_args(*args, **kwds) _args, _kwds = _keygen(user_function, ignore, *_args, **_kwds) return keymap(*_args, **_kwds) def lookup(*args, **kwds): """Get the stored value for the given *args,**kwds""" _args, _kwds = rounded_args(*args, **kwds) _args, _kwds = _keygen(user_function, ignore, *_args, **_kwds) return cache[keymap(*_args, **_kwds)] def __get_cache(): """Get the cache""" return cache def __get_mask(): """Get the (ignore) mask""" return ignore def __get_keymap(): """Get the keymap""" return keymap def clear(keepstats=False): """Clear the cache and statistics""" cache.clear() if not keepstats: stats[:] = [0, 0, 0] def info(): """Report cache statistics""" return CacheInfo(stats[HIT], stats[MISS], stats[LOAD], maxsize, len(cache)) # interface wrapper.__wrapped__ = user_function #XXX: better is handle to key_function=keygen(ignore)(user_function) ? wrapper.info = info wrapper.clear = clear wrapper.load = cache.load wrapper.dump = cache.dump wrapper.archive = archive wrapper.archived = cache.archived wrapper.key = key wrapper.lookup = lookup wrapper.__cache__ = __get_cache wrapper.__mask__ = __get_mask wrapper.__map__ = __get_keymap #wrapper._queue = None #XXX return update_wrapper(wrapper, user_function) def __get__(self, obj, objtype): """support instance methods""" return partial(self.__call__, obj) def __reduce__(self): cache = self.__state__['cache'] keymap = self.__state__['keymap'] ignore = self.__state__['ignore'] tol = self.__state__['tol'] deep = self.__state__['deep'] return (self.__class__, (None, cache, keymap, ignore, tol, deep, False)) class lfu_cache(object): """'safe' version of the least-frequenty-used (LFU) cache decorator. This decorator memoizes a function's return value each time it is called. If called later with the same arguments, the cached value is returned, and not re-evaluated. To avoid memory issues, a maximum cache size is imposed. For caches with an archive, the full cache dumps to archive upon reaching maxsize. For caches without an archive, the LFU algorithm manages the cache. Caches with an archive will use the latter behavior when 'purge' is False. This decorator takes an integer tolerance 'tol', equal to the number of decimal places to which it will round off floats, and a bool 'deep' for whether the rounding on inputs will be 'shallow' or 'deep'. Note that rounding is not applied to the calculation of new results, but rather as a simple form of cache interpolation. For example, with tol=0 and a cached value for f(3.0), f(3.1) will lookup f(3.0) in the cache while f(3.6) will store a new value; however if tol=1, both f(3.1) and f(3.6) will store new values. maxsize = maximum cache size cache = storage hashmap (default is {}) keymap = cache key encoder (default is keymaps.stringmap(flat=False)) ignore = function argument names and indicies to 'ignore' (default is None) tol = integer tolerance for rounding (default is None) deep = boolean for rounding depth (default is False, i.e. 'shallow') purge = boolean for purge cache to archive at maxsize (default is False) If *maxsize* is None, this cache will grow without bound. If *keymap* is given, it will replace the hashing algorithm for generating cache keys. Several hashing algorithms are available in 'keymaps'. The default keymap does not require arguments to the cached function to be hashable. If a hashing error occurs, the cached function will be evaluated. If the keymap retains type information, then arguments of different types will be cached separately. For example, f(3.0) and f(3) will be treated as distinct calls with distinct results. Cache typing has a memory penalty, and may also be ignored by some 'keymaps'. If *ignore* is given, the keymap will ignore the arguments with the names and/or positional indicies provided. For example, if ignore=(0,), then the key generated for f(1,2) will be identical to that of f(3,2) or f(4,2). If ignore=('y',), then the key generated for f(x=3,y=4) will be identical to that of f(x=3,y=0) or f(x=3,y=10). If ignore=('*','**'), all varargs and varkwds will be 'ignored'. Ignored arguments never trigger a recalculation (they only trigger cache lookups), and thus are 'ignored'. When caching class methods, it may be useful to ignore=('self',). View cache statistics (hit, miss, load, maxsize, size) with f.info(). Clear the cache and statistics with f.clear(). Replace the cache archive with f.archive(obj). Load from the archive with f.load(), and dump from the cache to the archive with f.dump(). See: http://en.wikipedia.org/wiki/Cache_algorithms#Least_Frequently_Used """ def __new__(cls, *args, **kwds): maxsize = kwds.get('maxsize', -1) if maxsize == 0: return no_cache(*args, **kwds) if maxsize is None: return inf_cache(*args, **kwds) return object.__new__(cls) def __init__(self, maxsize=100, cache=None, keymap=None, ignore=None, tol=None, deep=False, purge=False): if maxsize is None or maxsize == 0: return if cache is None: cache = archive_dict() elif type(cache) is dict: cache = archive_dict(cache) if keymap is None: keymap = stringmap(flat=False) if ignore is None: ignore = tuple() if deep: rounded = deep_round else: rounded = simple_round #else: rounded = shallow_round #FIXME: slow @rounded(tol) def rounded_args(*args, **kwds): return (args, kwds) # set state self.__state__ = { 'maxsize': maxsize, 'cache': cache, 'keymap': keymap, 'ignore': ignore, 'roundargs': rounded_args, 'tol': tol, 'deep': deep, 'purge': purge, } return def __call__(self, user_function): from heapq import nsmallest from operator import itemgetter #cache = dict() # mapping of args to results use_count = Counter() # times each key has been accessed stats = [0, 0, 0] # make statistics updateable non-locally HIT, MISS, LOAD = 0, 1, 2 # names for the stats fields _len = len # localize the global len() function #lock = RLock() # linkedlist updates aren't threadsafe maxsize = self.__state__['maxsize'] cache = self.__state__['cache'] keymap = self.__state__['keymap'] ignore = self.__state__['ignore'] rounded_args = self.__state__['roundargs'] purge = self.__state__['purge'] def wrapper(*args, **kwds): try: _args, _kwds = rounded_args(*args, **kwds) _args, _kwds = _keygen(user_function, ignore, *_args, **_kwds) key = keymap(*_args, **_kwds) except: #TypeError result = user_function(*args, **kwds) stats[MISS] += 1 return result try: # get cache entry result = cache[key] use_count[key] += 1 stats[HIT] += 1 except KeyError: # if not in cache, look in archive if cache.archived(): cache.load(key) try: result = cache[key] use_count[key] += 1 stats[LOAD] += 1 except KeyError: # if not found, then compute result = user_function(*args, **kwds) cache[key] = result use_count[key] += 1 stats[MISS] += 1 # purge cache if _len(cache) > maxsize: #XXX: better: if cache is cache.archive ? if cache.archived() and purge: cache.dump() cache.clear() use_count.clear() else: # purge least frequent cache entries for k, _ in nsmallest(max(2, maxsize // 10), iter(use_count.items()), key=itemgetter(1)): if cache.archived(): cache.dump(k) try: del cache[k] except KeyError: pass #FIXME: possible less purged use_count.pop(k, None) except: #TypeError: # unhashable key result = user_function(*args, **kwds) stats[MISS] += 1 return result def archive(obj): """Replace the cache archive""" if isinstance(obj, archive_dict): cache.archive = obj.archive else: cache.archive = obj def key(*args, **kwds): """Get the cache key for the given *args,**kwds""" _args, _kwds = rounded_args(*args, **kwds) _args, _kwds = _keygen(user_function, ignore, *_args, **_kwds) return keymap(*_args, **_kwds) def lookup(*args, **kwds): """Get the stored value for the given *args,**kwds""" _args, _kwds = rounded_args(*args, **kwds) _args, _kwds = _keygen(user_function, ignore, *_args, **_kwds) return cache[keymap(*_args, **_kwds)] def __get_cache(): """Get the cache""" return cache def __get_mask(): """Get the (ignore) mask""" return ignore def __get_keymap(): """Get the keymap""" return keymap def clear(keepstats=False): """Clear the cache and statistics""" cache.clear() use_count.clear() if not keepstats: stats[:] = [0, 0, 0] def info(): """Report cache statistics""" return CacheInfo(stats[HIT], stats[MISS], stats[LOAD], maxsize, len(cache)) # interface wrapper.__wrapped__ = user_function #XXX: better is handle to key_function=keygen(ignore)(user_function) ? wrapper.info = info wrapper.clear = clear wrapper.load = cache.load wrapper.dump = cache.dump wrapper.archive = archive wrapper.archived = cache.archived wrapper.key = key wrapper.lookup = lookup wrapper.__cache__ = __get_cache wrapper.__mask__ = __get_mask wrapper.__map__ = __get_keymap #wrapper._queue = use_count #XXX return update_wrapper(wrapper, user_function) def __get__(self, obj, objtype): """support instance methods""" return partial(self.__call__, obj) def __reduce__(self): maxsize = self.__state__['maxsize'] cache = self.__state__['cache'] keymap = self.__state__['keymap'] ignore = self.__state__['ignore'] tol = self.__state__['tol'] deep = self.__state__['deep'] purge = self.__state__['purge'] return (self.__class__, (maxsize, cache, keymap, ignore, tol, deep, purge)) class lru_cache(object): """'safe' version of the least-recently-used (LRU) cache decorator. This decorator memoizes a function's return value each time it is called. If called later with the same arguments, the cached value is returned, and not re-evaluated. To avoid memory issues, a maximum cache size is imposed. For caches with an archive, the full cache dumps to archive upon reaching maxsize. For caches without an archive, the LRU algorithm manages the cache. Caches with an archive will use the latter behavior when 'purge' is False. This decorator takes an integer tolerance 'tol', equal to the number of decimal places to which it will round off floats, and a bool 'deep' for whether the rounding on inputs will be 'shallow' or 'deep'. Note that rounding is not applied to the calculation of new results, but rather as a simple form of cache interpolation. For example, with tol=0 and a cached value for f(3.0), f(3.1) will lookup f(3.0) in the cache while f(3.6) will store a new value; however if tol=1, both f(3.1) and f(3.6) will store new values. maxsize = maximum cache size cache = storage hashmap (default is {}) keymap = cache key encoder (default is keymaps.stringmap(flat=False)) ignore = function argument names and indicies to 'ignore' (default is None) tol = integer tolerance for rounding (default is None) deep = boolean for rounding depth (default is False, i.e. 'shallow') purge = boolean for purge cache to archive at maxsize (default is False) If *maxsize* is None, this cache will grow without bound. If *keymap* is given, it will replace the hashing algorithm for generating cache keys. Several hashing algorithms are available in 'keymaps'. The default keymap does not require arguments to the cached function to be hashable. If a hashing error occurs, the cached function will be evaluated. If the keymap retains type information, then arguments of different types will be cached separately. For example, f(3.0) and f(3) will be treated as distinct calls with distinct results. Cache typing has a memory penalty, and may also be ignored by some 'keymaps'. If *ignore* is given, the keymap will ignore the arguments with the names and/or positional indicies provided. For example, if ignore=(0,), then the key generated for f(1,2) will be identical to that of f(3,2) or f(4,2). If ignore=('y',), then the key generated for f(x=3,y=4) will be identical to that of f(x=3,y=0) or f(x=3,y=10). If ignore=('*','**'), all varargs and varkwds will be 'ignored'. Ignored arguments never trigger a recalculation (they only trigger cache lookups), and thus are 'ignored'. When caching class methods, it may be useful to ignore=('self',). View cache statistics (hit, miss, load, maxsize, size) with f.info(). Clear the cache and statistics with f.clear(). Replace the cache archive with f.archive(obj). Load from the archive with f.load(), and dump from the cache to the archive with f.dump(). See: http://en.wikipedia.org/wiki/Cache_algorithms#Least_Recently_Used """ def __new__(cls, *args, **kwds): maxsize = kwds.get('maxsize', -1) if maxsize == 0: return no_cache(*args, **kwds) if maxsize is None: return inf_cache(*args, **kwds) return object.__new__(cls) def __init__(self, maxsize=100, cache=None, keymap=None, ignore=None, tol=None, deep=False, purge=False): if maxsize is None or maxsize == 0: return if cache is None: cache = archive_dict() elif type(cache) is dict: cache = archive_dict(cache) if keymap is None: keymap = stringmap(flat=False) if ignore is None: ignore = tuple() if deep: rounded = deep_round else: rounded = simple_round #else: rounded = shallow_round #FIXME: slow @rounded(tol) def rounded_args(*args, **kwds): return (args, kwds) # set state self.__state__ = { 'maxsize': maxsize, 'cache': cache, 'keymap': keymap, 'ignore': ignore, 'roundargs': rounded_args, 'tol': tol, 'deep': deep, 'purge': purge, } return def __call__(self, user_function): from collections import deque from itertools import filterfalse #cache = dict() # mapping of args to results queue = deque() # order that keys have been used refcount = Counter() # times each key is in the queue sentinel = object() # marker for looping around the queue stats = [0, 0, 0] # make statistics updateable non-locally HIT, MISS, LOAD = 0, 1, 2 # names for the stats fields _len = len # localize the global len() function #lock = RLock() # linkedlist updates aren't threadsafe maxsize = self.__state__['maxsize'] cache = self.__state__['cache'] keymap = self.__state__['keymap'] ignore = self.__state__['ignore'] rounded_args = self.__state__['roundargs'] purge = self.__state__['purge'] maxqueue = maxsize * 10 #XXX: settable? confirm this works as expected # lookup optimizations (ugly but fast) queue_append, queue_popleft = queue.append, queue.popleft queue_appendleft, queue_pop = queue.appendleft, queue.pop def wrapper(*args, **kwds): try: _args, _kwds = rounded_args(*args, **kwds) _args, _kwds = _keygen(user_function, ignore, *_args, **_kwds) key = keymap(*_args, **_kwds) except: #TypeError result = user_function(*args, **kwds) stats[MISS] += 1 return result try: # get cache entry result = cache[key] # record recent use of this key queue_append(key) refcount[key] += 1 stats[HIT] += 1 except KeyError: # if not in cache, look in archive if cache.archived(): cache.load(key) try: result = cache[key] # record recent use of this key queue_append(key) refcount[key] += 1 stats[LOAD] += 1 except KeyError: # if not found, then compute result = user_function(*args, **kwds) cache[key] = result # record recent use of this key queue_append(key) refcount[key] += 1 stats[MISS] += 1 # purge cache if _len(cache) > maxsize: #XXX: better: if cache is cache.archive ? if cache.archived() and purge: cache.dump() cache.clear() queue.clear() refcount.clear() else: # purge least recently used cache entry key = queue_popleft() refcount[key] -= 1 while refcount[key]: key = queue_popleft() refcount[key] -= 1 if cache.archived(): cache.dump(key) try: del cache[key] except KeyError: pass #FIXME: possible none purged refcount.pop(key, None) except: #TypeError: # unhashable key result = user_function(*args, **kwds) stats[MISS] += 1 return result # periodically compact the queue by eliminating duplicate keys # while preserving order of most recent access if _len(queue) > maxqueue: refcount.clear() queue_appendleft(sentinel) for key in filterfalse(refcount.__contains__, iter(queue_pop, sentinel)): queue_appendleft(key) refcount[key] = 1 return result def archive(obj): """Replace the cache archive""" if isinstance(obj, archive_dict): cache.archive = obj.archive else: cache.archive = obj def key(*args, **kwds): """Get the cache key for the given *args,**kwds""" _args, _kwds = rounded_args(*args, **kwds) _args, _kwds = _keygen(user_function, ignore, *_args, **_kwds) return keymap(*_args, **_kwds) def lookup(*args, **kwds): """Get the stored value for the given *args,**kwds""" _args, _kwds = rounded_args(*args, **kwds) _args, _kwds = _keygen(user_function, ignore, *_args, **_kwds) return cache[keymap(*_args, **_kwds)] def __get_cache(): """Get the cache""" return cache def __get_mask(): """Get the (ignore) mask""" return ignore def __get_keymap(): """Get the keymap""" return keymap def clear(keepstats=False): """Clear the cache and statistics""" cache.clear() queue.clear() refcount.clear() if not keepstats: stats[:] = [0, 0, 0] def info(): """Report cache statistics""" return CacheInfo(stats[HIT], stats[MISS], stats[LOAD], maxsize, len(cache)) # interface wrapper.__wrapped__ = user_function #XXX: better is handle to key_function=keygen(ignore)(user_function) ? wrapper.info = info wrapper.clear = clear wrapper.load = cache.load wrapper.dump = cache.dump wrapper.archive = archive wrapper.archived = cache.archived wrapper.key = key wrapper.lookup = lookup wrapper.__cache__ = __get_cache wrapper.__mask__ = __get_mask wrapper.__map__ = __get_keymap #wrapper._queue = queue #XXX return update_wrapper(wrapper, user_function) def __get__(self, obj, objtype): """support instance methods""" return partial(self.__call__, obj) def __reduce__(self): maxsize = self.__state__['maxsize'] cache = self.__state__['cache'] keymap = self.__state__['keymap'] ignore = self.__state__['ignore'] tol = self.__state__['tol'] deep = self.__state__['deep'] purge = self.__state__['purge'] return (self.__class__, (maxsize, cache, keymap, ignore, tol, deep, purge)) class mru_cache(object): """'safe' version of the most-recently-used (MRU) cache decorator. This decorator memoizes a function's return value each time it is called. If called later with the same arguments, the cached value is returned, and not re-evaluated. To avoid memory issues, a maximum cache size is imposed. For caches with an archive, the full cache dumps to archive upon reaching maxsize. For caches without an archive, the MRU algorithm manages the cache. Caches with an archive will use the latter behavior when 'purge' is False. This decorator takes an integer tolerance 'tol', equal to the number of decimal places to which it will round off floats, and a bool 'deep' for whether the rounding on inputs will be 'shallow' or 'deep'. Note that rounding is not applied to the calculation of new results, but rather as a simple form of cache interpolation. For example, with tol=0 and a cached value for f(3.0), f(3.1) will lookup f(3.0) in the cache while f(3.6) will store a new value; however if tol=1, both f(3.1) and f(3.6) will store new values. maxsize = maximum cache size cache = storage hashmap (default is {}) keymap = cache key encoder (default is keymaps.stringmap(flat=False)) ignore = function argument names and indicies to 'ignore' (default is None) tol = integer tolerance for rounding (default is None) deep = boolean for rounding depth (default is False, i.e. 'shallow') purge = boolean for purge cache to archive at maxsize (default is False) If *maxsize* is None, this cache will grow without bound. If *keymap* is given, it will replace the hashing algorithm for generating cache keys. Several hashing algorithms are available in 'keymaps'. The default keymap does not require arguments to the cached function to be hashable. If a hashing error occurs, the cached function will be evaluated. If the keymap retains type information, then arguments of different types will be cached separately. For example, f(3.0) and f(3) will be treated as distinct calls with distinct results. Cache typing has a memory penalty, and may also be ignored by some 'keymaps'. If *ignore* is given, the keymap will ignore the arguments with the names and/or positional indicies provided. For example, if ignore=(0,), then the key generated for f(1,2) will be identical to that of f(3,2) or f(4,2). If ignore=('y',), then the key generated for f(x=3,y=4) will be identical to that of f(x=3,y=0) or f(x=3,y=10). If ignore=('*','**'), all varargs and varkwds will be 'ignored'. Ignored arguments never trigger a recalculation (they only trigger cache lookups), and thus are 'ignored'. When caching class methods, it may be useful to ignore=('self',). View cache statistics (hit, miss, load, maxsize, size) with f.info(). Clear the cache and statistics with f.clear(). Replace the cache archive with f.archive(obj). Load from the archive with f.load(), and dump from the cache to the archive with f.dump(). See: http://en.wikipedia.org/wiki/Cache_algorithms#Most_Recently_Used """ def __new__(cls, *args, **kwds): maxsize = kwds.get('maxsize', -1) if maxsize == 0: return no_cache(*args, **kwds) if maxsize is None: return inf_cache(*args, **kwds) return object.__new__(cls) def __init__(self, maxsize=100, cache=None, keymap=None, ignore=None, tol=None, deep=False, purge=False): if maxsize is None or maxsize == 0: return if cache is None: cache = archive_dict() elif type(cache) is dict: cache = archive_dict(cache) if keymap is None: keymap = stringmap(flat=False) if ignore is None: ignore = tuple() if deep: rounded = deep_round else: rounded = simple_round #else: rounded = shallow_round #FIXME: slow @rounded(tol) def rounded_args(*args, **kwds): return (args, kwds) # set state self.__state__ = { 'maxsize': maxsize, 'cache': cache, 'keymap': keymap, 'ignore': ignore, 'roundargs': rounded_args, 'tol': tol, 'deep': deep, 'purge': purge, } return def __call__(self, user_function): from collections import deque #cache = dict() # mapping of args to results queue = deque() # order that keys have been used stats = [0, 0, 0] # make statistics updateable non-locally HIT, MISS, LOAD = 0, 1, 2 # names for the stats fields _len = len # localize the global len() function #lock = RLock() # linkedlist updates aren't threadsafe maxsize = self.__state__['maxsize'] cache = self.__state__['cache'] keymap = self.__state__['keymap'] ignore = self.__state__['ignore'] rounded_args = self.__state__['roundargs'] purge = self.__state__['purge'] # lookup optimizations (ugly but fast) queue_append, queue_popleft = queue.append, queue.popleft queue_appendleft, queue_pop = queue.appendleft, queue.pop def wrapper(*args, **kwds): try: _args, _kwds = rounded_args(*args, **kwds) _args, _kwds = _keygen(user_function, ignore, *_args, **_kwds) key = keymap(*_args, **_kwds) except: #TypeError result = user_function(*args, **kwds) stats[MISS] += 1 return result try: # get cache entry result = cache[key] try: queue.remove(key) except ValueError: pass stats[HIT] += 1 except KeyError: # if not in cache, look in archive if cache.archived(): cache.load(key) try: result = cache[key] stats[LOAD] += 1 except KeyError: # if not found, then compute result = user_function(*args, **kwds) cache[key] = result stats[MISS] += 1 # purge cache if _len(cache) > maxsize: #XXX: better: if cache is cache.archive ? if cache.archived() and purge: cache.dump() cache.clear() queue.clear() else: # purge most recently used cache entry k = queue_pop() if cache.archived(): cache.dump(k) try: del cache[k] except KeyError: pass #FIXME: possible none purged except: #TypeError: # unhashable key result = user_function(*args, **kwds) stats[MISS] += 1 return result # record recent use of this key queue_append(key) return result def archive(obj): """Replace the cache archive""" if isinstance(obj, archive_dict): cache.archive = obj.archive else: cache.archive = obj def key(*args, **kwds): """Get the cache key for the given *args,**kwds""" _args, _kwds = rounded_args(*args, **kwds) _args, _kwds = _keygen(user_function, ignore, *_args, **_kwds) return keymap(*_args, **_kwds) def lookup(*args, **kwds): """Get the stored value for the given *args,**kwds""" _args, _kwds = rounded_args(*args, **kwds) _args, _kwds = _keygen(user_function, ignore, *_args, **_kwds) return cache[keymap(*_args, **_kwds)] def __get_cache(): """Get the cache""" return cache def __get_mask(): """Get the (ignore) mask""" return ignore def __get_keymap(): """Get the keymap""" return keymap def clear(keepstats=False): """Clear the cache and statistics""" cache.clear() queue.clear() if not keepstats: stats[:] = [0, 0, 0] def info(): """Report cache statistics""" return CacheInfo(stats[HIT], stats[MISS], stats[LOAD], maxsize, len(cache)) # interface wrapper.__wrapped__ = user_function #XXX: better is handle to key_function=keygen(ignore)(user_function) ? wrapper.info = info wrapper.clear = clear wrapper.load = cache.load wrapper.dump = cache.dump wrapper.archive = archive wrapper.archived = cache.archived wrapper.key = key wrapper.lookup = lookup wrapper.__cache__ = __get_cache wrapper.__mask__ = __get_mask wrapper.__map__ = __get_keymap #wrapper._queue = queue #XXX return update_wrapper(wrapper, user_function) def __get__(self, obj, objtype): """support instance methods""" return partial(self.__call__, obj) def __reduce__(self): maxsize = self.__state__['maxsize'] cache = self.__state__['cache'] keymap = self.__state__['keymap'] ignore = self.__state__['ignore'] tol = self.__state__['tol'] deep = self.__state__['deep'] purge = self.__state__['purge'] return (self.__class__, (maxsize, cache, keymap, ignore, tol, deep, purge)) class rr_cache(object): """'safe' version of the random-replacement (RR) cache decorator. This decorator memoizes a function's return value each time it is called. If called later with the same arguments, the cached value is returned, and not re-evaluated. To avoid memory issues, a maximum cache size is imposed. For caches with an archive, the full cache dumps to archive upon reaching maxsize. For caches without an archive, the RR algorithm manages the cache. Caches with an archive will use the latter behavior when 'purge' is False. This decorator takes an integer tolerance 'tol', equal to the number of decimal places to which it will round off floats, and a bool 'deep' for whether the rounding on inputs will be 'shallow' or 'deep'. Note that rounding is not applied to the calculation of new results, but rather as a simple form of cache interpolation. For example, with tol=0 and a cached value for f(3.0), f(3.1) will lookup f(3.0) in the cache while f(3.6) will store a new value; however if tol=1, both f(3.1) and f(3.6) will store new values. maxsize = maximum cache size cache = storage hashmap (default is {}) keymap = cache key encoder (default is keymaps.stringmap(flat=False)) ignore = function argument names and indicies to 'ignore' (default is None) tol = integer tolerance for rounding (default is None) deep = boolean for rounding depth (default is False, i.e. 'shallow') purge = boolean for purge cache to archive at maxsize (default is False) If *maxsize* is None, this cache will grow without bound. If *keymap* is given, it will replace the hashing algorithm for generating cache keys. Several hashing algorithms are available in 'keymaps'. The default keymap does not require arguments to the cached function to be hashable. If a hashing error occurs, the cached function will be evaluated. If the keymap retains type information, then arguments of different types will be cached separately. For example, f(3.0) and f(3) will be treated as distinct calls with distinct results. Cache typing has a memory penalty, and may also be ignored by some 'keymaps'. If *ignore* is given, the keymap will ignore the arguments with the names and/or positional indicies provided. For example, if ignore=(0,), then the key generated for f(1,2) will be identical to that of f(3,2) or f(4,2). If ignore=('y',), then the key generated for f(x=3,y=4) will be identical to that of f(x=3,y=0) or f(x=3,y=10). If ignore=('*','**'), all varargs and varkwds will be 'ignored'. Ignored arguments never trigger a recalculation (they only trigger cache lookups), and thus are 'ignored'. When caching class methods, it may be useful to ignore=('self',). View cache statistics (hit, miss, load, maxsize, size) with f.info(). Clear the cache and statistics with f.clear(). Replace the cache archive with f.archive(obj). Load from the archive with f.load(), and dump from the cache to the archive with f.dump(). http://en.wikipedia.org/wiki/Cache_algorithms#Random_Replacement """ def __new__(cls, *args, **kwds): maxsize = kwds.get('maxsize', -1) if maxsize == 0: return no_cache(*args, **kwds) if maxsize is None: return inf_cache(*args, **kwds) return object.__new__(cls) def __init__(self, maxsize=100, cache=None, keymap=None, ignore=None, tol=None, deep=False, purge=False): if maxsize is None or maxsize == 0: return if cache is None: cache = archive_dict() elif type(cache) is dict: cache = archive_dict(cache) if keymap is None: keymap = stringmap(flat=False) if ignore is None: ignore = tuple() if deep: rounded = deep_round else: rounded = simple_round #else: rounded = shallow_round #FIXME: slow @rounded(tol) def rounded_args(*args, **kwds): return (args, kwds) # set state self.__state__ = { 'maxsize': maxsize, 'cache': cache, 'keymap': keymap, 'ignore': ignore, 'roundargs': rounded_args, 'tol': tol, 'deep': deep, 'purge': purge, } return def __call__(self, user_function): #cache = dict() # mapping of args to results stats = [0, 0, 0] # make statistics updateable non-locally HIT, MISS, LOAD = 0, 1, 2 # names for the stats fields _len = len # localize the global len() function #lock = RLock() # linkedlist updates aren't threadsafe maxsize = self.__state__['maxsize'] cache = self.__state__['cache'] keymap = self.__state__['keymap'] ignore = self.__state__['ignore'] rounded_args = self.__state__['roundargs'] purge = self.__state__['purge'] def wrapper(*args, **kwds): from random import choice #XXX: biased? try: _args, _kwds = rounded_args(*args, **kwds) _args, _kwds = _keygen(user_function, ignore, *_args, **_kwds) key = keymap(*_args, **_kwds) except: #TypeError result = user_function(*args, **kwds) stats[MISS] += 1 return result try: # get cache entry result = cache[key] stats[HIT] += 1 except KeyError: # if not in cache, look in archive if cache.archived(): cache.load(key) try: result = cache[key] stats[LOAD] += 1 except KeyError: # if not found, then compute result = user_function(*args, **kwds) cache[key] = result stats[MISS] += 1 # purge cache if _len(cache) > maxsize: #XXX: better: if cache is cache.archive ? if cache.archived() and purge: cache.dump() cache.clear() else: # purge random cache entry key = choice(list(cache.keys())) if cache.archived(): cache.dump(key) try: del cache[key] except KeyError: pass #FIXME: possible none purged except: #TypeError: # unhashable key result = user_function(*args, **kwds) stats[MISS] += 1 return result def archive(obj): """Replace the cache archive""" if isinstance(obj, archive_dict): cache.archive = obj.archive else: cache.archive = obj def key(*args, **kwds): """Get the cache key for the given *args,**kwds""" _args, _kwds = rounded_args(*args, **kwds) _args, _kwds = _keygen(user_function, ignore, *_args, **_kwds) return keymap(*_args, **_kwds) def lookup(*args, **kwds): """Get the stored value for the given *args,**kwds""" _args, _kwds = rounded_args(*args, **kwds) _args, _kwds = _keygen(user_function, ignore, *_args, **_kwds) return cache[keymap(*_args, **_kwds)] def __get_cache(): """Get the cache""" return cache def __get_mask(): """Get the (ignore) mask""" return ignore def __get_keymap(): """Get the keymap""" return keymap def clear(keepstats=False): """Clear the cache and statistics""" cache.clear() if not keepstats: stats[:] = [0, 0, 0] def info(): """Report cache statistics""" return CacheInfo(stats[HIT], stats[MISS], stats[LOAD], maxsize, len(cache)) # interface wrapper.__wrapped__ = user_function #XXX: better is handle to key_function=keygen(ignore)(user_function) ? wrapper.info = info wrapper.clear = clear wrapper.load = cache.load wrapper.dump = cache.dump wrapper.archive = archive wrapper.archived = cache.archived wrapper.key = key wrapper.lookup = lookup wrapper.__cache__ = __get_cache wrapper.__mask__ = __get_mask wrapper.__map__ = __get_keymap #wrapper._queue = None #XXX return update_wrapper(wrapper, user_function) def __get__(self, obj, objtype): """support instance methods""" return partial(self.__call__, obj) def __reduce__(self): maxsize = self.__state__['maxsize'] cache = self.__state__['cache'] keymap = self.__state__['keymap'] ignore = self.__state__['ignore'] tol = self.__state__['tol'] deep = self.__state__['deep'] purge = self.__state__['purge'] return (self.__class__, (maxsize, cache, keymap, ignore, tol, deep, purge)) if __name__ == '__main__': import dill #@no_cache(10, tol=0) #@inf_cache(10, tol=0) #@lfu_cache(10, tol=0) #@lru_cache(10, tol=0) #@mru_cache(10, tol=0) @rr_cache(10, tol=0) def squared(x): return x**2 res = squared(10) assert res == dill.loads(dill.dumps(squared))(10) # EOF uqfoundation-klepto-69cd6ce/klepto/tests/000077500000000000000000000000001455531556400207045ustar00rootroot00000000000000uqfoundation-klepto-69cd6ce/klepto/tests/__init__.py000066400000000000000000000007451455531556400230230ustar00rootroot00000000000000#!/usr/bin/env python # # Author: Mike McKerns (mmckerns @caltech and @uqfoundation) # Copyright (c) 2018-2024 The Uncertainty Quantification Foundation. # License: 3-clause BSD. The full license text is available at: # - https://github.com/uqfoundation/klepto/blob/master/LICENSE """ to run this test suite, first build and install `klepto`. $ python -m pip install ../.. then run the tests with: $ python -m klepto.tests or, if `nose` is installed: $ nosetests """ uqfoundation-klepto-69cd6ce/klepto/tests/__main__.py000066400000000000000000000016051455531556400230000ustar00rootroot00000000000000#!/usr/bin/env python # # Author: Mike McKerns (mmckerns @caltech and @uqfoundation) # Copyright (c) 2018-2024 The Uncertainty Quantification Foundation. # License: 3-clause BSD. The full license text is available at: # - https://github.com/uqfoundation/klepto/blob/master/LICENSE import glob import os import sys import subprocess as sp python = sys.executable try: import pox python = pox.which_python(version=True) or python except ImportError: pass shell = sys.platform[:3] == 'win' suite = os.path.dirname(__file__) or os.path.curdir tests = glob.glob(suite + os.path.sep + 'test_*.py') if __name__ == '__main__': failed = 0 for test in tests: p = sp.Popen([python, test], shell=shell).wait() if p: print('F', end='', flush=True) failed = 1 else: print('.', end='', flush=True) print('') exit(failed) uqfoundation-klepto-69cd6ce/klepto/tests/cleanup_basic.py000066400000000000000000000010051455531556400240420ustar00rootroot00000000000000#!/usr/bin/env python # # Author: Mike McKerns (mmckerns @caltech and @uqfoundation) # Copyright (c) 2015-2016 California Institute of Technology. # Copyright (c) 2016-2020 The Uncertainty Quantification Foundation. # License: 3-clause BSD. The full license text is available at: # - https://github.com/uqfoundation/klepto/blob/master/LICENSE from test_basic import _cleanup _cleanup() from test_hdf import _cleanup as _cleanme _cleanme() from test_frame import _cleanup as _cleanframe _cleanframe() uqfoundation-klepto-69cd6ce/klepto/tests/test_alchemy.py000066400000000000000000000052411455531556400237410ustar00rootroot00000000000000#!/usr/bin/env python # # Author: Mike McKerns (mmckerns @caltech and @uqfoundation) # Copyright (c) 2013-2016 California Institute of Technology. # Copyright (c) 2016-2024 The Uncertainty Quantification Foundation. # License: 3-clause BSD. The full license text is available at: # - https://github.com/uqfoundation/klepto/blob/master/LICENSE from klepto.archives import sqltable_archive as sqltable from klepto.archives import sql_archive as sql try: import sqlalchemy __alchemy = True except ImportError: __alchemy = False try: import psycopg2 __postgresql = True except ImportError: __postgresql = False try: import MySQLdb __mysqldb = True except ImportError: __mysqldb = False def test_basic(d): d['a'] = 1 d['b'] = '1' assert d['a'] == 1 assert d['b'] == '1' def test_alchemy(d): if __alchemy: d['c'] = min squared = lambda x:x**2 d['d'] = squared assert d['c'] == min assert d['d'](2) == squared(2) else: print("for greater capabilities, install sqlalchemy") def test_methods(d): if __alchemy: # __delitem__ del d['d'] # copy test = d.copy() # __eq__ assert d == test # __getitem__ i = d['a'] # get pop assert d.get('a') == d.pop('a') # __ne__ __setitem__ assert d != test d['a'] = i # __contains__ assert 'a' in d # fromkeys assert d.fromkeys('abc') == dict(a=None, b=None, c=None) # keys assert set(d.keys()) == set(test.keys()) # items assert set(d.items()) == set(test.items()) # values assert set(d.values()) == set(test.values()) # popitem setdefault d.setdefault(*d.popitem()) assert d == test # update popkeys`` d.update({'z':0}) d.popkeys(['z']) assert d == test # __iter__ assert next(iter(d)) in d # clear d.clear() # __asdict__ assert d.__asdict__() == {} # __len__ assert len(d) == 0 # __drop__ d.__drop__() else: pass def test_new(): if __alchemy: if __postgresql: z = sql('postgresql://user:pass@localhost/defaultdb', cached=False) z.__drop__() if __mysqldb: z = sql('mysql://user:pass@localhost/defaultdb', cached=False) z.__drop__() else: pass if __name__ == '__main__': test_new() z = sqltable(cached=False) test_basic(z) test_alchemy(z) test_methods(z) z = sql(cached=False) test_basic(z) test_alchemy(z) test_methods(z) uqfoundation-klepto-69cd6ce/klepto/tests/test_basic.py000066400000000000000000000132011455531556400233730ustar00rootroot00000000000000#!/usr/bin/env python # # Author: Mike McKerns (mmckerns @caltech and @uqfoundation) # Copyright (c) 2013-2016 California Institute of Technology. # Copyright (c) 2016-2024 The Uncertainty Quantification Foundation. # License: 3-clause BSD. The full license text is available at: # - https://github.com/uqfoundation/klepto/blob/master/LICENSE from klepto.safe import lru_cache as memoized from random import choice, seed N = 30 def _test_cache(cache, keymap=None, maxsize=50, rangelimit=10, tries=N): @memoized(maxsize=maxsize, cache=cache, keymap=keymap) def f(x, y): return 3*x+y domain = list(range(rangelimit)) domain += [float(i) for i in domain] for i in range(tries): r = f(choice(domain), choice(domain)) f.dump() return f def _cleanup(): import os import pox try: os.remove('memo.pkl') except: pass try: os.remove('xxxx.pkl') except: pass try: os.remove('memo.py') except: pass try: os.remove('memo.pyc') except: pass try: os.remove('memo.pyo') except: pass try: os.remove('memo.pyd') except: pass try: os.remove('xxxx.py') except: pass try: os.remove('xxxx.pyc') except: pass try: os.remove('xxxx.pyo') except: pass try: os.remove('xxxx.pyd') except: pass try: os.remove('memo.db') except: pass try: pox.rmtree('memoi') except: pass try: pox.rmtree('memoj') except: pass try: pox.rmtree('memom') except: pass try: pox.rmtree('memop') except: pass try: pox.rmtree('memoz') except: pass try: pox.rmtree('memo') except: pass return from klepto.archives import * from klepto.keymaps import keymap, hashmap, stringmap, picklemap from klepto.keymaps import SENTINEL, NOSENTINEL def test_combinations(): seed(1234) # random seed #XXX: archive/cache should allow scalar and list, also dict (as new table) ? dicts = [ {}, {'a':1}, {'a':[1,2]}, {'a':{'x':3}}, ] init = dicts[0] archives = [ null_archive(None,init), dict_archive(None,init), file_archive(None,init,serialized=True), file_archive(None,init,serialized=False), file_archive('xxxx.pkl',init,serialized=True), file_archive('xxxx.py',init,serialized=False), dir_archive('memoi',init,serialized=False), dir_archive('memop',init,serialized=True), dir_archive('memoj',init,serialized=True,fast=True), dir_archive('memoz',init,serialized=True,compression=1), dir_archive('memom',init,serialized=True,memmode='r+'), #sqltable_archive(None,init), #sqltable_archive('sqlite:///memo.db',init), #sqltable_archive('memo',init), #sql_archive(None,init), #sql_archive('sqlite:///memo.db',init), #sql_archive('memo',init), ] #FIXME: even 'safe' archives throw Error when cache.load, cache.dump fails # (often demonstrated in sqltable_archive, as barfs on tuple & dict) #XXX: when running a single map, there should be 3 possible results: # 1) flat=False may produce unhashable keys: all misses # 2) typed=False doesn't distinguish float & int: more hits & loads # 3) typed=True distingushes float & int: less hits & loads #XXX: due to the seed, each of the 3 cases should yield the same results maps = [ None, keymap(typed=False, flat=True, sentinel=NOSENTINEL), keymap(typed=False, flat=False, sentinel=NOSENTINEL), #FIXME: keymap of (typed=True,flat=True) fails w/ dir_archive on Windows b/c # keymap(typed=True, flat=True, sentinel=NOSENTINEL), # bad directory name? keymap(typed=True, flat=False, sentinel=NOSENTINEL), #keymap(typed=False, flat=True, sentinel=SENTINEL), #keymap(typed=False, flat=False, sentinel=SENTINEL), #keymap(typed=True, flat=True, sentinel=SENTINEL), #keymap(typed=True, flat=False, sentinel=SENTINEL), hashmap(typed=False, flat=True, sentinel=NOSENTINEL), hashmap(typed=False, flat=False, sentinel=NOSENTINEL), hashmap(typed=True, flat=True, sentinel=NOSENTINEL), hashmap(typed=True, flat=False, sentinel=NOSENTINEL), #hashmap(typed=False, flat=True, sentinel=SENTINEL), #hashmap(typed=False, flat=False, sentinel=SENTINEL), #hashmap(typed=True, flat=True, sentinel=SENTINEL), #hashmap(typed=True, flat=False, sentinel=SENTINEL), stringmap(typed=False, flat=True, sentinel=NOSENTINEL), stringmap(typed=False, flat=False, sentinel=NOSENTINEL), stringmap(typed=True, flat=True, sentinel=NOSENTINEL), stringmap(typed=True, flat=False, sentinel=NOSENTINEL), #stringmap(typed=False, flat=True, sentinel=SENTINEL), #stringmap(typed=False, flat=False, sentinel=SENTINEL), #stringmap(typed=True, flat=True, sentinel=SENTINEL), #stringmap(typed=True, flat=False, sentinel=SENTINEL), picklemap(typed=False, flat=True, sentinel=NOSENTINEL), picklemap(typed=False, flat=False, sentinel=NOSENTINEL), picklemap(typed=True, flat=True, sentinel=NOSENTINEL), picklemap(typed=True, flat=False, sentinel=NOSENTINEL), #picklemap(typed=False, flat=True, sentinel=SENTINEL), #picklemap(typed=False, flat=False, sentinel=SENTINEL), #picklemap(typed=True, flat=True, sentinel=SENTINEL), #picklemap(typed=True, flat=False, sentinel=SENTINEL), ] #XXX: should have option to serialize value (as well as key) ? for mapper in maps: #print (mapper) func = [_test_cache(cache, mapper) for cache in archives] _cleanup() for f in func: #print (f.info()) assert f.info().hit + f.info().miss + f.info().load == N if __name__ == '__main__': test_combinations() uqfoundation-klepto-69cd6ce/klepto/tests/test_bigdata.py000066400000000000000000000015031455531556400237070ustar00rootroot00000000000000#!/usr/bin/env python # # Author: Mike McKerns (mmckerns @caltech and @uqfoundation) # Copyright (c) 2013-2016 California Institute of Technology. # Copyright (c) 2016-2024 The Uncertainty Quantification Foundation. # License: 3-clause BSD. The full license text is available at: # - https://github.com/uqfoundation/klepto/blob/master/LICENSE from klepto.keymaps import * h = hashmap(algorithm='md5') p = picklemap(serializer='dill') hp = p + h def test_bigdata(): try: import numpy as np x = np.arange(2000) y = x.copy() y[1000] = -1 assert h(x) == h(y) # equal because repr for large np arrays uses '...' assert p(x) != p(y) assert hp(x) != hp(y) except ImportError: print("to test big data, install numpy") if __name__ == '__main__': test_bigdata() uqfoundation-klepto-69cd6ce/klepto/tests/test_cache.py000066400000000000000000000165271455531556400233730ustar00rootroot00000000000000#!/usr/bin/env python # # Author: Mike McKerns (mmckerns @caltech and @uqfoundation) # Copyright (c) 2013-2016 California Institute of Technology. # Copyright (c) 2016-2024 The Uncertainty Quantification Foundation. # License: 3-clause BSD. The full license text is available at: # - https://github.com/uqfoundation/klepto/blob/master/LICENSE """ The decorator should produce the behavior as displayed in the following: >>> s = Spam() >>> s.eggs() new: (), {} 42 >>> s.eggs() 42 >>> s.eggs(1) new: (1,), {} 64 >>> s.eggs(1) 64 >>> s.eggs(1, bar='spam') new: (1,), {'bar': 'spam'} 78 >>> s2 = Spam() >>> s2.eggs(1, bar='spam') 78 """ from klepto.safe import inf_cache as memoized #from klepto import inf_cache as memoized from klepto.keymaps import picklemap dumps = picklemap(flat=False, serializer='dill') class Spam(object): """A simple class with a memoized method""" @memoized(keymap=dumps, ignore='self') def eggs(self, *args, **kwds): #print ('new:', args, kwds) from random import random return int(100 * random()) def test_classmethod(): s = Spam() assert s.eggs() == s.eggs() assert s.eggs(1) == s.eggs(1) s2 = Spam() assert s.eggs(1, bar='spam') == s2.eggs(1, bar='spam') assert s.eggs.info().hit == 3 assert s.eggs.info().miss == 3 assert s.eggs.info().load == 0 #print ('=' * 30) # here caching saves time in a recursive function... @memoized(keymap=dumps) def fibonacci(n): "Return the nth fibonacci number." #print ('calculating %s' % n) if n in (0, 1): return n return fibonacci(n-1) + fibonacci(n-2) def test_recursive(): fibonacci(7) fibonacci(9) fibonacci(3) assert fibonacci.info().hit == 9 assert fibonacci.info().miss == 10 assert fibonacci.info().load == 0 #print ('=' * 30) def test_basic(): try: from numpy import sum, asarray @memoized(keymap=dumps, tol=3) def add(*args): #print ('new:', args) return sum(args) assert add(1,2,3.0001) == 6.0000999999999998 assert add(1,2,3.00012) == 6.0000999999999998 assert add(1,2,3.0234) == 6.0234000000000005 assert add(1,2,3.023) == 6.0234000000000005 assert add.info().hit == 2 assert add.info().miss == 2 assert add.info().load == 0 def cost(x,y): #print ('new: %s or %s' % (str(x), str(y))) x = asarray(x) y = asarray(y) return sum(x**2 - y**2) cost1 = memoized(keymap=dumps, tol=1)(cost) cost0 = memoized(keymap=dumps, tol=0)(cost) costD = memoized(keymap=dumps, tol=0, deep=True)(cost) #print ("rounding to one decimals...") cost1([1,2,3.1234], 3.9876)# == -32.94723372 cost1([1,2,3.1234], 3.9876)# == -32.94723372 cost1([1,2,3.1234], 3.6789)# == -25.84728807 cost1([1,2,3.4321], 3.6789)# == -23.82360522 assert cost1.info().hit == 1 assert cost1.info().miss == 3 assert cost1.info().load == 0 #print ("\nrerun the above with rounding to zero decimals...") cost0([1,2,3.1234], 3.9876)# == -32.94723372 cost0([1,2,3.1234], 3.9876)# == -32.94723372 cost0([1,2,3.1234], 3.6789)# == -32.94723372 cost0([1,2,3.4321], 3.6789)# == -23.82360522 assert cost0.info().hit == 2 assert cost0.info().miss == 2 assert cost0.info().load == 0 #print ("\nrerun again with deep rounding to zero decimals...") costD([1,2,3.1234], 3.9876)# == -32.94723372 costD([1,2,3.1234], 3.9876)# == -32.94723372 costD([1,2,3.1234], 3.6789)# == -32.94723372 costD([1,2,3.4321], 3.6789)# == -32.94723372 assert costD.info().hit == 3 assert costD.info().miss == 1 assert costD.info().load == 0 #print ("") except ImportError: pass import sys import dill from klepto.archives import cache, sql_archive, dict_archive def test_memoized(): @memoized(cache=sql_archive()) def add(x,y): return x+y add(1,2) add(1,2) add(1,3) #print ("sql_cache = %s" % add.__cache__()) _key4 = '((), '+str({'y':3, 'x':1})+')' _key3 = '((), '+str({'y':2, 'x':1})+')' key4_ = '((), '+str({'x':1, 'y':3})+')' key3_ = '((), '+str({'x':1, 'y':2})+')' assert add.__cache__() == {_key4: 4, _key3: 3} or {key4_: 4, key3_: 3} @memoized(cache=dict_archive(cached=False)) # use archive backend 'direcly' def add(x,y): return x+y add(1,2) add(1,2) add(1,3) #print ("dict_cache = %s" % add.__cache__()) assert add.__cache__() == {_key4: 4, _key3: 3} or {key4_: 4, key3_: 3} @memoized(cache=dict()) def add(x,y): return x+y add(1,2) add(1,2) add(1,3) #print ("dict_cache = %s" % add.__cache__()) assert add.__cache__() == {_key4: 4, _key3: 3} or {key4_: 4, key3_: 3} @memoized(cache=add.__cache__()) def add(x,y): return x+y add(1,2) add(2,2) #print ("re_dict_cache = %s" % add.__cache__()) _key2 = '((), '+str({'y':2, 'x':2})+')' key2_ = '((), '+str({'x':2, 'y':2})+')' assert add.__cache__() == {_key4: 4, _key3: 3, _key2: 4} or {key4_: 4, key3_: 3, key2_: 4} @memoized(keymap=dumps) def add(x,y): return x+y add(1,2) add(1,2) add(1,3) #print ("pickle_dict_cache = %s" % add.__cache__()) _pkey4 = dill.dumps(eval(_key4)) _pkey3 = dill.dumps(eval(_key3)) pkey4_ = dill.dumps(eval(key4_)) pkey3_ = dill.dumps(eval(key3_)) assert add.__cache__() == {_pkey4: 4, _pkey3: 3} or {pkey4_: 4, pkey3_: 3} from klepto import lru_cache def test_lru(): @lru_cache(maxsize=3, cache=dict_archive('test'), purge=True) def identity(x): return x identity(1) identity(2) identity(3) ic = identity.__cache__() assert len(ic.keys()) == 3 assert len(ic.archive.keys()) == 0 identity(4) assert len(ic.keys()) == 0 assert len(ic.archive.keys()) == 4 identity(5) assert len(ic.keys()) == 1 assert len(ic.archive.keys()) == 4 @lru_cache(maxsize=3, cache=dict_archive('test'), purge=False) def inverse(x): return -x inverse(1) inverse(2) inverse(3) ic = inverse.__cache__() assert len(ic.keys()) == 3 assert len(ic.archive.keys()) == 0 inverse(4) assert len(ic.keys()) == 3 assert len(ic.archive.keys()) == 1 inverse(5) assert len(ic.keys()) == 3 assert len(ic.archive.keys()) == 2 @lru_cache(maxsize=3, cache=dict_archive('test', cached=False)) def foo(x): return x foo(1) foo(2) foo(3) ic = foo.__cache__() assert len(ic.keys()) == 3 assert len(ic.archive.keys()) == 3 foo(4) assert len(ic.keys()) == 3 assert len(ic.archive.keys()) == 3 foo(5) assert len(ic.keys()) == 3 assert len(ic.archive.keys()) == 3 #XXX: should it be 'correct' expected behavior to ignore purge? @lru_cache(maxsize=3, cache=None, purge=True) def bar(x): return -x bar(1) bar(2) bar(3) ic = bar.__cache__() assert len(ic.keys()) == 3 assert len(ic.archive.keys()) == 0 bar(4) assert len(ic.keys()) == 3 assert len(ic.archive.keys()) == 0 bar(5) assert len(ic.keys()) == 3 assert len(ic.archive.keys()) == 0 if __name__ == '__main__': test_classmethod() test_recursive() test_basic() test_memoized() test_lru() uqfoundation-klepto-69cd6ce/klepto/tests/test_cache_info.py000066400000000000000000000120421455531556400243720ustar00rootroot00000000000000#!/usr/bin/env python # # Author: Mike McKerns (mmckerns @caltech and @uqfoundation) # Copyright (c) 2013-2016 California Institute of Technology. # Copyright (c) 2016-2024 The Uncertainty Quantification Foundation. # License: 3-clause BSD. The full license text is available at: # - https://github.com/uqfoundation/klepto/blob/master/LICENSE """ test speed and effectiveness of a selection of cache algorithms """ from klepto.archives import file_archive from random import choice, seed def _test_hits(algorithm, maxsize=20, keymap=None, rangelimit=5, tries=1000, archived=False): @algorithm(maxsize=maxsize, keymap=keymap, purge=True) def f(x, y): return 3*x+y if archived: f.archive(file_archive('cache.pkl',cached=False)) domain = list(range(rangelimit)) domain += [float(i) for i in domain] for i in range(tries): r = f(choice(domain), choice(domain)) f.dump() #print(f.info()) return f.info() import os import sys from klepto import * #from klepto.safe import * def test_info(): seed(1234) # random seed caches = [rr_cache,mru_cache,lru_cache,lfu_cache,inf_cache,no_cache] # clean-up if os.path.exists('cache.pkl'): os.remove('cache.pkl') #print ("WITHOUT ARCHIVE") results = [_test_hits(cache, maxsize=100, rangelimit=20, tries=100) for cache in caches] x = results[0] assert (x.hit, x.miss, x.load, x.maxsize, x.size) == (16,84,0,100,84) x = results[1] assert (x.hit, x.miss, x.load, x.maxsize, x.size) == (12,88,0,100,88) x = results[2] assert (x.hit, x.miss, x.load, x.maxsize, x.size) == (11,89,0,100,89) x = results[3] assert (x.hit, x.miss, x.load, x.maxsize, x.size) == (11,89,0,100,89) x = results[4] assert (x.hit, x.miss, x.load, x.maxsize, x.size) == (11,89,0,None,89) x = results[5] assert (x.hit, x.miss, x.load, x.maxsize, x.size) == (0,100,0,0,0) #for cache in caches: # msg = cache.__name__ + ":" # msg += "%s" % str(_test_hits(cache, maxsize=100, # rangelimit=20, tries=100)) # print (msg) #print ("\nWITH ARCHIVE") results = [_test_hits(cache, maxsize=100, rangelimit=20, tries=100, archived=True) for cache in caches] # clean-up if os.path.exists('cache.pkl'): os.remove('cache.pkl') x = results[0] assert (x.hit, x.miss, x.load, x.maxsize, x.size) == (11,89,0,100,89) x = results[1] assert (x.hit, x.miss, x.load, x.maxsize, x.size) == (10,66,24,100,90) x = results[2] assert (x.hit, x.miss, x.load, x.maxsize, x.size) == (10,58,32,100,90) x = results[3] assert (x.hit, x.miss, x.load, x.maxsize, x.size) == (10,37,53,100,90) x = results[4] assert (x.hit, x.miss, x.load, x.maxsize, x.size) == (5,37,58,None,95) x = results[5] assert (x.hit, x.miss, x.load, x.maxsize, x.size) == (0,25,75,0,0) #for cache in caches: # msg = cache.__name__ + ":" # msg += "%s" % str(_test_hits(cache, maxsize=100, # rangelimit=20, tries=100, archived=True)) # print (msg) ### again, w/o purging ### # clean-up if os.path.exists('cache.pkl'): os.remove('cache.pkl') #print ("WITHOUT ARCHIVE") results = [_test_hits(cache, maxsize=50, rangelimit=20, tries=100) for cache in caches] x = results[0] maxsize = x.maxsize assert x.size == x.maxsize # skip due to hash randomization x = results[1] assert x.size == x.maxsize # skip due to hash randomization x = results[2] assert x.size == x.maxsize # skip due to hash randomization x = results[3] assert x.size <= x.maxsize # skip due to hash randomization x = results[4] assert x.size > maxsize # skip due to hash randomization x = results[5] assert (x.hit, x.miss, x.load, x.maxsize, x.size) == (0,100,0,0,0) #for cache in caches: # msg = cache.__name__ + ":" # msg += "%s" % str(_test_hits(cache, maxsize=50, # rangelimit=20, tries=100)) # print (msg) #print ("\nWITH ARCHIVE") results = [_test_hits(cache, maxsize=50, rangelimit=20, tries=100, archived=True) for cache in caches] # clean-up if os.path.exists('cache.pkl'): os.remove('cache.pkl') x = results[0] assert x.size <= x.maxsize # skip due to hash randomization x = results[1] assert x.size <= x.maxsize # skip due to hash randomization x = results[2] assert x.size <= x.maxsize # skip due to hash randomization x = results[3] assert x.size <= x.maxsize # skip due to hash randomization x = results[4] assert x.size > maxsize # skip due to hash randomization x = results[5] assert x.hit == x.maxsize == x.size # skip due to hash randomization #for cache in caches: # msg = cache.__name__ + ":" # msg += "%s" % str(_test_hits(cache, maxsize=50, # rangelimit=20, tries=100, archived=True)) # print (msg) if __name__ == '__main__': test_info() uqfoundation-klepto-69cd6ce/klepto/tests/test_cachekeys.py000066400000000000000000000062011455531556400242530ustar00rootroot00000000000000#!/usr/bin/env python # # Author: Mike McKerns (mmckerns @caltech and @uqfoundation) # Copyright (c) 2014-2016 California Institute of Technology. # Copyright (c) 2016-2024 The Uncertainty Quantification Foundation. # License: 3-clause BSD. The full license text is available at: # - https://github.com/uqfoundation/klepto/blob/master/LICENSE from klepto import inf_cache as memoized from klepto.archives import * from klepto.keymaps import picklemap try: import ___________ #XXX: enable test w/o numpy.arrays import numpy as np data = np.arange(20) nprun = True except ImportError: data = range(20) nprun = False def remove(name): try: import os os.remove(name) except: #FileNotFoundError import pox pox.shutils.rmtree(name, self=True, ignore_errors=True) return archives = [ ### OK null_archive, dict_archive, file_archive, ### FIXME: on numpy array, throws ValueError('I/O operation on closed file') dir_archive, ### FIXME: throws RecursionError NOTE: '\x80' not valid, but r'\x80' is valid # sql_archive, ### FIXME: throws sql ProgrammingError (warns to use unicode strings) # sqltable_archiver, ] def runme(arxiv, expected=None): pm = picklemap(serializer='dill') @memoized(cache=arxiv, keymap=pm) def doit(x): return x doit(1) doit('2') doit(data) doit(lambda x:x**2) doit.load() doit.dump() c = doit.__cache__() r = getattr(c, '__archive__', '') info = doit.info() ck = c.keys() rk = r.keys() if r else ck #print(type(c)) #print(c) #print(r) #print(info) # check keys are identical in cache and archive assert sorted(ck) == sorted(rk) xx = len(ck) or max(info.hit, info.miss, info.load) # check size and behavior if expected == 'hit': assert (info.hit, info.miss, info.load) == (xx, 0, 0) elif expected == 'load': assert (info.hit, info.miss, info.load) == (0, 0, xx) else: assert (info.hit, info.miss, info.load) == (0, xx, 0) return def _test_cache(archive, name, delete=True): arname = 'xxxxxx'+ str(name) acname = 'xxxyyy'+ str(name) import os rerun = 'hit' hit = 'hit' if os.path.exists(arname) else None load = 'load' if os.path.exists(acname) else None # special cases if archive == null_archive: rerun, hit, load = None, None, None elif archive == dict_archive: hit, load = None, None ar = archive(arname, serialized=True, cached=False) runme(ar, hit) #FIXME: numpy.array fails on any 'rerun' of runme below if not nprun: runme(ar, rerun) if delete: remove(arname) if not nprun: ac = archive(acname, serialized=True, cached=True) runme(ac, load) runme(ac, 'hit') if delete: remove(acname) return def test_archives(): if not nprun: count = 0 for archive in archives: _test_cache(archive, count, delete=False) count += 1 count = 0 for archive in archives: _test_cache(archive, count, delete=True) count += 1 if __name__ == '__main__': test_archives() uqfoundation-klepto-69cd6ce/klepto/tests/test_chaining.py000066400000000000000000000022321455531556400240740ustar00rootroot00000000000000#!/usr/bin/env python # # Author: Mike McKerns (mmckerns @caltech and @uqfoundation) # Copyright (c) 2013-2016 California Institute of Technology. # Copyright (c) 2016-2024 The Uncertainty Quantification Foundation. # License: 3-clause BSD. The full license text is available at: # - https://github.com/uqfoundation/klepto/blob/master/LICENSE import sys from klepto.keymaps import * from klepto.crypto import * h = hashmap(algorithm='md5') p = picklemap(serializer='dill') hp = p + h def test_chaining(): assert p(1) == pickle(1, serializer='dill') assert h(1) == 'c4ca4238a0b923820dcc509a6f75849b' if sys.version_info[1] < 8: assert hp(1) == 'a2ed37e4f2f0ccf8be170d8c31c711b2' else: #XXX: because 3.x returns b'', 2.x returns '', and 3.8 is weird assert hp(1) == 'bfac8a39dc4b0d616a0805a453698556' assert h(p(1)) == hp(1) assert hp.inner(1) == p(1) assert hp.outer(1) == h(1) assert bool(h.inner) == False assert bool(p.inner) == False assert bool(hp.inner) == True assert bool(h.outer) == False assert bool(p.outer) == False assert bool(hp.outer) == True if __name__ == '__main__': test_chaining() uqfoundation-klepto-69cd6ce/klepto/tests/test_crypto.py000066400000000000000000000047531455531556400236460ustar00rootroot00000000000000#!/usr/bin/env python # # Author: Mike McKerns (mmckerns @caltech and @uqfoundation) # Copyright (c) 2013-2016 California Institute of Technology. # Copyright (c) 2016-2024 The Uncertainty Quantification Foundation. # License: 3-clause BSD. The full license text is available at: # - https://github.com/uqfoundation/klepto/blob/master/LICENSE from klepto.crypto import * from klepto.tools import _b from klepto.keymaps import * def test_encoding(): assert string('1') == '1' assert string('1', encoding='repr') == "'1'" x = [1,2,3,'4',"'5'", min] assert hash(x, 'sha1') == '3bdd73e79be4277dcb874d193b8dd08a46bc6885' assert pickle(x) == string(x, 'repr') assert string(x) == '[1, 2, 3, \'4\', "\'5\'", ]' assert string(x, encoding='repr') == '[1, 2, 3, \'4\', "\'5\'", ]' assert string(x, encoding='utf_8') == _b('[1, 2, 3, \'4\', "\'5\'", ]') # some encodings 'missing' from klepto in python 3.x (due to bytes madness) if 'unicode' in encodings(): assert string(x, encoding='unicode') == unicode('[1, 2, 3, \'4\', "\'5\'", ]') if 'zlib_codec' in encodings(): assert string(x, encoding='zlib_codec') == 'x\x9c\x8b6\xd4Q0\xd2Q0\xd6QP7Q\xd7QPR7UW\xd2Q\xb0I*\xcd\xcc)\xd1\xcd\xccSH+\xcdK.\xc9\xcc\xcfS\xc8\xcd\xcc\xb3\x8b\x05\x00\xf6(\x0c\x9c' if 'hex_codec' in encodings(): assert string(x, encoding='hex_codec') == '5b312c20322c20332c202734272c2022273527222c203c6275696c742d696e2066756e6374696f6e206d696e3e5d' s = stringmap() assert s(x) == '([1, 2, 3, \'4\', "\'5\'", ],)' s = stringmap(encoding='utf_8') assert s(x) == _b('([1, 2, 3, \'4\', "\'5\'", ],)') # some encodings 'missing' from klepto in python 3.x (due to bytes madness) if 'unicode' in encodings(): s = stringmap(encoding='unicode') assert s(x) == unicode('([1, 2, 3, \'4\', "\'5\'", ],)') if 'zlib_codec' in encodings(): s = stringmap(encoding='zlib_codec') assert s(x) == 'x\x9c\xd3\x886\xd4Q0\xd2Q0\xd6QP7Q\xd7QPR7UW\xd2Q\xb0I*\xcd\xcc)\xd1\xcd\xccSH+\xcdK.\xc9\xcc\xcfS\xc8\xcd\xcc\xb3\x8b\xd5\xd1\x04\x00\x17\x99\r\x19' if 'hex_codec' in encodings(): s = stringmap(encoding='hex_codec') assert s(x) == '285b312c20322c20332c202734272c2022273527222c203c6275696c742d696e2066756e6374696f6e206d696e3e5d2c29' if __name__ == '__main__': test_encoding() uqfoundation-klepto-69cd6ce/klepto/tests/test_frame.py000066400000000000000000000042331455531556400234110ustar00rootroot00000000000000#!/usr/bin/env python # # Author: Mike McKerns (mmckerns @caltech and @uqfoundation) # Copyright (c) 2019-2024 The Uncertainty Quantification Foundation. # License: 3-clause BSD. The full license text is available at: # - https://github.com/uqfoundation/klepto/blob/master/LICENSE import klepto as kl def test_roundtrip(archive): db = archive try: db_ = db.__type__.from_frame(db.to_frame()) except ValueError: db_ = db assert db_ == db assert db_.archive == db.archive assert db_.archive.state == db.archive.state assert db_.__type__ == db.__type__ def test_dict_archive(): d = kl.archives.dict_archive('foo', dict(a=1,b=2,c=3), cached=True) d.dump() test_roundtrip(d) def test_null_archive(): d = kl.archives.null_archive('foo', dict(a=1,b=2,c=3), cached=True) d.dump() test_roundtrip(d) def test_dir_archive(): d = kl.archives.dir_archive('foo', dict(a=1,b=2,c=3), cached=True) d.dump() test_roundtrip(d) def test_file_archive(): d = kl.archives.file_archive('foo.pkl', dict(a=1,b=2,c=3), cached=True) d.dump() test_roundtrip(d) def test_sql_archive(): d = kl.archives.sql_archive(None, dict(a=1,b=2,c=3), cached=True) d.dump() test_roundtrip(d) def test_sqltable_archive(): d = kl.archives.sqltable_archive(None, dict(a=1,b=2,c=3), cached=True) d.dump() test_roundtrip(d) def test_hdf_archvie(): d = kl.archives.hdf_archive('foo.h5', dict(a=1,b=2,c=3), cached=True) d.dump() test_roundtrip(d) def test_hdfdir_archive(): d = kl.archives.hdfdir_archive('bar', dict(a=1,b=2,c=3), cached=True) d.dump() test_roundtrip(d) def _cleanup(): import os import pox try: os.remove('foo.pkl') except: pass try: os.remove('foo.h5') except: pass try: pox.rmtree('foo') except: pass try: pox.rmtree('bar') except: pass return if __name__ == '__main__': test_dict_archive() test_null_archive() test_dir_archive() test_file_archive() test_sql_archive() test_sqltable_archive() try: test_hdf_archvie() test_hdfdir_archive() except ImportError: pass _cleanup() #EOF uqfoundation-klepto-69cd6ce/klepto/tests/test_hdf.py000066400000000000000000000071331455531556400230620ustar00rootroot00000000000000#!/usr/bin/env python # # Author: Mike McKerns (mmckerns @caltech and @uqfoundation) # Copyright (c) 2013-2016 California Institute of Technology. # Copyright (c) 2016-2024 The Uncertainty Quantification Foundation. # License: 3-clause BSD. The full license text is available at: # - https://github.com/uqfoundation/klepto/blob/master/LICENSE from klepto.safe import lru_cache as memoized from random import choice, seed N = 100 def _test_cache(cache, keymap=None, maxsize=50, rangelimit=10, tries=N): @memoized(maxsize=maxsize, cache=cache, keymap=keymap) def f(x, y): return 3*x+y domain = list(range(rangelimit)) domain += [float(i) for i in domain] for i in range(tries): r = f(choice(domain), choice(domain)) f.dump() return f def _cleanup(): import os import pox try: os.remove('memo.hdf5') except: pass try: os.remove('xxxx.hdf5') except: pass try: os.remove('memo.h5') except: pass try: os.remove('xxxx.h5') except: pass try: pox.rmtree('memoq') except: pass try: pox.rmtree('memor') except: pass try: pox.rmtree('memos') except: pass try: pox.rmtree('memot') except: pass return from klepto.archives import * from klepto.keymaps import keymap, hashmap, stringmap, picklemap from klepto.keymaps import SENTINEL, NOSENTINEL def test_combinations(): seed(1234) # random seed #XXX: archive/cache should allow scalar and list, also dict (as new table) ? dicts = [ {}, {'a':1}, {'a':[1,2]}, {'a':{'x':3}}, ] init = dicts[0] archives = [ hdf_archive('memo.hdf5',init,serialized=True,meta=False), hdf_archive('memo.h5',init,serialized=False,meta=False), hdf_archive('xxxx.hdf5',init,serialized=True,meta=True), hdf_archive('xxxx.h5',init,serialized=False,meta=True), hdfdir_archive('memoq',init,serialized=False,meta=False), hdfdir_archive('memor',init,serialized=True,meta=False), hdfdir_archive('memos',init,serialized=False,meta=True), hdfdir_archive('memot',init,serialized=True,meta=True), ] if tuple(int(i) for i in h5py.__version__.split('.',2)) < (3,0,0): #FIXME: hdfdir_archive fails with serialized=False in python 3.x archives = archives[:4] + archives[5::2] maps = [ None, keymap(typed=False, flat=True, sentinel=NOSENTINEL), keymap(typed=False, flat=False, sentinel=NOSENTINEL), keymap(typed=True, flat=False, sentinel=NOSENTINEL), hashmap(typed=False, flat=True, sentinel=NOSENTINEL), hashmap(typed=False, flat=False, sentinel=NOSENTINEL), hashmap(typed=True, flat=True, sentinel=NOSENTINEL), hashmap(typed=True, flat=False, sentinel=NOSENTINEL), stringmap(typed=False, flat=True, sentinel=NOSENTINEL), stringmap(typed=False, flat=False, sentinel=NOSENTINEL), stringmap(typed=True, flat=True, sentinel=NOSENTINEL), stringmap(typed=True, flat=False, sentinel=NOSENTINEL), picklemap(typed=False, flat=True, sentinel=NOSENTINEL), picklemap(typed=False, flat=False, sentinel=NOSENTINEL), picklemap(typed=True, flat=True, sentinel=NOSENTINEL), picklemap(typed=True, flat=False, sentinel=NOSENTINEL), ] for mapper in maps: #print (mapper) func = [_test_cache(cache, mapper) for cache in archives] _cleanup() for f in func: #print (f.info()) assert f.info().hit + f.info().miss + f.info().load == N if __name__ == '__main__': try: import h5py test_combinations() except ImportError: print("to test hdf, install h5py") uqfoundation-klepto-69cd6ce/klepto/tests/test_ignore.py000066400000000000000000000174371455531556400236140ustar00rootroot00000000000000#!/usr/bin/env python # # Author: Mike McKerns (mmckerns @caltech and @uqfoundation) # Copyright (c) 2013-2016 California Institute of Technology. # Copyright (c) 2016-2024 The Uncertainty Quantification Foundation. # License: 3-clause BSD. The full license text is available at: # - https://github.com/uqfoundation/klepto/blob/master/LICENSE import sys from functools import partial from klepto.keymaps import hashmap from klepto import NULL from klepto import signature, keygen from klepto import _keygen, isvalid from klepto.tools import IS_PYPY def bar(x,y,z,a=1,b=2,*args): return x+y+z+a+b def test_signature(): s = signature(bar) assert s == (('x', 'y', 'z', 'a', 'b'), {'a': 1, 'b': 2}, 'args', '') # a partial with a 'fixed' x, thus x is 'unsettable' as a keyword p = partial(bar, 0) s = signature(p) assert s == (('y', 'z', 'a', 'b'), {'a': 1, '!x': 0, 'b': 2}, 'args', '') ''' >>> p(0,1) 4 >>> p(0,1,2,3,4,5) 6 ''' # a partial where y is 'unsettable' as a positional argument p = partial(bar, y=10) s = signature(p) assert s == (('x', '!y', 'z', 'a', 'b'), {'a': 1, 'y': 10, 'b': 2}, 'args', '') ''' >>> p(0,1,2) Traceback (most recent call last): File "", line 1, in TypeError: bar() got multiple values for keyword argument 'y' >>> p(0,z=2) 15 >>> p(0,y=1,z=2) 6 ''' ################################################################# # test _keygen def test_keygen(): # a partial with a 'fixed' x, and positionally 'unsettable' b p = partial(bar, 0,b=10) s = signature(p) assert s == (('y', 'z', 'a', '!b'), {'a': 1, '!x': 0, 'b': 10}, 'args', '') ignored = (0,1,3,5,'*','b','c') user_args = ('0','1','2','3','4','5','6') user_kwds = {'a':'10','b':'20','c':'30','d':'40'} key_args,key_kwds = _keygen(p, ignored, *user_args, **user_kwds) assert key_args == () assert key_kwds == {'a': '2', 'c': NULL, 'b': NULL, 'd': '40', 'y': NULL, 'z': NULL} ignored = (0,1,3,5,'**','b','c') user_args = ('0','1','2','3','4','5','6') user_kwds = {'a':'10','b':'20','c':'30','d':'40'} key_args,key_kwds = _keygen(p, ignored, *user_args, **user_kwds) assert key_args == ('4', NULL, '6') assert key_kwds == {'a': '2', 'b': NULL, 'y': NULL, 'z': NULL} ignored = ('*','**') user_args = ('0','1','2','3','4','5','6') user_kwds = {'a':'10','b':'20','c':'30','d':'40'} key_args,key_kwds = _keygen(p, ignored, *user_args, **user_kwds) assert key_args == () assert key_kwds == {'a': '2', 'b': '3', 'y': '0', 'z': '1'} ignored = (0,2) user_args = ('0','1','2','3','4','5','6') user_kwds = {'a':'10','b':'20','c':'30','d':'40'} key_args,key_kwds = _keygen(p, ignored, *user_args, **user_kwds) assert key_args == ('4', '5', '6') assert key_kwds == {'a': NULL, 'c': '30', 'b': '3', 'd':'40', 'y': NULL, 'z': '1'} ignored = (0,) user_args = ('0','1','2','3','4','5','6') user_kwds = {'a':'10','b':'20','c':'30','d':'40','y':50} key_args,key_kwds = _keygen(p, ignored, *user_args, **user_kwds) assert key_args == ('4', '5', '6') assert key_kwds == {'a': '2', 'c': '30', 'b': '3', 'd':'40', 'y': NULL, 'z': '1'} ignored = ('a','y','c') user_args = ('0','1','2','3','4','5','6') user_kwds = {'a':'10','b':'20','c':'30','d':'40','y':50} key_args,key_kwds = _keygen(p, ignored, *user_args, **user_kwds) assert key_args == ('4', '5', '6') assert key_kwds == {'a': NULL, 'c': NULL, 'b': '3', 'd':'40', 'y': NULL, 'z': '1'} ignored = (1,5,'a','y','c') user_args = ('0','1') user_kwds = {} key_args,key_kwds = _keygen(p, ignored, *user_args, **user_kwds) assert key_args == () assert key_kwds == {'a': NULL, 'y': NULL, 'b': 10, 'z': NULL} #XXX: c? ignored = (1,5,'a','y','c') user_args = () user_kwds = {'c':'30','d':'40','y':50} key_args,key_kwds = _keygen(p, ignored, *user_args, **user_kwds) assert key_args == () assert key_kwds == {'a': NULL, 'y': NULL, 'c': NULL, 'd': '40', 'b': 10, 'z': NULL} ignored = (1,5,'a','c') user_args = ('0','1') user_kwds = {} key_args,key_kwds = _keygen(p, ignored, *user_args, **user_kwds) assert key_args == () assert key_kwds == {'a': NULL, 'y': '0', 'b': 10, 'z': NULL} #XXX: c? ignored = () user_args = ('0',) user_kwds = {'c':'30'} key_args,key_kwds = _keygen(p, ignored, *user_args, **user_kwds) assert key_args == () assert key_kwds == {'a': 1, 'y': '0', 'b': 10, 'c': '30'} ################################################################# @keygen('x','**') def foo(x,y,z=2): return x+y+z def test_keygen_foo(): assert foo(0,1,2) == ('x', NULL, 'y', 1, 'z', 2) assert foo.valid() == True assert foo(10,1,2) == ('x', NULL, 'y', 1, 'z', 2) assert foo(0,1) == ('x', NULL, 'y', 1, 'z', 2) assert foo(0,1,3) == ('x', NULL, 'y', 1, 'z', 3) assert foo(0,1,r=3) == ('x', NULL, 'y', 1, 'z', 2) assert foo.valid() == False assert foo(0,1,x=1) == ('x', NULL, 'y', 1, 'z', 2) assert foo.valid() == False res2 = ('x', NULL, 'y', 2, 'z', 10) assert foo(10,y=2,z=10) == res2 assert foo.valid() == True res1 = ('x', NULL, 'y', 1, 'z', 10) assert foo(0,1,z=10) == res1 assert foo.valid() == True assert foo.call() == 11 h = hashmap(algorithm='md5') foo.register(h) # hash randomization from klepto.crypto import hash _hash1 = hash(res1, 'md5') _hash2 = hash(res2, 'md5') assert foo(0,1,z=10) == _hash1 assert str(foo.keymap()) == str(h) assert foo.key() == _hash1 assert foo(10,y=1,z=10) == _hash1 assert foo(10,y=2,z=10) == _hash2 ################################################################# # test special cases (builtins) for signature, isvalid, _keygen def add(x,y): return x+y def test_special(): p = partial(add, 0,x=0) p2 = partial(add, z=0) p3 = partial(add, 0) if IS_PYPY: # builtins in PYPY are python functions if hex(sys.hexversion) < '0x3080cf0': base, exp, mod = 'base', 'exponent', 'modulus' else: base, exp, mod = 'base', 'exp', 'mod' assert signature(pow, safe=True) == ((base, exp, mod), {mod: None}, '', '') else: assert signature(pow, safe=True) == (None, None, None, None) assert signature(p, safe=True) == (None, None, None, None) assert signature(p2, safe=True) == (('x', 'y'), {'z': 0}, '', '') assert signature(p3, safe=True) == (('y',), {'!x': 0}, '', '') if IS_PYPY: # PYPY bug in ArgSpec for min, so use pow assert isvalid(pow, 0,1) == True assert isvalid(pow, 0) == False assert isvalid(pow) == False else: # python >= 3.5 bug in ArgSpec for pow, so use min assert isvalid(min, 0,1) == True assert isvalid(min, 0) == False assert isvalid(min) == False assert isvalid(p, 0,1) == False assert isvalid(p, 0) == False assert isvalid(p) == False assert isvalid(p2, 0,1) == False assert isvalid(p2, 0) == False assert isvalid(p2) == False assert isvalid(p3, 0,1) == False assert isvalid(p3, 0) == True assert isvalid(p3) == False assert _keygen(p3, [], 0) == ((), {'y': 0}) assert _keygen(p2, [], 0) == ((), {'x': 0, 'z': 0}) assert _keygen(p, [], 0) == ((0,), {}) assert _keygen(min, [], x=0,y=1) == ((), {'y': 1, 'x': 0}) assert _keygen(min, [], 0,1) == ((0,1), {}) assert _keygen(min, [], 0) == ((0,), {}) assert _keygen(min, 'x', 0) == ((0,), {}) assert _keygen(min, ['x','y'], 0) == ((0,), {}) assert _keygen(min, [0,1], 0) == ((NULL,), {}) if IS_PYPY else ((0,), {}) assert _keygen(min, ['*'], 0) == ((), {}) if IS_PYPY else ((0,), {}) if __name__ == '__main__': test_signature() test_keygen() test_keygen_foo() test_special() uqfoundation-klepto-69cd6ce/klepto/tests/test_keymaps.py000066400000000000000000000070011455531556400237640ustar00rootroot00000000000000#!/usr/bin/env python # # Author: Mike McKerns (mmckerns @caltech and @uqfoundation) # Copyright (c) 2013-2016 California Institute of Technology. # Copyright (c) 2016-2024 The Uncertainty Quantification Foundation. # License: 3-clause BSD. The full license text is available at: # - https://github.com/uqfoundation/klepto/blob/master/LICENSE from klepto.keymaps import * from dill import dumps, loads args = (1,2); kwds = {"a":3, "b":4} def test_keymap(): encode = keymap(typed=False, flat=True, sentinel=NOSENTINEL) assert encode(*args, **kwds) == (1, 2, 'a', 3, 'b', 4) encode = keymap(typed=False, flat=False, sentinel=NOSENTINEL) assert encode(*args, **kwds) == (args, kwds) encode = keymap(typed=True, flat=True, sentinel=NOSENTINEL) assert encode(*args, **kwds) == (1, 2, 'a', 3, 'b', 4, type(1), type(2), type(3), type(4)) encode = keymap(typed=True, flat=False, sentinel=NOSENTINEL) assert encode(*args, **kwds) == (args, kwds, (type(1), type(2)), (type(3), type(4))) def test_hashmap(): encode = hashmap(typed=False, flat=True, sentinel=NOSENTINEL) assert encode(*args, **kwds) == hash((1, 2, 'a', 3, 'b', 4)) #encode = hashmap(typed=False, flat=False, sentinel=NOSENTINEL) #assert encode(*args, **kwds) == TypeError("unhashable type: 'dict'") encode = hashmap(typed=True, flat=True, sentinel=NOSENTINEL) assert encode(*args, **kwds) == hash((1, 2, 'a', 3, 'b', 4, type(1), type(2), type(3), type(4))) #encode = hashmap(typed=True, flat=False, sentinel=NOSENTINEL) #assert encode(*args, **kwds) == TypeError("unhashable type: 'dict'") def test_stringmap(): encode = stringmap(typed=False, flat=True, sentinel=NOSENTINEL) assert encode(*args, **kwds) == "(1, 2, 'a', 3, 'b', 4)" encode = stringmap(typed=False, flat=False, sentinel=NOSENTINEL) assert eval(encode(*args, **kwds)) == (args, kwds) #res = encode(*args, **kwds) #assert res in ("({}, {})".format(str(args),_kwds), "({}, {})".format(str(args),kwds_)) encode = stringmap(typed=True, flat=True, sentinel=NOSENTINEL) assert encode(*args, **kwds) == str( (1, 2, 'a', 3, 'b', 4, type(1), type(2), type(3), type(4)) ) encode = stringmap(typed=True, flat=False, sentinel=NOSENTINEL) assert eval(encode(*args, **kwds).replace(str((type(1), type(2))), "''")) == (args, kwds, '', '') def test_picklemap(): encode = picklemap(typed=False, flat=True, serializer='dill') assert encode(*args, **kwds) == dumps((1, 2, 'a', 3, 'b', 4)) encode = picklemap(typed=False, flat=False, serializer='dill') assert loads(encode(*args, **kwds)) == loads(dumps((args, kwds))) encode = picklemap(typed=True, flat=True, serializer='dill') assert encode(*args, **kwds) == dumps( (1, 2, 'a', 3, 'b', 4, type(1), type(2), type(3), type(4)) ) encode = picklemap(typed=True, flat=False, serializer='dill') assert loads(encode(*args, **kwds)) == loads(dumps( (args, kwds, (type(1), type(2)), (type(3), type(4))) )) def test_stub_decode(): key = [1,2,3,4,5] from klepto.keymaps import _stub_decoder k = picklemap(serializer='pickle') assert _stub_decoder(k)(k(key))[0] == key assert _stub_decoder('pickle')(k(key))[0] == key k = picklemap() assert _stub_decoder(k)(k(key))[0] == key assert _stub_decoder('repr')(k(key))[0] == key k = keymap() assert _stub_decoder(k)(k(key))[0] == key assert _stub_decoder(None)(k(key))[0] == key if __name__ == '__main__': test_keymap() test_hashmap() test_stringmap() test_picklemap() #test_stub_decode() uqfoundation-klepto-69cd6ce/klepto/tests/test_pickles.py000066400000000000000000000012111455531556400237420ustar00rootroot00000000000000#!/usr/bin/env python # # Author: Mike McKerns (mmckerns @caltech and @uqfoundation) # Copyright (c) 2013-2016 California Institute of Technology. # Copyright (c) 2016-2024 The Uncertainty Quantification Foundation. # License: 3-clause BSD. The full license text is available at: # - https://github.com/uqfoundation/klepto/blob/master/LICENSE import dill import klepto @klepto.lru_cache() def squared(x): return x**2 squared(2) squared(4) squared(6) def test_pickles(): _s = dill.loads(dill.dumps(squared)) assert _s.lookup(4) == 16 assert squared.__cache__() == _s.__cache__() if __name__ == '__main__': test_pickles() uqfoundation-klepto-69cd6ce/klepto/tests/test_readwrite.py000066400000000000000000000063501455531556400243070ustar00rootroot00000000000000#!/usr/bin/env python # # Author: Mike McKerns (mmckerns @caltech and @uqfoundation) # Copyright (c) 2013-2016 California Institute of Technology. # Copyright (c) 2016-2024 The Uncertainty Quantification Foundation. # License: 3-clause BSD. The full license text is available at: # - https://github.com/uqfoundation/klepto/blob/master/LICENSE from klepto.archives import dir_archive from pox import rmtree def test_foo(): # start fresh rmtree('foo', ignore_errors=True) d = dir_archive('foo', cached=False) key = '1234TESTMETESTMETESTME1234' d._mkdir(key) #XXX: repeat mkdir does nothing, should it clear? I think not. _dir = d._mkdir(key) assert d._getdir(key) == _dir d._rmdir(key) # with _pickle x = [1,2,3,4,5] d._fast = True d[key] = x assert d[key] == x d._rmdir(key) # with dill d._fast = False d[key] = x assert d[key] == x d._rmdir(key) # with import d._serialized = False d[key] = x assert d[key] == x d._rmdir(key) d._serialized = True try: import numpy as np y = np.array(x) # with _pickle d._fast = True d[key] = y assert all(d[key] == y) d._rmdir(key) # with dill d._fast = False d[key] = y assert all(d[key] == y) d._rmdir(key) # with import d._serialized = False d[key] = y assert all(d[key] == y) d._rmdir(key) d._serialized = True except ImportError: pass # clean up rmtree('foo') # check archiving basic stuff def check_basic(archive): d = archive d['a'] = 1 d['b'] = '1' d['c'] = min squared = lambda x:x**2 d['d'] = squared d['e'] = None assert d['a'] == 1 assert d['b'] == '1' assert d['c'] == min assert d['d'](2) == squared(2) assert d['e'] == None return # check archiving numpy stuff def check_numpy(archive): try: import numpy as np except ImportError: return d = archive x = np.array([1,2,3,4,5]) y = np.arange(1000) t = np.dtype([('int32',np.int32),('float32',np.float32)]) d['a'] = x d['b'] = y d['c'] = np.inf d['d'] = np.ptp d['e'] = t assert all(d['a'] == x) assert all(d['b'] == y) assert d['c'] == np.inf assert d['d'](x) == np.ptp(x) assert d['e'] == t return # FIXME: add tests for classes and class instances as values # FIXME: add tests for non-string keys (e.g. d[1234] = 'hello') def test_archive(): # try some of the different __init__ archive = dir_archive(cached=False) check_basic(archive) check_numpy(archive) #rmtree('memo') archive = dir_archive(cached=False,fast=True) check_basic(archive) check_numpy(archive) #rmtree('memo') archive = dir_archive(cached=False,compression=3) check_basic(archive) check_numpy(archive) #rmtree('memo') archive = dir_archive(cached=False,memmode='r+') check_basic(archive) check_numpy(archive) #rmtree('memo') archive = dir_archive(cached=False,serialized=False) check_basic(archive) #check_numpy(archive) #FIXME: see issue #53 rmtree('memo') if __name__ == '__main__': test_foo() test_archive() uqfoundation-klepto-69cd6ce/klepto/tests/test_rounding.py000066400000000000000000000047051455531556400241500ustar00rootroot00000000000000#!/usr/bin/env python # # Author: Mike McKerns (mmckerns @caltech and @uqfoundation) # Copyright (c) 2013-2016 California Institute of Technology. # Copyright (c) 2016-2024 The Uncertainty Quantification Foundation. # License: 3-clause BSD. The full license text is available at: # - https://github.com/uqfoundation/klepto/blob/master/LICENSE from klepto.rounding import * def test_deep_round(): @deep_round(tol=1) def add(x,y): return x+y result = add(2.54, 5.47) assert result == 8.0 # rounds each float, regardless of depth in an object result = add([2.54, 'x'],[5.47, 'y']) assert result == [2.5, 'x', 5.5, 'y'] # rounds each float, regardless of depth in an object result = add([2.54, 'x'],[5.47, [8.99, 'y']]) assert result == [2.5, 'x', 5.5, [9.0, 'y']] def test_simple_round(): @simple_round(tol=1) def add(x,y): return x+y result = add(2.54, 5.47) assert result == 8.0 # does not round elements of iterables, only rounds at the top-level result = add([2.54, 'x'],[5.47, 'y']) assert result == [2.54, 'x', 5.4699999999999998, 'y'] # does not round elements of iterables, only rounds at the top-level result = add([2.54, 'x'],[5.47, [8.99, 'y']]) assert result == [2.54, 'x', 5.4699999999999998, [8.9900000000000002, 'y']] def test_shallow_round(): @shallow_round(tol=1) def add(x,y): return x+y result = add(2.54, 5.47) assert result == 8.0 # rounds each float, at the top-level or first-level of each object. result = add([2.54, 'x'],[5.47, 'y']) assert result == [2.5, 'x', 5.5, 'y'] # rounds each float, at the top-level or first-level of each object. result = add([2.54, 'x'],[5.47, [8.99, 'y']]) assert result == [2.5, 'x', 5.5, [8.9900000000000002, 'y']] # rounding integrated with key generation from klepto import keygen, NULL def test_keygen(): @keygen('x',2,tol=2) def add(w,x,y,z): return x+y+z+w assert add(1.11111,2.222222,3.333333,4.444444) == ('w', 1.11, 'x', NULL, 'y', NULL, 'z', 4.44) assert add.call() == 11.111108999999999 assert add(1.11111,2.2229,100,4.447) == ('w', 1.11, 'x', NULL, 'y', NULL, 'z', 4.45) assert add.call() == 107.78101 assert add(1.11111,100,100,4.441) == ('w', 1.11, 'x', NULL, 'y', NULL, 'z', 4.44) assert add.call() == 205.55211 if __name__ == '__main__': test_deep_round() test_simple_round() test_shallow_round() test_keygen() uqfoundation-klepto-69cd6ce/klepto/tests/test_validate.py000066400000000000000000000160451455531556400241140ustar00rootroot00000000000000#!/usr/bin/env python # # Author: Mike McKerns (mmckerns @caltech and @uqfoundation) # Copyright (c) 2013-2016 California Institute of Technology. # Copyright (c) 2016-2024 The Uncertainty Quantification Foundation. # License: 3-clause BSD. The full license text is available at: # - https://github.com/uqfoundation/klepto/blob/master/LICENSE import sys from functools import partial from klepto import validate def foo(x,y,z,a=1,b=2): return x+y+z+a+b class Bar(object): def foo(self, x,y,z,a=1,b=2): return foo(x,y,z,a=a,b=b) def __call__(self, x,y,z,a=1,b=2): #NOTE: *args, **kwds): return foo(x,y,z,a=a,b=b) def test_foo(): p = foo try: res1 = p(1,2,3,4,b=5) res2 = Exception() except: res1,res2 = sys.exc_info()[:2] try: re_1 = validate(p,1,2,3,4,b=5) re_2 = Exception() except: re_1,re_2 = sys.exc_info()[:2] assert re_1 is None try: res1 = p() res2 = Exception() except: res1,res2 = sys.exc_info()[:2] try: re_1 = validate(p) re_2 = Exception() except: re_1,re_2 = sys.exc_info()[:2] assert res1 == re_1 #XXX: "foo() missing 3 required positional arguments" try: res1 = p(1,2,3,4,r=5) res2 = Exception() except: res1,res2 = sys.exc_info()[:2] try: re_1 = validate(p,1,2,3,4,r=5) re_2 = Exception() except: re_1,re_2 = sys.exc_info()[:2] assert res1 == re_1 #XXX: "foo() got unexpected keyword argument 'r'" def test_Bar_foo(): p = Bar().foo try: res1 = p(1,2,3,4,b=5) res2 = Exception() except: res1,res2 = sys.exc_info()[:2] try: re_1 = validate(p,1,2,3,4,b=5) re_2 = Exception() except: re_1,re_2 = sys.exc_info()[:2] assert re_1 is None try: res1 = p() res2 = Exception() except: res1,res2 = sys.exc_info()[:2] try: re_1 = validate(p) re_2 = Exception() except: re_1,re_2 = sys.exc_info()[:2] assert res1 == re_1 #XXX: "foo() missing 3 required positional arguments" try: res1 = p(1,2,3,4,r=5) res2 = Exception() except: res1,res2 = sys.exc_info()[:2] try: re_1 = validate(p,1,2,3,4,r=5) re_2 = Exception() except: re_1,re_2 = sys.exc_info()[:2] assert res1 == re_1 #XXX: "foo() got unexpected keyword argument 'r'" def test_Bar(): p = Bar() try: res1 = p(1,2,3,4,b=5) res2 = Exception() except: res1,res2 = sys.exc_info()[:2] try: re_1 = validate(p,1,2,3,4,b=5) re_2 = Exception() except: re_1,re_2 = sys.exc_info()[:2] assert re_1 is None try: res1 = p() res2 = Exception() except: res1,res2 = sys.exc_info()[:2] try: re_1 = validate(p) re_2 = Exception() except: re_1,re_2 = sys.exc_info()[:2] assert res1 == re_1 #XXX: "foo() missing 3 required positional arguments" try: res1 = p(1,2,3,4,r=5) res2 = Exception() except: res1,res2 = sys.exc_info()[:2] try: re_1 = validate(p,1,2,3,4,r=5) re_2 = Exception() except: re_1,re_2 = sys.exc_info()[:2] assert res1 == re_1 #XXX: "foo() got unexpected keyword argument 'r'" def test_partial_foo_xy(): p = partial(foo, 0,1) try: res1 = p(1,2,3,4,b=5) res2 = Exception() except: res1,res2 = sys.exc_info()[:2] try: re_1 = validate(p,1,2,3,4,b=5) re_2 = Exception() except: re_1,re_2 = sys.exc_info()[:2] #print(res2) #print(re_2) assert res1 == re_1 #assert str(res2)[:20] == str(re_2)[:20] #XXX: "foo() got multiple values for argument 'b'" try: res1 = p() res2 = Exception() except: res1,res2 = sys.exc_info()[:2] try: re_1 = validate(p) re_2 = Exception() except: re_1,re_2 = sys.exc_info()[:2] #print(res2) #print(re_2) assert res1 == re_1 #assert str(res2)[:21] == str(re_2)[:21] #XXX: "foo() missing 1 required positional argument 'z'" try: res1 = p(1,2,3,4,r=5) res2 = Exception() except: res1,res2 = sys.exc_info()[:2] try: re_1 = validate(p,1,2,3,4,r=5) re_2 = Exception() except: re_1,re_2 = sys.exc_info()[:2] #print(res2) #print(re_2) assert res1 == re_1 #assert str(res2)[:20] == str(re_2)[:20] #XXX: "foo() got unexpected keyword argument 'r'" def test_partial_foo_xx(): p = partial(foo, 0,x=4) try: res1 = p(1,2,3,4,r=5) res2 = Exception() except: res1,res2 = sys.exc_info()[:2] try: re_1 = validate(p,1,2,3,4,r=5) re_2 = Exception() except: re_1,re_2 = sys.exc_info()[:2] #print(res2) #print(re_2) assert res1 == re_1 # assert str(res2)[:25] == str(re_2)[:25] #XXX: "foo() got unexpected keyword argument 'r'" def test_partial_foo_xyzabcde(): p = partial(foo, 0,1,2,3,4,5,6,7) try: res1 = p() res2 = Exception() except: res1,res2 = sys.exc_info()[:2] try: re_1 = validate(p) re_2 = Exception() except: re_1,re_2 = sys.exc_info()[:2] #print(res2) #print(re_2) assert res1 == re_1 #assert str(res2)[:20] == str(re_2)[:20] #XXX: "foo() takes from 3 to 5 positional arguments but 8 were given" def test_partial_foo_xzb(): p = partial(foo, 0,z=1,b=2) try: res1 = p() res2 = Exception() except: res1,res2 = sys.exc_info()[:2] try: re_1 = validate(p) re_2 = Exception() except: re_1,re_2 = sys.exc_info()[:2] #print(res2) #print(re_2) assert res1 == re_1 #assert str(res2)[:21] == str(re_2)[:21] #XXX: "foo() missing 1 required positional argument: 'y'" def test_partial_foo_xr(): p = partial(foo, 0,r=4) try: res1 = p(1) res2 = Exception() except: res1,res2 = sys.exc_info()[:2] try: re_1 = validate(p,1) re_2 = Exception() except: re_1,re_2 = sys.exc_info()[:2] #print(res2) #print(re_2) assert res1 == re_1 assert str(res2)[:24] == str(re_2)[:24] def test_partial_foo_xa(): p = partial(foo, 0,a=2) try: res1 = p(1) res2 = Exception() except: res1,res2 = sys.exc_info()[:2] try: re_1 = validate(p,1) re_2 = Exception() except: re_1,re_2 = sys.exc_info()[:2] #print(res2) #print(re_2) assert res1 == re_1 #assert str(res2)[:21] == str(re_2)[:21] #XXX: "foo() missing 1 required positional argument: 'z'" assert validate(p,1,2) == None #XXX: better return ((1,2),{}) ? ''' >>> p(1,2) 7 ''' if __name__ == '__main__': test_foo() test_Bar_foo() test_Bar() test_partial_foo_xy() test_partial_foo_xx() test_partial_foo_xyzabcde() test_partial_foo_xzb() test_partial_foo_xr() test_partial_foo_xa() uqfoundation-klepto-69cd6ce/klepto/tests/test_workflow.py000066400000000000000000000064701455531556400241760ustar00rootroot00000000000000#!/usr/bin/env python # # Author: Mike McKerns (mmckerns @caltech and @uqfoundation) # Copyright (c) 2013-2016 California Institute of Technology. # Copyright (c) 2016-2024 The Uncertainty Quantification Foundation. # License: 3-clause BSD. The full license text is available at: # - https://github.com/uqfoundation/klepto/blob/master/LICENSE from klepto.keymaps import hashmap from klepto import lru_cache as memoize from klepto import inf_cache from klepto import keygen hasher = hashmap(algorithm='md5') class Adder(object): """A simple class with a memoized method""" @memoize(keymap=hasher, ignore=('self','**')) def __call__(self, x, *args, **kwds): debug = kwds.get('debug', False) if debug: print ('debug:', x, args, kwds) return sum((x,)+args) add = __call__ def test_adder(): add = Adder() assert add(2,0) == 2 assert add(2,0,z=4) == 2 # cached (ignore z) assert add(2,0,debug=False) == 2 # cached (ignore debug) assert add(1,2,debug=False) == 3 assert add(1,2,debug=True) == 3 # cached (ignore debug) assert add(4) == 4 assert add(x=4) == 4 # cached plus = Adder() assert plus(4,debug=True) == 4 # cached (ignore debug) assert plus(2,0,3) == 5 assert plus(2,0,4) == 6 info = add.__call__.info() assert info.hit == 5 assert info.miss == 5 cache = add.__call__.__cache__() assert sorted(cache.values()) == [2,3,4,5,6] # careful... remember we have self as first argument key = add.__call__.key(add,2,0) assert cache[key] == add.__call__.__wrapped__(add,2,0) assert cache[key] == add.__call__.lookup(add,2,0) ###################################################### def test_memoize(): @memoize(keymap=hasher, ignore=('self','**')) def _add(x, *args, **kwds): debug = kwds.get('debug', False) if debug: print ('debug:', x, args, kwds) return sum((x,)+args) _add(2,0) _add(2,0,z=4) _add(2,0,debug=False) _add(1,2,debug=False) _add(1,2,debug=True) _add(4) _add(x=4) _add(4,debug=True) _add(2,0,3) _add(2,0,4) _cache = _add.__cache__() _func = _add.__wrapped__ # do a lookup assert _add.lookup(2,0) == _func(2,0) # generate the key, and do a look-up key = _add.key(2,0) assert _cache[key] == _func(2,0) # look-up the key again, doing a little more work... lookup = keygen('self','**')(_func) lookup.register(hasher) key = lookup(2,0) assert _cache[key] == _func(2,0) # since we have the 'key lookup', let's play with lookup a bit assert lookup.valid() assert lookup.call() == _func(2,0) ###################################################### # more of the same... class Foo(object): @keygen('self') def bar(self, x,y): return x+y class _Foo(object): @inf_cache(ignore='self') def bar(self, x,y): return x+y def test_foo(): fu = Foo() assert fu.bar(1,2) == ('x', 1, 'y', 2) assert Foo.bar(fu,1,2) == ('x', 1, 'y', 2) _fu = _Foo() _fu.bar(1,2) _fu.bar(2,2) _fu.bar(2,3) _fu.bar(1,2) assert len(_fu.bar.__cache__()) == 3 assert _fu.bar.__cache__()[_fu.bar.key(_fu,1,2)] == 3 assert _fu.bar.lookup(_fu,1,2) == 3 if __name__ == '__main__': test_adder() test_memoize() test_foo() uqfoundation-klepto-69cd6ce/klepto/tools.py000066400000000000000000000023531455531556400212570ustar00rootroot00000000000000#!/usr/bin/env python # # Author: Mike McKerns (mmckerns @caltech and @uqfoundation) # Copyright (c) 2013-2016 California Institute of Technology. # Copyright (c) 2016-2024 The Uncertainty Quantification Foundation. # License: 3-clause BSD. The full license text is available at: # - https://github.com/uqfoundation/klepto/blob/master/LICENSE """ Assorted python tools Main functions exported are:: - isiterable: check if an object is iterable """ try: import ctypes # if using `pypy`, pythonapi is not found IS_PYPY = not hasattr(ctypes, 'pythonapi') except ImportError: IS_PYPY = False from collections import namedtuple CacheInfo = namedtuple("CacheInfo", ['hit','miss','load','maxsize','size']) __all__ = ['isiterable'] def isiterable(x): """check if an object is iterable""" #try: # from collections import Iterable # return isinstance(x, Iterable) #except ImportError: try: iter(x) return True except TypeError: return False #return hasattr(x, '__len__') or hasattr(x, '__iter__') def _b(message): """convert string to correct format for buffer object""" import codecs return codecs.latin_1_encode(message)[0] if __name__=='__main__': pass # End of file uqfoundation-klepto-69cd6ce/pyproject.toml000066400000000000000000000002461455531556400211620ustar00rootroot00000000000000[build-system] # Further build requirements come from setup.py via the PEP 517 interface requires = [ "setuptools>=42", ] build-backend = "setuptools.build_meta" uqfoundation-klepto-69cd6ce/setup.cfg000066400000000000000000000001761455531556400200710ustar00rootroot00000000000000[egg_info] #tag_build = .dev0 [bdist_wheel] #python-tag = py3 #plat-name = manylinux_2_24_x86_64 [sdist] #formats=zip,gztar uqfoundation-klepto-69cd6ce/setup.py000066400000000000000000000112231455531556400177550ustar00rootroot00000000000000#!/usr/bin/env python # # Author: Mike McKerns (mmckerns @caltech and @uqfoundation) # Copyright (c) 2013-2016 California Institute of Technology. # Copyright (c) 2016-2024 The Uncertainty Quantification Foundation. # License: 3-clause BSD. The full license text is available at: # - https://github.com/uqfoundation/klepto/blob/master/LICENSE import os import sys # drop support for older python if sys.version_info < (3, 8): unsupported = 'Versions of Python before 3.8 are not supported' raise ValueError(unsupported) # get distribution meta info here = os.path.abspath(os.path.dirname(__file__)) sys.path.append(here) from version import (__version__, __author__, __contact__ as AUTHOR_EMAIL, get_license_text, get_readme_as_rst, write_info_file) LICENSE = get_license_text(os.path.join(here, 'LICENSE')) README = get_readme_as_rst(os.path.join(here, 'README.md')) # write meta info file write_info_file(here, 'klepto', doc=README, license=LICENSE, version=__version__, author=__author__) del here, get_license_text, get_readme_as_rst, write_info_file # check if setuptools is available try: from setuptools import setup from setuptools.dist import Distribution has_setuptools = True except ImportError: from distutils.core import setup Distribution = object has_setuptools = False # build the 'setup' call setup_kwds = dict( name='klepto', version=__version__, description='persistent caching to memory, disk, or database', long_description = README.strip(), author = __author__, author_email = AUTHOR_EMAIL, maintainer = __author__, maintainer_email = AUTHOR_EMAIL, license = 'BSD-3-Clause', platforms = ['Linux', 'Windows', 'Mac'], url = 'https://github.com/uqfoundation/klepto', download_url = 'https://pypi.org/project/klepto/#files', project_urls = { 'Documentation':'http://klepto.rtfd.io', 'Source Code':'https://github.com/uqfoundation/klepto', 'Bug Tracker':'https://github.com/uqfoundation/klepto/issues', }, python_requires = '>=3.8', classifiers = [ 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', 'Intended Audience :: Science/Research', 'License :: OSI Approved :: BSD License', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', 'Programming Language :: Python :: 3.12', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', 'Topic :: Database', 'Topic :: Scientific/Engineering', 'Topic :: Software Development', ], packages = ['klepto','klepto.tests'], package_dir = {'klepto':'klepto', 'klepto.tests':'klepto/tests'}, ) # force python-, abi-, and platform-specific naming of bdist_wheel class BinaryDistribution(Distribution): """Distribution which forces a binary package with platform name""" def has_ext_modules(foo): return True # define dependencies sysversion = sys.version_info[:3] dill_version = 'dill>=0.3.8' pox_version = 'pox>=0.3.4' jsonpickle_version = 'jsonpickle>=0.9.6' cloudpickle_version = 'cloudpickle>=0.5.2' sqlalchemy_version = 'sqlalchemy>=1.4.0' h5py_version = 'h5py>=2.8.0' pandas_version = 'pandas>=0.17.0' # add dependencies depend = [pox_version, dill_version] extras = {'archives': [h5py_version, sqlalchemy_version, pandas_version], 'crypto': [jsonpickle_version, cloudpickle_version]} # update setup kwds if has_setuptools: setup_kwds.update( zip_safe=False, # distclass=BinaryDistribution, install_requires=depend, extras_require=extras, ) # call setup setup(**setup_kwds) # if dependencies are missing, print a warning try: import dill import pox #import jsonpickle #import cloudpickle #import sqlalchemy #import h5py #import pandas except ImportError: print ("\n***********************************************************") print ("WARNING: One of the following dependencies is unresolved:") print (" %s" % dill_version) print (" %s" % pox_version) print (" %s (optional)" % jsonpickle_version) print (" %s (optional)" % cloudpickle_version) print (" %s (optional)" % sqlalchemy_version) print (" %s (optional)" % h5py_version) print (" %s (optional)" % pandas_version) print ("***********************************************************\n") if __name__=='__main__': pass # end of file uqfoundation-klepto-69cd6ce/tox.ini000066400000000000000000000006421455531556400175610ustar00rootroot00000000000000[tox] skip_missing_interpreters= True envlist = py38 py39 py310 py311 py312 py313 pypy38 pypy39 pypy310 [testenv] setenv = PYTHONHASHSEED = 0 recreate = True deps = # numpy # pandas sqlalchemy<2.0.0 whitelist_externals = # bash commands = {envpython} -m pip install . {envpython} klepto/tests/__main__.py {envpython} klepto/tests/cleanup_basic.py uqfoundation-klepto-69cd6ce/version.py000066400000000000000000000061521455531556400203070ustar00rootroot00000000000000#!/usr/bin/env python # # Author: Mike McKerns (mmckerns @caltech and @uqfoundation) # Copyright (c) 2022-2024 The Uncertainty Quantification Foundation. # License: 3-clause BSD. The full license text is available at: # - https://github.com/uqfoundation/klepto/blob/master/LICENSE __version__ = '0.2.5'#.dev0' __author__ = 'Mike McKerns' __contact__ = 'mmckerns@uqfoundation.org' def get_license_text(filepath): "open the LICENSE file and read the contents" try: LICENSE = open(filepath).read() except: LICENSE = '' return LICENSE def get_readme_as_rst(filepath): "open the README file and read the markdown as rst" try: fh = open(filepath) name, null = fh.readline().rstrip(), fh.readline() tag, null = fh.readline(), fh.readline() tag = "%s: %s" % (name, tag) split = '-'*(len(tag)-1)+'\n' README = ''.join((null,split,tag,split,'\n')) skip = False for line in fh: if line.startswith('['): continue elif skip and line.startswith(' http'): README += '\n' + line elif line.startswith('* '): README += line.replace('* ',' - ',1) elif line.startswith('-'): README += line.replace('-','=') + '\n' elif line.startswith('!['): # image alt,img = line.split('](',1) if img.startswith('docs'): # relative path img = img.split('docs/source/',1)[-1] # make is in docs README += '.. image:: ' + img.replace(')','') README += ' :alt: ' + alt.replace('![','') + '\n' #elif ')[http' in line: # alt text link (`text `_) else: README += line skip = line.endswith(':\n') fh.close() except: README = '' return README def write_info_file(dirpath, modulename, **info): """write the given info to 'modulename/__info__.py' info expects: doc: the module's long_description version: the module's version string author: the module's author string license: the module's license contents """ import os infofile = os.path.join(dirpath, '%s/__info__.py' % modulename) header = '''#!/usr/bin/env python # # Author: Mike McKerns (mmckerns @caltech and @uqfoundation) # Copyright (c) 2024 The Uncertainty Quantification Foundation. # License: 3-clause BSD. The full license text is available at: # - https://github.com/uqfoundation/%s/blob/master/LICENSE ''' % modulename #XXX: author and email are hardwired in the header doc = info.get('doc', None) version = info.get('version', None) author = info.get('author', None) license = info.get('license', None) with open(infofile, 'w') as fh: fh.write(header) if doc is not None: fh.write("'''%s'''\n\n" % doc) if version is not None: fh.write("__version__ = %r\n" % version) if author is not None: fh.write("__author__ = %r\n\n" % author) if license is not None: fh.write("__license__ = '''\n%s'''\n" % license) return